Skip to main content

consensus_core/
synchronizer.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3use std::{
4    collections::{BTreeMap, BTreeSet},
5    sync::Arc,
6    time::Duration,
7};
8
9use bytes::Bytes;
10use consensus_config::AuthorityIndex;
11use consensus_types::block::{BlockRef, Round, TransactionIndex};
12use futures::{StreamExt as _, stream::FuturesUnordered};
13use itertools::Itertools as _;
14use mysten_common::{ZipDebugEqIteratorExt, debug_fatal};
15use mysten_metrics::{
16    monitored_future,
17    monitored_mpsc::{Receiver, Sender, channel},
18    monitored_scope,
19};
20use parking_lot::{Mutex, RwLock};
21use rand::{prelude::SliceRandom as _, rngs::ThreadRng};
22use sui_macros::fail_point_async;
23use tap::TapFallible;
24use tokio::{
25    sync::{mpsc::error::TrySendError, oneshot},
26    task::{JoinError, JoinSet},
27    time::{Instant, sleep, sleep_until, timeout},
28};
29use tracing::{debug, info, trace, warn};
30
31use crate::{
32    BlockAPI,
33    block::{ExtendedBlock, SignedBlock, VerifiedBlock},
34    block_verifier::BlockVerifier,
35    commit::CommitIndex,
36    commit_vote_monitor::{CommitVoteMonitor, is_commit_lagging},
37    context::Context,
38    core_thread::CoreThreadDispatcher,
39    dag_state::DagState,
40    error::{ConsensusError, ConsensusResult},
41    network::{ObserverNetworkClient, PeerId, SynchronizerClient, ValidatorNetworkClient},
42    peers_pool::PeersPool,
43    round_tracker::RoundTracker,
44    task::{shutdown_join_set, spawn_blocking},
45    transaction_vote_tracker::TransactionVoteTracker,
46};
47
48/// The number of concurrent fetch blocks requests per authority
49const FETCH_BLOCKS_CONCURRENCY: usize = 5;
50
51/// Timeouts when fetching blocks.
52const FETCH_REQUEST_TIMEOUT: Duration = Duration::from_millis(2_000);
53const FETCH_FROM_PEERS_TIMEOUT: Duration = Duration::from_millis(4_000);
54
55const MAX_AUTHORITIES_TO_FETCH_PER_BLOCK: usize = 2;
56
57// Max number of peers to request missing blocks concurrently in periodic sync.
58const MAX_PERIODIC_SYNC_PEERS: usize = 3;
59
60/// How long commit must be stalled before periodic sync kicks in as fallback.
61const COMMIT_PROGRESS_TIMEOUT: Duration = Duration::from_secs(10);
62
63struct BlocksGuard {
64    map: Arc<InflightBlocksMap>,
65    block_refs: BTreeSet<BlockRef>,
66    peer: PeerId,
67}
68
69impl Drop for BlocksGuard {
70    fn drop(&mut self) {
71        self.map.unlock_blocks(&self.block_refs, self.peer.clone());
72    }
73}
74
75// Keeps a mapping between the missing blocks that have been instructed to be fetched and the peers
76// that are currently fetching them. For a block ref there is a maximum number of peers that can
77// concurrently fetch it. The peer ids that are currently fetching a block are set on the corresponding
78// `BTreeSet` and basically they act as "locks".
79struct InflightBlocksMap {
80    inner: Mutex<BTreeMap<BlockRef, BTreeSet<PeerId>>>,
81}
82
83impl InflightBlocksMap {
84    fn new() -> Arc<Self> {
85        Arc::new(Self {
86            inner: Mutex::new(BTreeMap::new()),
87        })
88    }
89
90    /// Locks the blocks to be fetched for the assigned `peer`. We want to avoid re-fetching the
91    /// missing blocks from too many peers at the same time, thus we limit the concurrency
92    /// per block by attempting to lock per block. If a block is already fetched by the maximum allowed
93    /// number of peers, then the block ref will not be included in the returned set. The method
94    /// returns all the block refs that have been successfully locked and allowed to be fetched.
95    fn lock_blocks(
96        self: &Arc<Self>,
97        missing_block_refs: BTreeSet<BlockRef>,
98        peer: PeerId,
99    ) -> Option<BlocksGuard> {
100        let mut blocks = BTreeSet::new();
101        let mut inner = self.inner.lock();
102
103        for block_ref in missing_block_refs {
104            // check that the number of peers that are already instructed to fetch the block is not
105            // higher than the allowed and the `peer` has not already been instructed to do that.
106            let peers = inner.entry(block_ref).or_default();
107            if peers.len() < MAX_AUTHORITIES_TO_FETCH_PER_BLOCK && !peers.contains(&peer) {
108                assert!(peers.insert(peer.clone()));
109                blocks.insert(block_ref);
110            }
111        }
112
113        if blocks.is_empty() {
114            None
115        } else {
116            Some(BlocksGuard {
117                map: self.clone(),
118                block_refs: blocks,
119                peer,
120            })
121        }
122    }
123
124    /// Unlocks the provided block references for the given `peer`. The unlocking is strict, meaning that
125    /// if this method is called for a specific block ref and peer more times than the corresponding lock
126    /// has been called, it will panic.
127    fn unlock_blocks(self: &Arc<Self>, block_refs: &BTreeSet<BlockRef>, peer: PeerId) {
128        // Now mark all the blocks as fetched from the map
129        let mut blocks_to_fetch = self.inner.lock();
130        for block_ref in block_refs {
131            let peers = blocks_to_fetch
132                .get_mut(block_ref)
133                .expect("Should have found a non empty map");
134
135            assert!(peers.remove(&peer), "Peer should be present!");
136
137            // if the last one then just clean up
138            if peers.is_empty() {
139                blocks_to_fetch.remove(block_ref);
140            }
141        }
142    }
143
144    /// Drops the provided `blocks_guard` which will force to unlock the blocks, and lock now again the
145    /// referenced block refs. The swap is best effort and there is no guarantee that the `peer` will
146    /// be able to acquire the new locks.
147    fn swap_locks(
148        self: &Arc<Self>,
149        blocks_guard: BlocksGuard,
150        peer: PeerId,
151    ) -> Option<BlocksGuard> {
152        let block_refs = blocks_guard.block_refs.clone();
153
154        // Explicitly drop the guard
155        drop(blocks_guard);
156
157        // Now create new guard
158        self.lock_blocks(block_refs, peer)
159    }
160
161    #[cfg(test)]
162    fn num_of_locked_blocks(self: &Arc<Self>) -> usize {
163        let inner = self.inner.lock();
164        inner.len()
165    }
166}
167
168enum Command {
169    FetchBlocks {
170        missing_block_refs: BTreeSet<BlockRef>,
171        peer: PeerId,
172        result: oneshot::Sender<Result<(), ConsensusError>>,
173    },
174    FetchOwnLastBlock,
175    KickOffScheduler,
176    Shutdown {
177        result: oneshot::Sender<()>,
178    },
179}
180
181pub(crate) struct SynchronizerHandle {
182    commands_sender: Sender<Command>,
183    tasks: tokio::sync::Mutex<JoinSet<()>>,
184}
185
186impl SynchronizerHandle {
187    /// Explicitly asks from the synchronizer to fetch the blocks - provided the block_refs set - from
188    /// the peer.
189    pub(crate) async fn fetch_blocks(
190        &self,
191        missing_block_refs: BTreeSet<BlockRef>,
192        peer: PeerId,
193    ) -> ConsensusResult<()> {
194        let (sender, receiver) = oneshot::channel();
195        self.commands_sender
196            .send(Command::FetchBlocks {
197                missing_block_refs,
198                peer,
199                result: sender,
200            })
201            .await
202            .map_err(|_err| ConsensusError::Shutdown)?;
203        receiver.await.map_err(|_err| ConsensusError::Shutdown)?
204    }
205
206    pub(crate) async fn stop(&self) {
207        let (shutdown_sender, shutdown_receiver) = oneshot::channel();
208        let _ = self
209            .commands_sender
210            .send(Command::Shutdown {
211                result: shutdown_sender,
212            })
213            .await;
214        let _ = shutdown_receiver.await;
215
216        let mut tasks = self.tasks.lock().await;
217        shutdown_join_set(&mut tasks).await;
218    }
219
220    #[cfg(test)]
221    /// Creates a mock synchronizer handle for testing
222    pub(crate) fn new_for_test() -> Arc<Self> {
223        let (tx, _rx) = channel("test_synchronizer", 1);
224        Arc::new(Self {
225            commands_sender: tx,
226            tasks: tokio::sync::Mutex::new(JoinSet::new()),
227        })
228    }
229}
230
231/// `Synchronizer` oversees live block synchronization, crucial for node progress. Live synchronization
232/// refers to the process of retrieving missing blocks, particularly those essential for advancing a node
233/// when data from only a few rounds is absent. If a node significantly lags behind the network,
234/// `commit_syncer` handles fetching missing blocks via a more efficient approach. `Synchronizer`
235/// aims for swift catch-up employing two mechanisms:
236///
237/// 1. Explicitly requesting missing blocks from designated authorities via the "block send" path.
238///    This includes attempting to fetch any missing ancestors necessary for processing a received block.
239///    Such requests prioritize the block author, maximizing the chance of prompt retrieval.
240///    A locking mechanism allows concurrent requests for missing blocks from up to two authorities
241///    simultaneously, enhancing the chances of timely retrieval. Notably, if additional missing blocks
242///    arise during block processing, requests to the same authority are deferred to the scheduler.
243///
244/// 2. Periodically requesting missing blocks via a scheduler. This primarily serves to retrieve
245///    missing blocks that were not ancestors of a received block via the "block send" path.
246///    The scheduler operates on either a fixed periodic basis or is triggered immediately
247///    after explicit fetches described in (1), ensuring continued block retrieval if gaps persist.
248///
249/// Additionally to the above, the synchronizer can synchronize and fetch the last own proposed block
250/// from the network peers as best effort approach to recover node from amnesia and avoid making the
251/// node equivocate.
252pub(crate) struct Synchronizer<
253    V: BlockVerifier,
254    D: CoreThreadDispatcher,
255    VC: ValidatorNetworkClient,
256    OC: ObserverNetworkClient,
257> {
258    context: Arc<Context>,
259    commands_receiver: Receiver<Command>,
260    fetch_block_senders: BTreeMap<PeerId, Sender<BlocksGuard>>,
261    core_dispatcher: Arc<D>,
262    commit_vote_monitor: Arc<CommitVoteMonitor>,
263    dag_state: Arc<RwLock<DagState>>,
264    fetch_blocks_scheduler_task: JoinSet<()>,
265    fetch_own_last_block_task: JoinSet<()>,
266    network_client: Arc<SynchronizerClient<VC, OC>>,
267    block_verifier: Arc<V>,
268    transaction_vote_tracker: TransactionVoteTracker,
269    round_tracker: Arc<RwLock<RoundTracker>>,
270    inflight_blocks_map: Arc<InflightBlocksMap>,
271    commands_sender: Sender<Command>,
272    last_changed_commit_index: CommitIndex,
273    last_commit_change_time: Instant,
274    // When commit is not progressing, commit sync fails over to periodic sync for catchup.
275    commit_sync_failover: bool,
276    peers_pool: Arc<PeersPool>,
277}
278
279impl<V, D, VC, OC> Synchronizer<V, D, VC, OC>
280where
281    V: BlockVerifier,
282    D: CoreThreadDispatcher,
283    VC: ValidatorNetworkClient,
284    OC: ObserverNetworkClient,
285{
286    pub(crate) fn start(
287        network_client: Arc<SynchronizerClient<VC, OC>>,
288        context: Arc<Context>,
289        core_dispatcher: Arc<D>,
290        commit_vote_monitor: Arc<CommitVoteMonitor>,
291        block_verifier: Arc<V>,
292        transaction_vote_tracker: TransactionVoteTracker,
293        round_tracker: Arc<RwLock<RoundTracker>>,
294        dag_state: Arc<RwLock<DagState>>,
295        peers_pool: Arc<PeersPool>,
296        sync_last_known_own_block: bool,
297    ) -> Arc<SynchronizerHandle> {
298        let (commands_sender, commands_receiver) =
299            channel("consensus_synchronizer_commands", 1_000);
300        let inflight_blocks_map = InflightBlocksMap::new();
301
302        // Spawn the tasks to fetch the blocks from the others
303        let mut fetch_block_senders = BTreeMap::new();
304        let mut tasks = JoinSet::new();
305
306        // Create fetch tasks for all known peers (validators and observers)
307        // TODO: refactor to update the sender tasks based on the registered/removed pool peers.
308        let known_peers = peers_pool.get_known_peers();
309        for peer in known_peers {
310            let (sender, receiver) =
311                channel("consensus_synchronizer_fetches", FETCH_BLOCKS_CONCURRENCY);
312            let fetch_blocks_from_peer_async = Self::fetch_blocks_from_peer(
313                peer.clone(),
314                network_client.clone(),
315                block_verifier.clone(),
316                transaction_vote_tracker.clone(),
317                commit_vote_monitor.clone(),
318                context.clone(),
319                core_dispatcher.clone(),
320                dag_state.clone(),
321                receiver,
322                commands_sender.clone(),
323                round_tracker.clone(),
324            );
325            tasks.spawn(monitored_future!(fetch_blocks_from_peer_async));
326            fetch_block_senders.insert(peer, sender);
327        }
328
329        let commands_sender_clone = commands_sender.clone();
330
331        if sync_last_known_own_block {
332            commands_sender
333                .try_send(Command::FetchOwnLastBlock)
334                .expect("Failed to sync our last block");
335        }
336
337        // Spawn the task to listen to the requests & periodic runs
338        tasks.spawn(monitored_future!(async move {
339            let mut s = Self {
340                context,
341                commands_receiver,
342                fetch_block_senders,
343                core_dispatcher,
344                commit_vote_monitor,
345                fetch_blocks_scheduler_task: JoinSet::new(),
346                fetch_own_last_block_task: JoinSet::new(),
347                network_client,
348                block_verifier,
349                transaction_vote_tracker,
350                inflight_blocks_map,
351                commands_sender: commands_sender_clone,
352                dag_state,
353                round_tracker,
354                last_changed_commit_index: 0,
355                last_commit_change_time: Instant::now(),
356                commit_sync_failover: false,
357                peers_pool,
358            };
359            s.run().await;
360        }));
361
362        Arc::new(SynchronizerHandle {
363            commands_sender,
364            tasks: tokio::sync::Mutex::new(tasks),
365        })
366    }
367
368    // The main loop to listen for the submitted commands.
369    async fn run(&mut self) {
370        // We want the synchronizer to run periodically every 200ms to fetch any missing blocks.
371        const PERIODIC_FETCH_INTERVAL: Duration = Duration::from_millis(200);
372        let scheduler_timeout = sleep_until(Instant::now() + PERIODIC_FETCH_INTERVAL);
373
374        tokio::pin!(scheduler_timeout);
375
376        loop {
377            tokio::select! {
378                Some(command) = self.commands_receiver.recv() => {
379                    match command {
380                        Command::FetchBlocks{ missing_block_refs, peer, result } => {
381                            // Check if peer is available. This check also makes sure that we are not trying to fetch from ourselves.
382                            if !self.peers_pool.is_peer_known(&peer) {
383                                result.send(Err(ConsensusError::PeerUnavailable(format!("{:?}", peer)))).ok();
384                                continue;
385                            }
386
387                            // Keep only the max allowed blocks to request. Additional missing blocks
388                            // will be fetched via periodic sync.
389                            // Fetch from the lowest to highest round, to ensure progress.
390                            let missing_block_refs = missing_block_refs
391                                .into_iter()
392                                .take(self.context.parameters.max_blocks_per_sync)
393                                .collect();
394
395                            let blocks_guard = self.inflight_blocks_map.lock_blocks(missing_block_refs, peer.clone());
396                            let Some(blocks_guard) = blocks_guard else {
397                                result.send(Ok(())).ok();
398                                continue;
399                            };
400
401                            // We don't block if the corresponding peer task is saturated - but we rather drop the request. That's ok as the periodic
402                            // synchronization task will handle any still missing blocks in next run.
403                            let r = self
404                                .fetch_block_senders
405                                .get(&peer)
406                                .ok_or(ConsensusError::PeerNotFound(format!("Peer {} not found in fetch_block_senders", peer)))
407                                .and_then(|sender| {
408                                    sender
409                                        .try_send(blocks_guard)
410                                        .map_err(|err| {
411                                            match err {
412                                                TrySendError::Full(_) => {
413                                                    let peer_name = peer.labelname(&self.context);
414                                                    self.context
415                                                        .metrics
416                                                        .node_metrics
417                                                        .synchronizer_skipped_fetch_requests
418                                                        .with_label_values(&[peer_name])
419                                                        .inc();
420                                                    ConsensusError::SynchronizerSaturated(format!("{:?}", peer))
421                                                },
422                                                TrySendError::Closed(_) => ConsensusError::Shutdown
423                                            }
424                                        })
425                                });
426
427                            result.send(r).ok();
428                        }
429                        Command::FetchOwnLastBlock => {
430                            if self.fetch_own_last_block_task.is_empty() {
431                                self.start_fetch_own_last_block_task();
432                            }
433                        }
434                        Command::KickOffScheduler => {
435                            // just reset the scheduler timeout timer to run immediately if not already running.
436                            // If the scheduler is already running then just reduce the remaining time to run.
437                            let timeout = if self.fetch_blocks_scheduler_task.is_empty() {
438                                Instant::now()
439                            } else {
440                                Instant::now() + PERIODIC_FETCH_INTERVAL / 2
441                            };
442
443                            // only reset if it is earlier than the next deadline
444                            if timeout < scheduler_timeout.deadline() {
445                                scheduler_timeout.as_mut().reset(timeout);
446                            }
447                        }
448                        Command::Shutdown { result } => {
449                            self.shutdown_tasks().await;
450                            self.fetch_block_senders.clear();
451                            let _ = result.send(());
452                            return;
453                        }
454                    }
455                },
456                Some(result) = self.fetch_own_last_block_task.join_next(), if !self.fetch_own_last_block_task.is_empty() => {
457                    Self::handle_task_result(result, "fetch our last block");
458                },
459                Some(result) = self.fetch_blocks_scheduler_task.join_next(), if !self.fetch_blocks_scheduler_task.is_empty() => {
460                    Self::handle_task_result(result, "fetch blocks scheduler");
461                },
462                () = &mut scheduler_timeout => {
463                    // we want to start a new task only if the previous one has already finished.
464                    // TODO: consider starting backup fetches in parallel, when a fetch takes too long?
465                    if self.fetch_blocks_scheduler_task.is_empty()
466                        && let Err(err) = self.start_fetch_missing_blocks_task().await {
467                            debug!("Core is shutting down, synchronizer is shutting down: {err:?}");
468                            self.shutdown_tasks().await;
469                            return;
470                        };
471
472                    scheduler_timeout
473                        .as_mut()
474                        .reset(Instant::now() + PERIODIC_FETCH_INTERVAL);
475                }
476            }
477        }
478    }
479
480    // Propagates panics of a finished task. Cancellations are expected during shutdown.
481    fn handle_task_result(result: Result<(), JoinError>, task_name: &str) {
482        let Err(e) = result else {
483            return;
484        };
485        if e.is_cancelled() {
486        } else if e.is_panic() {
487            std::panic::resume_unwind(e.into_panic());
488        } else {
489            panic!("{task_name} task failed: {e}");
490        }
491    }
492
493    // Must be called before exiting run(), so no task is dropped without being awaited
494    // and task panics propagate.
495    async fn shutdown_tasks(&mut self) {
496        shutdown_join_set(&mut self.fetch_own_last_block_task).await;
497        shutdown_join_set(&mut self.fetch_blocks_scheduler_task).await;
498    }
499
500    async fn fetch_blocks_from_peer(
501        peer: PeerId,
502        network_client: Arc<SynchronizerClient<VC, OC>>,
503        block_verifier: Arc<V>,
504        transaction_vote_tracker: TransactionVoteTracker,
505        commit_vote_monitor: Arc<CommitVoteMonitor>,
506        context: Arc<Context>,
507        core_dispatcher: Arc<D>,
508        dag_state: Arc<RwLock<DagState>>,
509        mut receiver: Receiver<BlocksGuard>,
510        commands_sender: Sender<Command>,
511        round_tracker: Arc<RwLock<RoundTracker>>,
512    ) {
513        const MAX_ATTEMPTS: u32 = 3;
514        let mut requests = FuturesUnordered::new();
515
516        loop {
517            tokio::select! {
518                Some(blocks_guard) = receiver.recv(), if requests.len() < FETCH_BLOCKS_CONCURRENCY => {
519                    let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
520
521                    requests.push(Self::fetch_blocks_request(network_client.clone(), peer.clone(), blocks_guard, fetch_after_rounds, true, FETCH_REQUEST_TIMEOUT, 0))
522                },
523                Some((response, blocks_guard, failed_attempts, _peer, fetch_after_rounds)) = requests.next() => {
524                    match response {
525                        Ok(blocks) => {
526                            if let Err(err) = Self::process_fetched_blocks(blocks,
527                                peer.clone(),
528                                blocks_guard,
529                                core_dispatcher.clone(),
530                                block_verifier.clone(),
531                                transaction_vote_tracker.clone(),
532                                commit_vote_monitor.clone(),
533                                context.clone(),
534                                commands_sender.clone(),
535                                round_tracker.clone(),
536                                "live"
537                            ).await {
538                                warn!("Error while processing fetched blocks from peer {}: {err}", peer.hostname(&context));
539                                context.metrics.node_metrics.synchronizer_process_fetched_failures.with_label_values(&[peer.labelname(&context).as_str(), "live"]).inc();
540                            }
541                        },
542                        Err(_) => {
543                            context.metrics.node_metrics.synchronizer_fetch_failures.with_label_values(&[peer.labelname(&context).as_str(), "live"]).inc();
544                            if failed_attempts < MAX_ATTEMPTS {
545                                requests.push(Self::fetch_blocks_request(network_client.clone(), peer.clone(), blocks_guard, fetch_after_rounds, true, FETCH_REQUEST_TIMEOUT, failed_attempts))
546                            } else {
547                                warn!("Max attempts {MAX_ATTEMPTS} reached while trying to fetch blocks from peer {}.", peer.hostname(&context));
548                                // we don't necessarily need to do, but dropping the guard here to unlock the blocks
549                                drop(blocks_guard);
550                            }
551                        }
552                    }
553                },
554                else => {
555                    info!("Fetching blocks from peer {} task will now abort.", peer.hostname(&context));
556                    break;
557                }
558            }
559        }
560    }
561
562    /// Processes the requested raw fetched blocks from peer. If no error is returned then
563    /// the verified blocks are immediately sent to Core for processing.
564    async fn process_fetched_blocks(
565        mut serialized_blocks: Vec<Bytes>,
566        peer: PeerId,
567        requested_blocks_guard: BlocksGuard,
568        core_dispatcher: Arc<D>,
569        block_verifier: Arc<V>,
570        transaction_vote_tracker: TransactionVoteTracker,
571        commit_vote_monitor: Arc<CommitVoteMonitor>,
572        context: Arc<Context>,
573        commands_sender: Sender<Command>,
574        round_tracker: Arc<RwLock<RoundTracker>>,
575        sync_method: &str,
576    ) -> ConsensusResult<()> {
577        if serialized_blocks.is_empty() {
578            return Ok(());
579        }
580
581        // Limit the number of the returned blocks processed.
582        serialized_blocks.truncate(context.parameters.max_blocks_per_sync);
583
584        // Verify all the fetched blocks
585        let (blocks, voted_blocks) = spawn_blocking({
586            let block_verifier = block_verifier.clone();
587            let context = context.clone();
588            let peer = peer.clone();
589            move || Self::verify_blocks(serialized_blocks, block_verifier, &context, peer)
590        })
591        .await??;
592
593        if context.protocol_config.transaction_voting_enabled() {
594            transaction_vote_tracker.add_voted_blocks(voted_blocks);
595        }
596
597        // Record commit votes from the verified blocks.
598        for block in &blocks {
599            commit_vote_monitor.observe_block(block);
600        }
601
602        // Update round tracker from the verified blocks. For fetched blocks,
603        // excluded_ancestors are not available so we use an empty vector.
604        {
605            let mut tracker = round_tracker.write();
606            for block in &blocks {
607                tracker.update_from_verified_block(&ExtendedBlock {
608                    block: block.clone(),
609                    excluded_ancestors: vec![],
610                });
611            }
612        }
613
614        let metrics = &context.metrics.node_metrics;
615        metrics
616            .synchronizer_fetched_blocks_by_peer
617            .with_label_values(&[peer.labelname(&context).as_str(), sync_method])
618            .inc_by(blocks.len() as u64);
619        for block in &blocks {
620            let block_hostname = &context.committee.authority(block.author()).hostname;
621            metrics
622                .synchronizer_fetched_blocks_by_authority
623                .with_label_values(&[block_hostname.as_str(), sync_method])
624                .inc();
625        }
626
627        debug!(
628            "Synced {} missing blocks from peer {:?}: {}",
629            blocks.len(),
630            peer,
631            blocks.iter().map(|b| b.reference().to_string()).join(", "),
632        );
633
634        // Now send them to core for processing. Ignore the returned missing blocks as we don't want
635        // this mechanism to keep feedback looping on fetching more blocks. The periodic synchronization
636        // will take care of that.
637        let missing_blocks = core_dispatcher
638            .add_blocks(blocks)
639            .await
640            .map_err(|_| ConsensusError::Shutdown)?;
641
642        // now release all the locked blocks as they have been fetched, verified & processed
643        drop(requested_blocks_guard);
644
645        // kick off immediately the scheduled synchronizer
646        if !missing_blocks.is_empty() {
647            // do not block here, so we avoid any possible cycles.
648            if let Err(TrySendError::Full(_)) = commands_sender.try_send(Command::KickOffScheduler)
649            {
650                warn!("Commands channel is full")
651            }
652        }
653
654        context
655            .metrics
656            .node_metrics
657            .missing_blocks_after_fetch_total
658            .inc_by(missing_blocks.len() as u64);
659
660        Ok(())
661    }
662
663    fn get_fetch_after_rounds(context: &Context, dag_state: &RwLock<DagState>) -> Vec<Round> {
664        let (blocks, gc_round) = {
665            let dag_state = dag_state.read();
666            (
667                dag_state.get_last_cached_block_per_authority(Round::MAX),
668                dag_state.gc_round(),
669            )
670        };
671        assert_eq!(blocks.len(), context.committee.size());
672
673        blocks
674            .into_iter()
675            .map(|(block, _)| block.round().max(gc_round))
676            .collect::<Vec<_>>()
677    }
678
679    fn verify_blocks(
680        serialized_blocks: Vec<Bytes>,
681        block_verifier: Arc<V>,
682        context: &Context,
683        peer: PeerId,
684    ) -> ConsensusResult<(
685        Vec<VerifiedBlock>,
686        Vec<(VerifiedBlock, Vec<TransactionIndex>)>,
687    )> {
688        let mut verified_blocks = Vec::new();
689        let mut voted_blocks = Vec::new();
690        for serialized_block in serialized_blocks {
691            let signed_block: SignedBlock =
692                bcs::from_bytes(&serialized_block).map_err(ConsensusError::MalformedBlock)?;
693
694            // TODO: cache received and verified block refs to avoid duplicated work.
695            let (verified_block, reject_txn_votes) = block_verifier
696                .verify_and_vote(signed_block, serialized_block)
697                .tap_err(|e| {
698                    let peer_label = peer.labelname(context);
699                    context
700                        .metrics
701                        .node_metrics
702                        .invalid_blocks
703                        .with_label_values(&[peer_label.as_str(), "synchronizer", e.name()])
704                        .inc();
705                    info!("Invalid block received from {}: {}", peer, e);
706                })?;
707
708            // TODO: improve efficiency, maybe suspend and continue processing the block asynchronously.
709            let now = context.clock.timestamp_utc_ms();
710            let drift = verified_block.timestamp_ms().saturating_sub(now);
711            if drift > 0 {
712                let author_hostname = &context
713                    .committee
714                    .authority(verified_block.author())
715                    .hostname;
716                context
717                    .metrics
718                    .node_metrics
719                    .block_timestamp_drift_ms
720                    .with_label_values(&[author_hostname.as_str(), "synchronizer"])
721                    .inc_by(drift);
722
723                trace!(
724                    "Synced block {} timestamp {} is in the future (now={}).",
725                    verified_block.reference(),
726                    verified_block.timestamp_ms(),
727                    now
728                );
729            }
730
731            verified_blocks.push(verified_block.clone());
732            voted_blocks.push((verified_block, reject_txn_votes));
733        }
734
735        Ok((verified_blocks, voted_blocks))
736    }
737
738    async fn fetch_blocks_request(
739        network_client: Arc<SynchronizerClient<VC, OC>>,
740        peer: PeerId,
741        blocks_guard: BlocksGuard,
742        fetch_after_rounds: Vec<Round>,
743        fetch_missing_ancestors: bool,
744        request_timeout: Duration,
745        mut failed_attempts: u32,
746    ) -> (
747        ConsensusResult<Vec<Bytes>>,
748        BlocksGuard,
749        u32,
750        PeerId,
751        Vec<Round>,
752    ) {
753        let start = Instant::now();
754        let resp = timeout(
755            request_timeout,
756            network_client.fetch_blocks(
757                peer.clone(),
758                blocks_guard.block_refs.iter().copied().collect(),
759                fetch_after_rounds.clone(),
760                fetch_missing_ancestors,
761                request_timeout,
762            ),
763        )
764        .await;
765
766        fail_point_async!("consensus-delay");
767
768        let resp = match resp {
769            Ok(Err(err)) => {
770                // Add a delay before retrying - if that is needed. If request has timed out then eventually
771                // this will be a no-op.
772                sleep_until(start + request_timeout).await;
773                failed_attempts += 1;
774                Err(err)
775            } // network error
776            Err(err) => {
777                // timeout
778                sleep_until(start + request_timeout).await;
779                failed_attempts += 1;
780                Err(ConsensusError::NetworkRequestTimeout(err.to_string()))
781            }
782            Ok(result) => result,
783        };
784        (
785            resp,
786            blocks_guard,
787            failed_attempts,
788            peer,
789            fetch_after_rounds,
790        )
791    }
792
793    fn start_fetch_own_last_block_task(&mut self) {
794        const FETCH_OWN_BLOCK_RETRY_DELAY: Duration = Duration::from_millis(1_000);
795        const MAX_RETRY_DELAY_STEP: Duration = Duration::from_millis(4_000);
796
797        let context = self.context.clone();
798        let dag_state = self.dag_state.clone();
799        let network_client = self.network_client.clone();
800        let block_verifier = self.block_verifier.clone();
801        let core_dispatcher = self.core_dispatcher.clone();
802
803        self.fetch_own_last_block_task
804            .spawn(monitored_future!(async move {
805                let _scope = monitored_scope("FetchOwnLastBlockTask");
806
807                let fetch_own_block = |authority_index: AuthorityIndex, fetch_own_block_delay: Duration| {
808                    let network_client_cloned = network_client.clone();
809                    let own_index = context.own_index;
810                    async move {
811                        sleep(fetch_own_block_delay).await;
812                        let r = network_client_cloned.fetch_latest_blocks(authority_index, vec![own_index], FETCH_REQUEST_TIMEOUT).await;
813                        (r, authority_index)
814                    }
815                };
816
817
818                let process_blocks = |blocks: Vec<Bytes>, authority_index: AuthorityIndex| -> ConsensusResult<Vec<VerifiedBlock>> {
819                    let mut result = Vec::new();
820                    for serialized_block in blocks {
821                        let signed_block = bcs::from_bytes(&serialized_block).map_err(ConsensusError::MalformedBlock)?;
822                        let (verified_block, _) = block_verifier.verify_and_vote(signed_block, serialized_block).tap_err(|err|{
823                            let hostname = context.committee.authority(authority_index).hostname.clone();
824                            context
825                                .metrics
826                                .node_metrics
827                                .invalid_blocks
828                                .with_label_values(&[hostname.as_str(), "synchronizer_own_block", err.name()])
829                                .inc();
830                            warn!("Invalid block received from {}: {}", authority_index, err);
831                        })?;
832
833                        if verified_block.author() != context.own_index {
834                            return Err(ConsensusError::UnexpectedLastOwnBlock { index: authority_index, block_ref: verified_block.reference()});
835                        }
836                        result.push(verified_block);
837                    }
838                    Ok(result)
839                };
840
841                // Get the highest of all the results. Retry until at least `f+1` results have been gathered.
842                let mut highest_round;
843                let mut retries = 0;
844                let mut retry_delay_step = Duration::from_millis(500);
845                'main:loop {
846                    if context.committee.size() == 1 {
847                        highest_round = dag_state.read().get_last_proposed_block().expect("Last proposed block should be returned on validators").round();
848                        info!("Only one node in the network, will not try fetching own last block from peers.");
849                        break 'main;
850                    }
851
852                    let mut total_stake = 0;
853                    highest_round = 0;
854
855                    // Ask all the other peers about our last block
856                    let mut results = FuturesUnordered::new();
857
858                    for (authority_index, _authority) in context.committee.authorities() {
859                        if authority_index != context.own_index {
860                            results.push(fetch_own_block(authority_index, Duration::from_millis(0)));
861                        }
862                    }
863
864                    // Gather the results but wait to timeout as well
865                    let timer = sleep_until(Instant::now() + context.parameters.sync_last_known_own_block_timeout);
866                    tokio::pin!(timer);
867
868                    'inner: loop {
869                        tokio::select! {
870                            result = results.next() => {
871                                let Some((result, authority_index)) = result else {
872                                    break 'inner;
873                                };
874                                match result {
875                                    Ok(result) => {
876                                        match process_blocks(result, authority_index) {
877                                            Ok(blocks) => {
878                                                let max_round = blocks.into_iter().map(|b|b.round()).max().unwrap_or(0);
879                                                highest_round = highest_round.max(max_round);
880
881                                                total_stake += context.committee.stake(authority_index);
882                                            },
883                                            Err(err) => {
884                                                warn!("Invalid result returned from {authority_index} while fetching last own block: {err}");
885                                            }
886                                        }
887                                    },
888                                    Err(err) => {
889                                        warn!("Error {err} while fetching our own block from peer {authority_index}. Will retry.");
890                                        results.push(fetch_own_block(authority_index, FETCH_OWN_BLOCK_RETRY_DELAY));
891                                    }
892                                }
893                            },
894                            () = &mut timer => {
895                                info!("Timeout while trying to sync our own last block from peers");
896                                break 'inner;
897                            }
898                        }
899                    }
900
901                    // Request at least f+1 stake to have replied back.
902                    if context.committee.reached_validity(total_stake) {
903                        info!("{} out of {} total stake returned acceptable results for our own last block with highest round {}, with {retries} retries.", total_stake, context.committee.total_stake(), highest_round);
904                        break 'main;
905                    }
906
907                    retries += 1;
908                    context.metrics.node_metrics.sync_last_known_own_block_retries.inc();
909                    warn!("Not enough stake: {} out of {} total stake returned acceptable results for our own last block with highest round {}. Will now retry {retries}.", total_stake, context.committee.total_stake(), highest_round);
910
911                    sleep(retry_delay_step).await;
912
913                    retry_delay_step = Duration::from_secs_f64(retry_delay_step.as_secs_f64() * 1.5);
914                    retry_delay_step = retry_delay_step.min(MAX_RETRY_DELAY_STEP);
915                }
916
917                // Update the Core with the highest detected round
918                context.metrics.node_metrics.last_known_own_block_round.set(highest_round as i64);
919
920                if let Err(err) = core_dispatcher.set_last_known_proposed_round(highest_round) {
921                    warn!("Error received while calling dispatcher, probably dispatcher is shutting down, will now exit: {err:?}");
922                }
923            }));
924    }
925
926    async fn start_fetch_missing_blocks_task(&mut self) -> ConsensusResult<()> {
927        if self.context.committee.size() == 1 {
928            trace!(
929                "Only one node in the network, will not try fetching missing blocks from peers."
930            );
931            return Ok(());
932        }
933
934        // If commit is lagging and commit sync is making progress, skip periodic sync.
935        // Commit syncer fetches certified commits with all necessary causal history.
936        // If commit sync is not making progress, periodic sync resumes as a fallback.
937        if !self.should_run_periodic_sync() {
938            return Ok(());
939        }
940
941        let context = self.context.clone();
942        let network_client = self.network_client.clone();
943        let block_verifier = self.block_verifier.clone();
944        let transaction_vote_tracker = self.transaction_vote_tracker.clone();
945        let commit_vote_monitor = self.commit_vote_monitor.clone();
946        let core_dispatcher = self.core_dispatcher.clone();
947        let blocks_to_fetch = self.inflight_blocks_map.clone();
948        let commands_sender = self.commands_sender.clone();
949        let dag_state = self.dag_state.clone();
950        let round_tracker = self.round_tracker.clone();
951        let peers_pool = self.peers_pool.clone();
952
953        let mut missing_blocks = self
954            .core_dispatcher
955            .get_missing_blocks()
956            .await
957            .map_err(|_err| ConsensusError::Shutdown)?;
958        if self.commit_sync_failover {
959            // Keep missing blocks to those that must be included in fetch request.
960            // Filtered out missing blocks that will eventually be fetched with fetch_after_rounds.
961            let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
962            missing_blocks.retain(|block| block.round <= fetch_after_rounds[block.author.value()]);
963        } else if missing_blocks.is_empty() {
964            return Ok(());
965        }
966
967        self.fetch_blocks_scheduler_task
968            .spawn(monitored_future!(async move {
969                let _scope = monitored_scope("FetchMissingBlocksScheduler");
970                context
971                    .metrics
972                    .node_metrics
973                    .fetch_blocks_scheduler_inflight
974                    .inc();
975                let total_requested = missing_blocks.len();
976
977                let results = if missing_blocks.is_empty() {
978                    let _scope = monitored_scope("BlockSync::Periodic::HighestAcceptedRounds");
979                    // Fetch blocks from a random peer using highest accepted rounds (commit sync failover)
980                    Self::fetch_blocks_with_fetch_after_rounds(
981                        context.clone(),
982                        blocks_to_fetch.clone(),
983                        network_client,
984                        dag_state,
985                        peers_pool,
986                    )
987                    .await
988                } else {
989                    let _scope = monitored_scope("BlockSync::Periodic::MissingBlocks");
990                    // Fetch blocks from 1 to MAX_PERIODIC_SYNC_PEERS peers
991                    Self::fetch_blocks_from_peers(
992                        context.clone(),
993                        blocks_to_fetch.clone(),
994                        network_client,
995                        missing_blocks,
996                        dag_state,
997                        peers_pool,
998                    )
999                    .await
1000                };
1001                context
1002                    .metrics
1003                    .node_metrics
1004                    .fetch_blocks_scheduler_inflight
1005                    .dec();
1006                if results.is_empty() {
1007                    return;
1008                }
1009
1010                fail_point_async!("consensus-delay");
1011
1012                // Now process the returned results
1013                let mut total_fetched = 0;
1014                for (blocks_guard, fetched_blocks, peer) in results {
1015                    total_fetched += fetched_blocks.len();
1016
1017                    if let Err(err) = Self::process_fetched_blocks(
1018                        fetched_blocks,
1019                        peer.clone(),
1020                        blocks_guard,
1021                        core_dispatcher.clone(),
1022                        block_verifier.clone(),
1023                        transaction_vote_tracker.clone(),
1024                        commit_vote_monitor.clone(),
1025                        context.clone(),
1026                        commands_sender.clone(),
1027                        round_tracker.clone(),
1028                        "periodic",
1029                    )
1030                    .await
1031                    {
1032                        warn!(
1033                            "Error occurred while processing fetched blocks from peer {:?}: {err}",
1034                            peer
1035                        );
1036                        let peer_name = peer.labelname(&context);
1037                        context
1038                            .metrics
1039                            .node_metrics
1040                            .synchronizer_process_fetched_failures
1041                            .with_label_values(&[peer_name.as_str(), "periodic"])
1042                            .inc();
1043                    }
1044                }
1045
1046                debug!(
1047                    "Total blocks requested to fetch: {}, total fetched: {}",
1048                    total_requested, total_fetched
1049                );
1050            }));
1051
1052        Ok(())
1053    }
1054
1055    fn should_run_periodic_sync(&mut self) -> bool {
1056        let current_commit_index = self.dag_state.read().last_commit_index();
1057        let quorum_commit_index = self.commit_vote_monitor.quorum_commit_index();
1058        let now = Instant::now();
1059        let metrics = &self.context.metrics.node_metrics;
1060
1061        // Commit is not lagging.
1062        if !is_commit_lagging(
1063            self.context.as_ref(),
1064            current_commit_index,
1065            quorum_commit_index,
1066        ) {
1067            metrics
1068                .synchronizer_periodic_sync_decision
1069                .with_label_values(&["true", "default"])
1070                .inc();
1071            // Reset last commit state.
1072            self.last_changed_commit_index = current_commit_index;
1073            self.last_commit_change_time = now;
1074            self.commit_sync_failover = false;
1075            // Run periodic sync.
1076            return true;
1077        }
1078        // Commit is lagging.
1079
1080        // When in commit sync failover, check if enough progress has been made.
1081        if self.commit_sync_failover {
1082            if current_commit_index
1083                < self.last_changed_commit_index + self.context.parameters.commit_sync_batch_size
1084            {
1085                metrics
1086                    .synchronizer_periodic_sync_decision
1087                    .with_label_values(&["true", "commit_catchup::run"])
1088                    .inc();
1089                // Not enough progress has been made yet. Keep running periodic sync.
1090                // Do not update last commit state yet.
1091                // Run periodic sync.
1092                return true;
1093            } else {
1094                metrics
1095                    .synchronizer_periodic_sync_decision
1096                    .with_label_values(&["false", "commit_catchup::end"])
1097                    .inc();
1098                // Enough progress has been made. Disable commit sync failover and reset last commit state.
1099                self.last_changed_commit_index = current_commit_index;
1100                self.last_commit_change_time = now;
1101                self.commit_sync_failover = false;
1102                // Skip periodic sync because of commit lag.
1103                return false;
1104            }
1105        }
1106
1107        // The node is commit lagging and not in commit sync failover yet.
1108        if current_commit_index == self.last_changed_commit_index {
1109            // Enter commit sync failover if not enough progress has been made.
1110            if now.duration_since(self.last_commit_change_time) >= COMMIT_PROGRESS_TIMEOUT {
1111                metrics
1112                    .synchronizer_periodic_sync_decision
1113                    .with_label_values(&["true", "commit_catchup::start"])
1114                    .inc();
1115                self.commit_sync_failover = true;
1116                // Run periodic sync.
1117                return true;
1118            }
1119        } else {
1120            // IMPORTANT: Only update last commit state when commit index is changing.
1121            self.last_changed_commit_index = current_commit_index;
1122            self.last_commit_change_time = now;
1123        }
1124
1125        metrics
1126            .synchronizer_periodic_sync_decision
1127            .with_label_values(&["false", "commit_lag"])
1128            .inc();
1129        false
1130    }
1131
1132    /// Fetches blocks from a random peer using only fetch_after_rounds (no specific missing blocks).
1133    /// Used during commit sync failover to make progress when commit sync is stalled.
1134    async fn fetch_blocks_with_fetch_after_rounds(
1135        context: Arc<Context>,
1136        inflight_blocks: Arc<InflightBlocksMap>,
1137        network_client: Arc<SynchronizerClient<VC, OC>>,
1138        dag_state: Arc<RwLock<DagState>>,
1139        peers_pool: Arc<PeersPool>,
1140    ) -> Vec<(BlocksGuard, Vec<Bytes>, PeerId)> {
1141        let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
1142
1143        // Pick a random peer (excluding self).
1144        // Get available peers from the PeersPool
1145        let peers = peers_pool.get_known_peers();
1146
1147        // TODO: in the future it would be possible, temporarily, for an Observer node to not have peers to fetch from.
1148        // We should change this assertion to allow for this case.
1149        assert!(!peers.is_empty(), "No known peers to fetch blocks from");
1150
1151        let peer = if cfg!(test) {
1152            peers.first()
1153        } else {
1154            peers.choose(&mut ThreadRng::default())
1155        }
1156        .unwrap()
1157        .clone();
1158
1159        let response = timeout(
1160            FETCH_REQUEST_TIMEOUT,
1161            network_client.fetch_blocks(
1162                peer.clone(),
1163                vec![],
1164                fetch_after_rounds,
1165                false,
1166                FETCH_REQUEST_TIMEOUT,
1167            ),
1168        )
1169        .await;
1170
1171        let serialized_blocks = match response {
1172            Ok(Ok(blocks)) => blocks,
1173            Ok(Err(err)) => {
1174                debug!("Failed to fetch blocks with fetch_after_rounds from peer {peer}: {err}");
1175                return vec![];
1176            }
1177            Err(_) => {
1178                debug!("Timed out fetching blocks with fetch_after_rounds from peer {peer}");
1179                return vec![];
1180            }
1181        };
1182
1183        let blocks_guard = BlocksGuard {
1184            map: inflight_blocks,
1185            block_refs: BTreeSet::new(),
1186            peer: peer.clone(),
1187        };
1188
1189        vec![(blocks_guard, serialized_blocks, peer)]
1190    }
1191
1192    /// Fetches the `missing_blocks` from peers. Requests the same number of authorities with missing blocks from each peer.
1193    /// Each response from peer can contain the requested blocks, and additional blocks from the last accepted round for
1194    /// authorities with missing blocks.
1195    /// Each element of the vector is a tuple which contains the requested missing block refs, the returned blocks and
1196    /// the peer.
1197    async fn fetch_blocks_from_peers(
1198        context: Arc<Context>,
1199        inflight_blocks: Arc<InflightBlocksMap>,
1200        network_client: Arc<SynchronizerClient<VC, OC>>,
1201        missing_blocks: BTreeSet<BlockRef>,
1202        dag_state: Arc<RwLock<DagState>>,
1203        peers_pool: Arc<PeersPool>,
1204    ) -> Vec<(BlocksGuard, Vec<Bytes>, PeerId)> {
1205        // Preliminary truncation of missing blocks to fetch. Since each peer can have different
1206        // number of missing blocks and the fetching is batched by peer, so keep more than max_blocks_per_fetch
1207        // per peer on average.
1208        let missing_blocks = missing_blocks
1209            .into_iter()
1210            .take(2 * MAX_PERIODIC_SYNC_PEERS * context.parameters.max_blocks_per_fetch)
1211            .collect::<Vec<_>>();
1212
1213        // Maps authorities to the missing blocks they have.
1214        let mut authorities = BTreeMap::<AuthorityIndex, Vec<BlockRef>>::new();
1215        for block_ref in &missing_blocks {
1216            authorities
1217                .entry(block_ref.author)
1218                .or_default()
1219                .push(*block_ref);
1220        }
1221
1222        // Get known peers from the PeersPool
1223        let mut peers = peers_pool.get_known_peers();
1224
1225        // Distribute the same number of authorities into each peer to sync.
1226        // Use the number of known peers from the pool, capped at MAX_PERIODIC_SYNC_PEERS
1227        // TODO: in the future it would be possible, temporarily, for an Observer node to not have peers to fetch from.
1228        // We should change this assertion to allow for this case.
1229        assert!(!peers.is_empty(), "No known peers to fetch blocks from");
1230
1231        let num_authorities_per_peer = authorities
1232            .len()
1233            .div_ceil(peers.len().min(MAX_PERIODIC_SYNC_PEERS));
1234
1235        // Update metrics related to missing blocks.
1236        let mut missing_blocks_per_authority = vec![0; context.committee.size()];
1237        for (authority, blocks) in &authorities {
1238            missing_blocks_per_authority[*authority] += blocks.len();
1239        }
1240        for (missing, (_, authority)) in missing_blocks_per_authority
1241            .into_iter()
1242            .zip_debug_eq(context.committee.authorities())
1243        {
1244            context
1245                .metrics
1246                .node_metrics
1247                .synchronizer_missing_blocks_by_authority
1248                .with_label_values(&[&authority.hostname])
1249                .inc_by(missing as u64);
1250            context
1251                .metrics
1252                .node_metrics
1253                .synchronizer_current_missing_blocks_by_authority
1254                .with_label_values(&[&authority.hostname])
1255                .set(missing as i64);
1256        }
1257
1258        // TODO: probably inject the RNG to allow unit testing - this is a work around for now.
1259        if cfg!(not(test)) {
1260            // Shuffle the peers
1261            peers.shuffle(&mut ThreadRng::default());
1262        }
1263
1264        let mut peers = peers.into_iter();
1265        let mut request_futures = FuturesUnordered::new();
1266
1267        // Shuffle the authorities for each request.
1268        let mut authorities = authorities.into_values().collect::<Vec<_>>();
1269        if cfg!(not(test)) {
1270            // Shuffle the authorities
1271            authorities.shuffle(&mut ThreadRng::default());
1272        }
1273
1274        let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
1275
1276        // Send the fetch requests
1277        for batch in authorities.chunks(num_authorities_per_peer) {
1278            let Some(peer) = peers.next() else {
1279                debug_fatal!("No more peers left to fetch blocks!");
1280                break;
1281            };
1282            let peer_name = peer.hostname(&context);
1283            // Fetch from the lowest round missing blocks to ensure progress.
1284            // This may reduce efficiency and increase the chance of duplicated data transfer in edge cases.
1285            let block_refs = batch
1286                .iter()
1287                .flatten()
1288                .cloned()
1289                .collect::<BTreeSet<_>>()
1290                .into_iter()
1291                .take(context.parameters.max_blocks_per_fetch)
1292                .collect::<BTreeSet<_>>();
1293
1294            // lock the blocks to be fetched. If no lock can be acquired for any of the blocks then don't bother
1295            if let Some(blocks_guard) = inflight_blocks.lock_blocks(block_refs, peer.clone()) {
1296                info!(
1297                    "Periodic sync of {} missing blocks from peer {} {:?}: {}",
1298                    blocks_guard.block_refs.len(),
1299                    peer_name.as_str(),
1300                    peer,
1301                    blocks_guard
1302                        .block_refs
1303                        .iter()
1304                        .map(|b| b.to_string())
1305                        .collect::<Vec<_>>()
1306                        .join(", ")
1307                );
1308                request_futures.push(Self::fetch_blocks_request(
1309                    network_client.clone(),
1310                    peer,
1311                    blocks_guard,
1312                    fetch_after_rounds.clone(),
1313                    false,
1314                    FETCH_REQUEST_TIMEOUT,
1315                    0,
1316                ));
1317            }
1318        }
1319
1320        let mut results = Vec::new();
1321        let fetcher_timeout = sleep(FETCH_FROM_PEERS_TIMEOUT);
1322
1323        tokio::pin!(fetcher_timeout);
1324
1325        // Stop as soon as no pending requests are left, instead of waiting for the timeout.
1326        while !request_futures.is_empty() {
1327            tokio::select! {
1328                Some((response, blocks_guard, _failed_attempts, peer, fetch_after_rounds)) = request_futures.next() => {
1329                    match response {
1330                        Ok(fetched_blocks) => {
1331                            results.push((blocks_guard, fetched_blocks, peer));
1332                        },
1333                        Err(_) => {
1334                            let peer_name = peer.labelname(&context);
1335                            context.metrics.node_metrics.synchronizer_fetch_failures.with_label_values(&[peer_name.as_str(), "periodic"]).inc();
1336                            // try again if there is any peer left
1337                            if let Some(next_peer) = peers.next() {
1338                                // do best effort to lock guards. If we can't lock then don't bother at this run.
1339                                if let Some(blocks_guard) = inflight_blocks.swap_locks(blocks_guard, next_peer.clone()) {
1340                                    info!(
1341                                        "Retrying syncing {} missing blocks from peer {:?}: {}",
1342                                        blocks_guard.block_refs.len(),
1343                                        next_peer,
1344                                        blocks_guard.block_refs
1345                                            .iter()
1346                                            .map(|b| b.to_string())
1347                                            .collect::<Vec<_>>()
1348                                            .join(", ")
1349                                    );
1350                                    request_futures.push(Self::fetch_blocks_request(
1351                                        network_client.clone(),
1352                                        next_peer,
1353                                        blocks_guard,
1354                                        fetch_after_rounds,
1355                                        false,
1356                                        FETCH_REQUEST_TIMEOUT,
1357                                        0,
1358                                    ));
1359                                } else {
1360                                    debug!("Couldn't acquire locks to fetch blocks from peer {:?}.", next_peer)
1361                                }
1362                            } else {
1363                                debug!("No more peers left to fetch blocks");
1364                            }
1365                        }
1366                    }
1367                },
1368                _ = &mut fetcher_timeout => {
1369                    debug!("Timed out while fetching missing blocks");
1370                    break;
1371                }
1372            }
1373        }
1374
1375        results
1376    }
1377}
1378
1379#[cfg(test)]
1380mod tests {
1381    use std::{
1382        collections::{BTreeMap, BTreeSet},
1383        sync::Arc,
1384        time::Duration,
1385    };
1386
1387    use async_trait::async_trait;
1388    use bytes::Bytes;
1389    use consensus_config::{AuthorityIndex, Parameters};
1390    use consensus_types::block::{BlockDigest, BlockRef, Round};
1391    use mysten_metrics::monitored_mpsc;
1392    use parking_lot::RwLock;
1393    use tokio::{sync::Mutex, time::sleep};
1394
1395    use crate::commit::{CommitVote, TrustedCommit};
1396    use crate::{
1397        CommitDigest, CommitIndex,
1398        block::{TestBlock, VerifiedBlock},
1399        block_verifier::NoopBlockVerifier,
1400        commit_vote_monitor::CommitVoteMonitor,
1401        context::Context,
1402        core_thread::CoreThreadDispatcher,
1403        dag_state::DagState,
1404        error::{ConsensusError, ConsensusResult},
1405        network::{
1406            BlockStream, ObserverNetworkClient, PeerId, SynchronizerClient, ValidatorNetworkClient,
1407        },
1408        storage::mem_store::MemStore,
1409        synchronizer::{
1410            COMMIT_PROGRESS_TIMEOUT, FETCH_BLOCKS_CONCURRENCY, FETCH_FROM_PEERS_TIMEOUT,
1411            FETCH_REQUEST_TIMEOUT, InflightBlocksMap, Synchronizer,
1412        },
1413    };
1414    use crate::{
1415        commit_vote_monitor::COMMIT_LAG_MULTIPLIER, core_thread::MockCoreThreadDispatcher,
1416        peers_pool::PeersPool, round_tracker::RoundTracker,
1417        transaction_vote_tracker::TransactionVoteTracker,
1418    };
1419
1420    type FetchRequestKey = (Vec<BlockRef>, AuthorityIndex);
1421    type FetchRequestResponse = (Vec<VerifiedBlock>, Option<Duration>);
1422    type FetchLatestBlockKey = (AuthorityIndex, Vec<AuthorityIndex>);
1423    type FetchLatestBlockResponse = (Vec<VerifiedBlock>, Option<Duration>);
1424
1425    #[derive(Default)]
1426    struct MockNetworkClient {
1427        fetch_blocks_requests: Mutex<BTreeMap<FetchRequestKey, FetchRequestResponse>>,
1428        fetch_latest_blocks_requests:
1429            Mutex<BTreeMap<FetchLatestBlockKey, Vec<FetchLatestBlockResponse>>>,
1430    }
1431
1432    impl MockNetworkClient {
1433        async fn stub_fetch_blocks(
1434            &self,
1435            blocks: Vec<VerifiedBlock>,
1436            peer: AuthorityIndex,
1437            latency: Option<Duration>,
1438        ) {
1439            let mut lock = self.fetch_blocks_requests.lock().await;
1440            let block_refs = blocks
1441                .iter()
1442                .map(|block| block.reference())
1443                .collect::<Vec<_>>();
1444            lock.insert((block_refs, peer), (blocks, latency));
1445        }
1446
1447        async fn stub_fetch_blocks_for_key(
1448            &self,
1449            key_refs: Vec<BlockRef>,
1450            response_blocks: Vec<VerifiedBlock>,
1451            peer: AuthorityIndex,
1452            latency: Option<Duration>,
1453        ) {
1454            let mut lock = self.fetch_blocks_requests.lock().await;
1455            lock.insert((key_refs, peer), (response_blocks, latency));
1456        }
1457
1458        async fn stub_fetch_latest_blocks(
1459            &self,
1460            blocks: Vec<VerifiedBlock>,
1461            peer: AuthorityIndex,
1462            authorities: Vec<AuthorityIndex>,
1463            latency: Option<Duration>,
1464        ) {
1465            let mut lock = self.fetch_latest_blocks_requests.lock().await;
1466            lock.entry((peer, authorities))
1467                .or_default()
1468                .push((blocks, latency));
1469        }
1470
1471        async fn fetch_latest_blocks_pending_calls(&self) -> usize {
1472            let lock = self.fetch_latest_blocks_requests.lock().await;
1473            lock.len()
1474        }
1475    }
1476
1477    #[async_trait]
1478    impl ValidatorNetworkClient for MockNetworkClient {
1479        async fn subscribe_blocks(
1480            &self,
1481            _peer: AuthorityIndex,
1482            _last_received: Round,
1483            _timeout: Duration,
1484        ) -> ConsensusResult<BlockStream> {
1485            unimplemented!("subscribe_blocks not implemented in mock")
1486        }
1487
1488        async fn fetch_blocks(
1489            &self,
1490            peer: AuthorityIndex,
1491            block_refs: Vec<BlockRef>,
1492            _fetch_after_rounds: Vec<Round>,
1493            _fetch_missing_ancestors: bool,
1494            _timeout: Duration,
1495        ) -> ConsensusResult<Vec<Bytes>> {
1496            let mut lock = self.fetch_blocks_requests.lock().await;
1497            let response = lock.remove(&(block_refs.clone(), peer)).unwrap_or_else(|| {
1498                panic!(
1499                    "Unexpected fetch blocks request made: {:?} {}. Current lock: {:?}",
1500                    block_refs, peer, lock
1501                );
1502            });
1503
1504            let serialised = response
1505                .0
1506                .into_iter()
1507                .map(|block| block.serialized().clone())
1508                .collect::<Vec<_>>();
1509
1510            drop(lock);
1511
1512            if let Some(latency) = response.1 {
1513                sleep(latency).await;
1514            }
1515
1516            Ok(serialised)
1517        }
1518
1519        async fn fetch_commits(
1520            &self,
1521            _peer: AuthorityIndex,
1522            _commit_range: crate::commit::CommitRange,
1523            _timeout: Duration,
1524        ) -> ConsensusResult<(Vec<Bytes>, Vec<Bytes>)> {
1525            unimplemented!("fetch_commits not implemented in mock")
1526        }
1527
1528        async fn fetch_latest_blocks(
1529            &self,
1530            peer: AuthorityIndex,
1531            authorities: Vec<AuthorityIndex>,
1532            _timeout: Duration,
1533        ) -> ConsensusResult<Vec<Bytes>> {
1534            let mut lock = self.fetch_latest_blocks_requests.lock().await;
1535            let mut responses = lock
1536                .remove(&(peer, authorities.clone()))
1537                .expect("Unexpected fetch blocks request made");
1538
1539            let response = responses.remove(0);
1540            let serialised = response
1541                .0
1542                .into_iter()
1543                .map(|block| block.serialized().clone())
1544                .collect::<Vec<_>>();
1545
1546            if !responses.is_empty() {
1547                lock.insert((peer, authorities), responses);
1548            }
1549
1550            drop(lock);
1551
1552            if let Some(latency) = response.1 {
1553                sleep(latency).await;
1554            }
1555
1556            Ok(serialised)
1557        }
1558
1559        async fn get_latest_rounds(
1560            &self,
1561            _peer: AuthorityIndex,
1562            _timeout: Duration,
1563        ) -> ConsensusResult<(Vec<Round>, Vec<Round>)> {
1564            unimplemented!("get_latest_rounds not implemented in mock")
1565        }
1566
1567        #[cfg(all(test, not(msim)))]
1568        async fn send_block(
1569            &self,
1570            _peer: AuthorityIndex,
1571            _block: &VerifiedBlock,
1572            _timeout: Duration,
1573        ) -> ConsensusResult<()> {
1574            unimplemented!("send_block not implemented in mock")
1575        }
1576    }
1577
1578    #[async_trait]
1579    impl ObserverNetworkClient for MockNetworkClient {
1580        async fn stream_blocks(
1581            &self,
1582            _peer: crate::network::PeerId,
1583            _highest_round_per_authority: Vec<Round>,
1584            _filter: crate::network::BlockStreamFilter,
1585            _timeout: Duration,
1586        ) -> ConsensusResult<crate::network::ObserverBlockStream> {
1587            unimplemented!("stream_blocks not implemented in mock")
1588        }
1589
1590        async fn fetch_blocks(
1591            &self,
1592            _peer: crate::network::PeerId,
1593            _block_refs: Vec<BlockRef>,
1594            _fetch_after_rounds: Vec<Round>,
1595            _fetch_missing_ancestors: bool,
1596            _timeout: Duration,
1597        ) -> ConsensusResult<Vec<Bytes>> {
1598            unimplemented!("Observer fetch_blocks not implemented in mock")
1599        }
1600
1601        async fn fetch_commits(
1602            &self,
1603            _peer: crate::network::PeerId,
1604            _commit_range: crate::commit::CommitRange,
1605            _timeout: Duration,
1606        ) -> ConsensusResult<(Vec<Bytes>, Vec<Bytes>)> {
1607            unimplemented!("Observer fetch_commits not implemented in mock")
1608        }
1609    }
1610
1611    #[test]
1612    fn test_inflight_blocks_map() {
1613        // GIVEN
1614        let map = InflightBlocksMap::new();
1615        let some_block_refs = [
1616            BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN),
1617            BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN),
1618            BlockRef::new(12, AuthorityIndex::new_for_test(3), BlockDigest::MIN),
1619            BlockRef::new(15, AuthorityIndex::new_for_test(2), BlockDigest::MIN),
1620        ];
1621        let missing_block_refs = some_block_refs.iter().cloned().collect::<BTreeSet<_>>();
1622
1623        // Lock & unlock blocks
1624        {
1625            let mut all_guards = Vec::new();
1626
1627            // Try to acquire the block locks for authorities 1 & 2
1628            for i in 1..=2 {
1629                let authority = AuthorityIndex::new_for_test(i);
1630                let peer = PeerId::Validator(authority);
1631
1632                let guard = map.lock_blocks(missing_block_refs.clone(), peer.clone());
1633                let guard = guard.expect("Guard should be created");
1634                assert_eq!(guard.block_refs.len(), 4);
1635
1636                all_guards.push(guard);
1637
1638                // trying to acquire any of them again will not succeed
1639                let guard = map.lock_blocks(missing_block_refs.clone(), peer);
1640                assert!(guard.is_none());
1641            }
1642
1643            // Trying to acquire for authority 3 it will fail - as we have maxed out the number of allowed peers
1644            let authority_3 = AuthorityIndex::new_for_test(3);
1645            let peer_3 = PeerId::Validator(authority_3);
1646
1647            let guard = map.lock_blocks(missing_block_refs.clone(), peer_3.clone());
1648            assert!(guard.is_none());
1649
1650            // Explicitly drop the guard of authority 1 and try for authority 3 again - it will now succeed
1651            drop(all_guards.remove(0));
1652
1653            let guard = map.lock_blocks(missing_block_refs.clone(), peer_3);
1654            let guard = guard.expect("Guard should be successfully acquired");
1655
1656            assert_eq!(guard.block_refs, missing_block_refs);
1657
1658            // Dropping all guards should unlock on the block refs
1659            drop(guard);
1660            drop(all_guards);
1661
1662            assert_eq!(map.num_of_locked_blocks(), 0);
1663        }
1664
1665        // Swap locks
1666        {
1667            // acquire a lock for authority 1
1668            let authority_1 = AuthorityIndex::new_for_test(1);
1669            let peer_1 = PeerId::Validator(authority_1);
1670            let guard = map.lock_blocks(missing_block_refs.clone(), peer_1).unwrap();
1671
1672            // Now swap the locks for authority 2
1673            let authority_2 = AuthorityIndex::new_for_test(2);
1674            let peer_2 = PeerId::Validator(authority_2);
1675            let guard = map.swap_locks(guard, peer_2);
1676
1677            assert_eq!(guard.unwrap().block_refs, missing_block_refs);
1678        }
1679    }
1680
1681    #[tokio::test]
1682    async fn successful_fetch_blocks_from_peer() {
1683        // GIVEN
1684        let (context, _) = Context::new_for_test(4);
1685        let context = Arc::new(context);
1686        let block_verifier = Arc::new(NoopBlockVerifier {});
1687        let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
1688        let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
1689        let mock_client = Arc::new(MockNetworkClient::default());
1690        let store = Arc::new(MemStore::new());
1691        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1692        let transaction_vote_tracker =
1693            TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
1694        let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
1695
1696        let network_client = Arc::new(SynchronizerClient::new(
1697            context.clone(),
1698            Some(mock_client.clone()),
1699            Some(mock_client.clone()),
1700        ));
1701        let peers_pool = Arc::new(PeersPool::new(context.clone()));
1702        let handle = Synchronizer::start(
1703            network_client,
1704            context.clone(),
1705            core_dispatcher.clone(),
1706            commit_vote_monitor,
1707            block_verifier,
1708            transaction_vote_tracker,
1709            round_tracker,
1710            dag_state,
1711            peers_pool.clone(),
1712            false,
1713        );
1714
1715        // Create some test blocks
1716        let expected_blocks = (0..10)
1717            .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
1718            .collect::<Vec<_>>();
1719        let missing_blocks = expected_blocks
1720            .iter()
1721            .map(|block| block.reference())
1722            .collect::<BTreeSet<_>>();
1723
1724        // AND stub the fetch_blocks request from peer 1
1725        let peer = AuthorityIndex::new_for_test(1);
1726        mock_client
1727            .stub_fetch_blocks(expected_blocks.clone(), peer, None)
1728            .await;
1729
1730        // WHEN request missing blocks from peer 1
1731        assert!(
1732            handle
1733                .fetch_blocks(missing_blocks, PeerId::Validator(peer))
1734                .await
1735                .is_ok()
1736        );
1737
1738        // Wait a little bit until those have been added in core
1739        sleep(Duration::from_millis(1_000)).await;
1740
1741        // THEN ensure those ended up in Core
1742        let added_blocks = core_dispatcher.get_add_blocks().await;
1743        assert_eq!(added_blocks, expected_blocks);
1744    }
1745
1746    #[tokio::test]
1747    async fn saturate_fetch_blocks_from_peer() {
1748        // GIVEN
1749        let (context, _) = Context::new_for_test(4);
1750        let context = Arc::new(context);
1751        let block_verifier = Arc::new(NoopBlockVerifier {});
1752        let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
1753        let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
1754        let mock_client = Arc::new(MockNetworkClient::default());
1755        let store = Arc::new(MemStore::new());
1756        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1757        let transaction_vote_tracker =
1758            TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
1759        let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
1760
1761        let network_client = Arc::new(SynchronizerClient::new(
1762            context.clone(),
1763            Some(mock_client.clone()),
1764            Some(mock_client.clone()),
1765        ));
1766        let peers_pool = Arc::new(PeersPool::new(context.clone()));
1767        let handle = Synchronizer::start(
1768            network_client,
1769            context.clone(),
1770            core_dispatcher.clone(),
1771            commit_vote_monitor,
1772            block_verifier,
1773            transaction_vote_tracker,
1774            round_tracker,
1775            dag_state,
1776            peers_pool.clone(),
1777            false,
1778        );
1779
1780        // Create some test blocks
1781        let expected_blocks = (0..=2 * FETCH_BLOCKS_CONCURRENCY)
1782            .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round as Round, 0).build()))
1783            .collect::<Vec<_>>();
1784
1785        // Now start sending requests to fetch blocks by trying to saturate peer 1 task
1786        let peer = AuthorityIndex::new_for_test(1);
1787        let mut iter = expected_blocks.iter().peekable();
1788        while let Some(block) = iter.next() {
1789            // stub the fetch_blocks request from peer 1 and give some high response latency so requests
1790            // can start blocking the peer task.
1791            mock_client
1792                .stub_fetch_blocks(
1793                    vec![block.clone()],
1794                    peer,
1795                    Some(Duration::from_millis(5_000)),
1796                )
1797                .await;
1798
1799            let mut missing_blocks = BTreeSet::new();
1800            missing_blocks.insert(block.reference());
1801
1802            // WHEN requesting to fetch the blocks, it should not succeed for the last request and get
1803            // an error with "saturated" synchronizer
1804            if iter.peek().is_none() {
1805                match handle
1806                    .fetch_blocks(missing_blocks, PeerId::Validator(peer))
1807                    .await
1808                {
1809                    Err(ConsensusError::SynchronizerSaturated(peer_str)) => {
1810                        assert_eq!(peer_str, format!("{:?}", PeerId::Validator(peer)));
1811                    }
1812                    _ => panic!("A saturated synchronizer error was expected"),
1813                }
1814            } else {
1815                assert!(
1816                    handle
1817                        .fetch_blocks(missing_blocks, PeerId::Validator(peer))
1818                        .await
1819                        .is_ok()
1820                );
1821            }
1822        }
1823    }
1824
1825    #[tokio::test(flavor = "current_thread", start_paused = true)]
1826    async fn synchronizer_periodic_task_fetch_blocks() {
1827        // GIVEN
1828        let (context, _) = Context::new_for_test(4);
1829        let context = Arc::new(context);
1830        let block_verifier = Arc::new(NoopBlockVerifier {});
1831        let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
1832        let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
1833        let mock_client = Arc::new(MockNetworkClient::default());
1834        let store = Arc::new(MemStore::new());
1835        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1836        let transaction_vote_tracker =
1837            TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
1838        let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
1839
1840        // Create some test blocks
1841        let expected_blocks = (0..10)
1842            .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
1843            .collect::<Vec<_>>();
1844        let missing_blocks = expected_blocks
1845            .iter()
1846            .map(|block| block.reference())
1847            .collect::<BTreeSet<_>>();
1848
1849        // AND stub the missing blocks
1850        core_dispatcher
1851            .stub_missing_blocks(missing_blocks.clone())
1852            .await;
1853
1854        // AND stub the requests for authority 1 & 2
1855        // Make the first authority timeout, so the second will be called. "We" are authority = 0, so
1856        // we are skipped anyways.
1857        mock_client
1858            .stub_fetch_blocks(
1859                expected_blocks.clone(),
1860                AuthorityIndex::new_for_test(1),
1861                Some(FETCH_REQUEST_TIMEOUT),
1862            )
1863            .await;
1864        mock_client
1865            .stub_fetch_blocks(
1866                expected_blocks.clone(),
1867                AuthorityIndex::new_for_test(2),
1868                None,
1869            )
1870            .await;
1871
1872        // WHEN start the synchronizer and wait for a couple of seconds
1873        let network_client = Arc::new(SynchronizerClient::new(
1874            context.clone(),
1875            Some(mock_client.clone()),
1876            Some(mock_client.clone()),
1877        ));
1878        let peers_pool = Arc::new(PeersPool::new(context.clone()));
1879        let _handle = Synchronizer::start(
1880            network_client,
1881            context.clone(),
1882            core_dispatcher.clone(),
1883            commit_vote_monitor,
1884            block_verifier,
1885            transaction_vote_tracker,
1886            round_tracker,
1887            dag_state,
1888            peers_pool.clone(),
1889            false,
1890        );
1891
1892        sleep(2 * FETCH_REQUEST_TIMEOUT).await;
1893
1894        // THEN the missing blocks should now be fetched and added to core
1895        let added_blocks = core_dispatcher.get_add_blocks().await;
1896        assert_eq!(added_blocks, expected_blocks);
1897
1898        // AND missing blocks should have been consumed by the stub
1899        assert!(
1900            core_dispatcher
1901                .get_missing_blocks()
1902                .await
1903                .unwrap()
1904                .is_empty()
1905        );
1906    }
1907
1908    #[tokio::test(flavor = "current_thread", start_paused = true)]
1909    async fn fetch_blocks_from_peers_returns_when_no_requests_are_pending() {
1910        type TestSynchronizer = Synchronizer<
1911            NoopBlockVerifier,
1912            MockCoreThreadDispatcher,
1913            MockNetworkClient,
1914            MockNetworkClient,
1915        >;
1916
1917        // GIVEN
1918        let (context, _) = Context::new_for_test(4);
1919        let context = Arc::new(context);
1920        let mock_client = Arc::new(MockNetworkClient::default());
1921        let store = Arc::new(MemStore::new());
1922        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1923        let network_client = Arc::new(SynchronizerClient::new(
1924            context.clone(),
1925            Some(mock_client.clone()),
1926            Some(mock_client.clone()),
1927        ));
1928        let peers_pool = Arc::new(PeersPool::new(context.clone()));
1929        let inflight_blocks = InflightBlocksMap::new();
1930
1931        // One missing block per authority 1, 2 & 3, so each peer gets assigned one of them.
1932        let blocks = (1..=3)
1933            .map(|author| VerifiedBlock::new_for_test(TestBlock::new(1, author).build()))
1934            .collect::<Vec<_>>();
1935        let missing_blocks = blocks
1936            .iter()
1937            .map(|block| block.reference())
1938            .collect::<BTreeSet<_>>();
1939
1940        // Peers 1 & 2 respond immediately, peer 3 times out and there is no peer left to retry.
1941        for (i, block) in blocks.iter().enumerate() {
1942            let latency = (i == 2).then_some(2 * FETCH_REQUEST_TIMEOUT);
1943            mock_client
1944                .stub_fetch_blocks(
1945                    vec![block.clone()],
1946                    AuthorityIndex::new_for_test(i as u32 + 1),
1947                    latency,
1948                )
1949                .await;
1950        }
1951
1952        // WHEN
1953        let start = tokio::time::Instant::now();
1954        let results = TestSynchronizer::fetch_blocks_from_peers(
1955            context.clone(),
1956            inflight_blocks.clone(),
1957            network_client.clone(),
1958            missing_blocks.clone(),
1959            dag_state.clone(),
1960            peers_pool.clone(),
1961        )
1962        .await;
1963
1964        // THEN the successful results are returned once the failed request completes, without
1965        // waiting for the overall timeout.
1966        assert_eq!(results.len(), 2);
1967        assert_eq!(start.elapsed(), FETCH_REQUEST_TIMEOUT);
1968        assert!(start.elapsed() < FETCH_FROM_PEERS_TIMEOUT);
1969        drop(results);
1970        assert_eq!(inflight_blocks.num_of_locked_blocks(), 0);
1971
1972        // AND WHEN all the missing blocks are already locked by the max allowed number of peers
1973        let _guards = (1..=2)
1974            .map(|i| {
1975                let peer = PeerId::Validator(AuthorityIndex::new_for_test(i));
1976                inflight_blocks
1977                    .lock_blocks(missing_blocks.clone(), peer)
1978                    .unwrap()
1979            })
1980            .collect::<Vec<_>>();
1981
1982        let start = tokio::time::Instant::now();
1983        let results = TestSynchronizer::fetch_blocks_from_peers(
1984            context,
1985            inflight_blocks.clone(),
1986            network_client,
1987            missing_blocks,
1988            dag_state,
1989            peers_pool,
1990        )
1991        .await;
1992
1993        // THEN no request is made and the method returns immediately.
1994        assert!(results.is_empty());
1995        assert_eq!(start.elapsed(), Duration::ZERO);
1996    }
1997
1998    #[tokio::test(flavor = "current_thread", start_paused = true)]
1999    async fn synchronizer_periodic_task_when_commit_lagging_gets_disabled() {
2000        // GIVEN
2001        let (context, _) = Context::new_for_test(4);
2002        let context = Arc::new(context);
2003        let block_verifier = Arc::new(NoopBlockVerifier {});
2004        let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2005        let mock_client = Arc::new(MockNetworkClient::default());
2006        let store = Arc::new(MemStore::new());
2007        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2008        let transaction_vote_tracker =
2009            TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2010        let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2011        let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2012
2013        // AND stub some missing blocks. The highest accepted round is 0. Create blocks that are above the sync threshold.
2014        let sync_missing_block_round_threshold = context.parameters.commit_sync_batch_size;
2015        let stub_blocks = (sync_missing_block_round_threshold * 2
2016            ..sync_missing_block_round_threshold * 3)
2017            .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
2018            .collect::<Vec<_>>();
2019        let missing_blocks = stub_blocks
2020            .iter()
2021            .map(|block| block.reference())
2022            .collect::<BTreeSet<_>>();
2023        core_dispatcher
2024            .stub_missing_blocks(missing_blocks.clone())
2025            .await;
2026
2027        // AND stub the requests for authority 1 & 2
2028        // Make the first authority timeout, so the second will be called. "We" are authority = 0, so
2029        // we are skipped anyways. Stub all blocks since the full set is sent in one request.
2030        // Only the first max_blocks_per_sync blocks will be processed by process_fetched_blocks.
2031        mock_client
2032            .stub_fetch_blocks(
2033                stub_blocks.clone(),
2034                AuthorityIndex::new_for_test(1),
2035                Some(FETCH_REQUEST_TIMEOUT),
2036            )
2037            .await;
2038        mock_client
2039            .stub_fetch_blocks(stub_blocks.clone(), AuthorityIndex::new_for_test(2), None)
2040            .await;
2041        let mut expected_blocks = stub_blocks
2042            .iter()
2043            .take(context.parameters.max_blocks_per_sync)
2044            .cloned()
2045            .collect::<Vec<_>>();
2046
2047        // Now create some blocks to simulate a commit lag
2048        let round = context.parameters.commit_sync_batch_size * COMMIT_LAG_MULTIPLIER * 2;
2049        let commit_index: CommitIndex = round - 1;
2050        let blocks = (0..4)
2051            .map(|authority| {
2052                let commit_votes = vec![CommitVote::new(commit_index, CommitDigest::MIN)];
2053                let block = TestBlock::new(round, authority)
2054                    .set_commit_votes(commit_votes)
2055                    .build();
2056
2057                VerifiedBlock::new_for_test(block)
2058            })
2059            .collect::<Vec<_>>();
2060
2061        // Pass them through the commit vote monitor - so now there will be a big commit lag to prevent
2062        // the scheduled synchronizer from running
2063        for block in blocks {
2064            commit_vote_monitor.observe_block(&block);
2065        }
2066
2067        // WHEN start the synchronizer and wait for a couple of seconds where normally the synchronizer should have kicked in.
2068        let network_client = Arc::new(SynchronizerClient::new(
2069            context.clone(),
2070            Some(mock_client.clone()),
2071            Some(mock_client.clone()),
2072        ));
2073        let peers_pool = Arc::new(PeersPool::new(context.clone()));
2074        let _handle = Synchronizer::start(
2075            network_client,
2076            context.clone(),
2077            core_dispatcher.clone(),
2078            commit_vote_monitor.clone(),
2079            block_verifier,
2080            transaction_vote_tracker,
2081            round_tracker,
2082            dag_state.clone(),
2083            peers_pool.clone(),
2084            false,
2085        );
2086
2087        // Wait long enough for periodic sync to have run, but stay under COMMIT_PROGRESS_TIMEOUT
2088        // to avoid triggering commit sync failover.
2089        sleep(COMMIT_PROGRESS_TIMEOUT / 2).await;
2090
2091        // Since we should be in commit lag mode none of the missed blocks should have been fetched - hence nothing should be
2092        // sent to core for processing.
2093        let added_blocks = core_dispatcher.get_add_blocks().await;
2094        assert_eq!(added_blocks, vec![]);
2095
2096        // AND advance now the local commit index by adding a new commit that matches the commit index
2097        // of quorum
2098        {
2099            let mut d = dag_state.write();
2100            for index in 1..=commit_index {
2101                let commit =
2102                    TrustedCommit::new_for_test(index, CommitDigest::MIN, 0, BlockRef::MIN, vec![]);
2103
2104                d.add_commit(commit);
2105            }
2106
2107            assert_eq!(
2108                d.last_commit_index(),
2109                commit_vote_monitor.quorum_commit_index()
2110            );
2111        }
2112
2113        // Now stub again the missing blocks to fetch the exact same ones.
2114        core_dispatcher
2115            .stub_missing_blocks(missing_blocks.clone())
2116            .await;
2117
2118        sleep(2 * FETCH_REQUEST_TIMEOUT).await;
2119
2120        // THEN the missing blocks should now be fetched and added to core
2121        let mut added_blocks = core_dispatcher.get_add_blocks().await;
2122
2123        added_blocks.sort_by_key(|block| block.reference());
2124        expected_blocks.sort_by_key(|block| block.reference());
2125
2126        assert_eq!(added_blocks, expected_blocks);
2127    }
2128
2129    #[tokio::test(flavor = "current_thread", start_paused = true)]
2130    async fn synchronizer_periodic_sync_resumes_when_commit_sync_stalled() {
2131        // GIVEN
2132        let (context, _) = Context::new_for_test(4);
2133        let context = Arc::new(context);
2134        let block_verifier = Arc::new(NoopBlockVerifier {});
2135        let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2136        let mock_client = Arc::new(MockNetworkClient::default());
2137        let store = Arc::new(MemStore::new());
2138        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2139        let transaction_vote_tracker =
2140            TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2141        let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2142        let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2143
2144        // AND create missing blocks to fetch
2145        let sync_missing_block_round_threshold = context.parameters.commit_sync_batch_size;
2146        let stub_blocks = (sync_missing_block_round_threshold * 2
2147            ..sync_missing_block_round_threshold * 3)
2148            .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
2149            .collect::<Vec<_>>();
2150        let missing_blocks = stub_blocks
2151            .iter()
2152            .map(|block| block.reference())
2153            .collect::<BTreeSet<_>>();
2154        core_dispatcher
2155            .stub_missing_blocks(missing_blocks.clone())
2156            .await;
2157
2158        // AND stub the fetch responses for authority 1 & 2.
2159        // Stub all blocks since the full set is sent in one request.
2160        mock_client
2161            .stub_fetch_blocks(
2162                stub_blocks.clone(),
2163                AuthorityIndex::new_for_test(1),
2164                Some(FETCH_REQUEST_TIMEOUT),
2165            )
2166            .await;
2167        mock_client
2168            .stub_fetch_blocks(stub_blocks.clone(), AuthorityIndex::new_for_test(2), None)
2169            .await;
2170        let mut expected_blocks = stub_blocks
2171            .iter()
2172            .take(context.parameters.max_blocks_per_sync)
2173            .cloned()
2174            .collect::<Vec<_>>();
2175
2176        // AND create commit lag by observing high commit votes
2177        let round = context.parameters.commit_sync_batch_size * COMMIT_LAG_MULTIPLIER * 2;
2178        let commit_index: CommitIndex = round - 1;
2179        let blocks = (0..4)
2180            .map(|authority| {
2181                let commit_votes = vec![CommitVote::new(commit_index, CommitDigest::MIN)];
2182                let block = TestBlock::new(round, authority)
2183                    .set_commit_votes(commit_votes)
2184                    .build();
2185                VerifiedBlock::new_for_test(block)
2186            })
2187            .collect::<Vec<_>>();
2188        for block in blocks {
2189            commit_vote_monitor.observe_block(&block);
2190        }
2191
2192        // WHEN start the synchronizer
2193        let network_client = Arc::new(SynchronizerClient::new(
2194            context.clone(),
2195            Some(mock_client.clone()),
2196            Some(mock_client.clone()),
2197        ));
2198        let peers_pool = Arc::new(PeersPool::new(context.clone()));
2199        let _handle = Synchronizer::start(
2200            network_client,
2201            context.clone(),
2202            core_dispatcher.clone(),
2203            commit_vote_monitor.clone(),
2204            block_verifier,
2205            transaction_vote_tracker,
2206            round_tracker,
2207            dag_state.clone(),
2208            peers_pool.clone(),
2209            false,
2210        );
2211
2212        // Wait just under COMMIT_PROGRESS_TIMEOUT — sync should be skipped (commit lagging).
2213        sleep(COMMIT_PROGRESS_TIMEOUT - Duration::from_millis(100)).await;
2214        let added_blocks = core_dispatcher.get_add_blocks().await;
2215        assert_eq!(added_blocks, vec![]);
2216
2217        // Re-stub missing blocks (consumed by earlier get_missing_blocks calls).
2218        core_dispatcher
2219            .stub_missing_blocks(missing_blocks.clone())
2220            .await;
2221        // Stub the failover fetch (empty block_refs key, peer 1) before the stall triggers.
2222        // Use latency to prevent immediate completion and re-triggering.
2223        mock_client
2224            .stub_fetch_blocks_for_key(
2225                vec![],
2226                expected_blocks.clone(),
2227                AuthorityIndex::new_for_test(1),
2228                Some(Duration::from_millis(500)),
2229            )
2230            .await;
2231
2232        // Now sleep past the stall trigger + time for the fetch to complete.
2233        sleep(Duration::from_millis(200) + FETCH_REQUEST_TIMEOUT).await;
2234
2235        let mut added_blocks = core_dispatcher.get_add_blocks().await;
2236        assert!(
2237            !added_blocks.is_empty(),
2238            "Expected periodic sync to resume after commit sync stall"
2239        );
2240        added_blocks.sort_by_key(|block| block.reference());
2241        expected_blocks.sort_by_key(|block| block.reference());
2242        assert_eq!(added_blocks, expected_blocks);
2243
2244        // AND advance commit index enough to reach commit sync batch size from stall start.
2245        // This should resolve the failover and skip periodic sync again.
2246        core_dispatcher.get_add_blocks().await;
2247        {
2248            let current = dag_state.read().last_commit_index();
2249            let mut d = dag_state.write();
2250            // Advance well past the recovery threshold.
2251            for index in (current + 1)..=(current + context.parameters.commit_sync_batch_size) {
2252                let commit =
2253                    TrustedCommit::new_for_test(index, CommitDigest::MIN, 0, BlockRef::MIN, vec![]);
2254                d.add_commit(commit);
2255            }
2256        }
2257
2258        // Re-stub missing blocks and fetch response for another round of checking
2259        core_dispatcher
2260            .stub_missing_blocks(missing_blocks.clone())
2261            .await;
2262        mock_client
2263            .stub_fetch_blocks(
2264                stub_blocks
2265                    .iter()
2266                    .take(context.parameters.max_blocks_per_sync)
2267                    .cloned()
2268                    .collect::<Vec<_>>(),
2269                AuthorityIndex::new_for_test(2),
2270                None,
2271            )
2272            .await;
2273
2274        // Wait for a sync cycle — periodic sync should be skipped again since stall resolved
2275        // but commit is still lagging
2276        sleep(2 * FETCH_REQUEST_TIMEOUT).await;
2277        let added_blocks = core_dispatcher.get_add_blocks().await;
2278        assert_eq!(
2279            added_blocks,
2280            vec![],
2281            "Expected periodic sync to be skipped after stall resolved"
2282        );
2283    }
2284
2285    #[tokio::test(flavor = "current_thread", start_paused = true)]
2286    async fn synchronizer_fetch_own_last_block() {
2287        // GIVEN
2288        let (context, _) = Context::new_for_test(4);
2289        let context = Arc::new(context.with_parameters(Parameters {
2290            sync_last_known_own_block_timeout: Duration::from_millis(2_000),
2291            ..Default::default()
2292        }));
2293        let block_verifier = Arc::new(NoopBlockVerifier {});
2294        let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2295        let mock_client = Arc::new(MockNetworkClient::default());
2296        let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2297        let store = Arc::new(MemStore::new());
2298        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2299        let transaction_vote_tracker =
2300            TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2301        let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2302        let our_index = AuthorityIndex::new_for_test(0);
2303
2304        // Create some test blocks
2305        let mut expected_blocks = (9..=10)
2306            .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
2307            .collect::<Vec<_>>();
2308
2309        // Now set different latest blocks for the peers
2310        // For peer 1 we give the block of round 10 (highest)
2311        let block_1 = expected_blocks.pop().unwrap();
2312        mock_client
2313            .stub_fetch_latest_blocks(
2314                vec![block_1.clone()],
2315                AuthorityIndex::new_for_test(1),
2316                vec![our_index],
2317                None,
2318            )
2319            .await;
2320        mock_client
2321            .stub_fetch_latest_blocks(
2322                vec![block_1],
2323                AuthorityIndex::new_for_test(1),
2324                vec![our_index],
2325                None,
2326            )
2327            .await;
2328
2329        // For peer 2 we give the block of round 9
2330        let block_2 = expected_blocks.pop().unwrap();
2331        mock_client
2332            .stub_fetch_latest_blocks(
2333                vec![block_2.clone()],
2334                AuthorityIndex::new_for_test(2),
2335                vec![our_index],
2336                Some(Duration::from_secs(10)),
2337            )
2338            .await;
2339        mock_client
2340            .stub_fetch_latest_blocks(
2341                vec![block_2],
2342                AuthorityIndex::new_for_test(2),
2343                vec![our_index],
2344                None,
2345            )
2346            .await;
2347
2348        // For peer 3 we don't give any block - and it should return an empty vector
2349        mock_client
2350            .stub_fetch_latest_blocks(
2351                vec![],
2352                AuthorityIndex::new_for_test(3),
2353                vec![our_index],
2354                Some(Duration::from_secs(10)),
2355            )
2356            .await;
2357        mock_client
2358            .stub_fetch_latest_blocks(
2359                vec![],
2360                AuthorityIndex::new_for_test(3),
2361                vec![our_index],
2362                None,
2363            )
2364            .await;
2365
2366        // WHEN start the synchronizer and wait for a couple of seconds
2367        let network_client = Arc::new(SynchronizerClient::new(
2368            context.clone(),
2369            Some(mock_client.clone()),
2370            Some(mock_client.clone()),
2371        ));
2372        let peers_pool = Arc::new(PeersPool::new(context.clone()));
2373        let handle = Synchronizer::start(
2374            network_client,
2375            context.clone(),
2376            core_dispatcher.clone(),
2377            commit_vote_monitor,
2378            block_verifier,
2379            transaction_vote_tracker,
2380            round_tracker,
2381            dag_state,
2382            peers_pool.clone(),
2383            true,
2384        );
2385
2386        // Wait at least for the timeout time
2387        sleep(context.parameters.sync_last_known_own_block_timeout * 2).await;
2388
2389        // Assert that core has been called to set the min propose round
2390        assert_eq!(
2391            core_dispatcher.get_last_own_proposed_round().await,
2392            vec![10]
2393        );
2394
2395        // Ensure that all the requests have been called
2396        assert_eq!(mock_client.fetch_latest_blocks_pending_calls().await, 0);
2397
2398        // And we got one retry
2399        assert_eq!(
2400            context
2401                .metrics
2402                .node_metrics
2403                .sync_last_known_own_block_retries
2404                .get(),
2405            1
2406        );
2407
2408        // Ensure that no panic occurred: stop() propagates panics from synchronizer tasks.
2409        handle.stop().await;
2410    }
2411
2412    #[tokio::test]
2413    async fn test_process_fetched_blocks() {
2414        // GIVEN
2415        let (context, _) = Context::new_for_test(4);
2416        let context = Arc::new(context);
2417        let block_verifier = Arc::new(NoopBlockVerifier {});
2418        let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2419        let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2420        let store = Arc::new(MemStore::new());
2421        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2422        let transaction_vote_tracker =
2423            TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2424        let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2425        let (commands_sender, _commands_receiver) =
2426            monitored_mpsc::channel("consensus_synchronizer_commands", 1000);
2427
2428        // Create input test blocks:
2429        // - Authority 0 block at round 60.
2430        // - Authority 1 blocks from round 30 to 60.
2431        let mut expected_blocks = vec![VerifiedBlock::new_for_test(TestBlock::new(60, 0).build())];
2432        expected_blocks.extend(
2433            (30..=60).map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 1).build())),
2434        );
2435        assert_eq!(
2436            expected_blocks.len(),
2437            context.parameters.max_blocks_per_sync
2438        );
2439
2440        let expected_serialized_blocks = expected_blocks
2441            .iter()
2442            .map(|b| b.serialized().clone())
2443            .collect::<Vec<_>>();
2444
2445        let expected_block_refs = expected_blocks
2446            .iter()
2447            .map(|b| b.reference())
2448            .collect::<BTreeSet<_>>();
2449
2450        // GIVEN peer to fetch blocks from
2451        let peer_index = AuthorityIndex::new_for_test(2);
2452        let peer = PeerId::Validator(peer_index);
2453
2454        // Create blocks_guard
2455        let inflight_blocks_map = InflightBlocksMap::new();
2456        let blocks_guard = inflight_blocks_map
2457            .lock_blocks(expected_block_refs.clone(), peer.clone())
2458            .expect("Failed to lock blocks");
2459
2460        assert_eq!(
2461            inflight_blocks_map.num_of_locked_blocks(),
2462            expected_block_refs.len()
2463        );
2464
2465        // Create a Synchronizer
2466        let result = Synchronizer::<
2467            NoopBlockVerifier,
2468            MockCoreThreadDispatcher,
2469            MockNetworkClient,
2470            MockNetworkClient,
2471        >::process_fetched_blocks(
2472            expected_serialized_blocks,
2473            peer,
2474            blocks_guard, // The guard is consumed here
2475            core_dispatcher.clone(),
2476            block_verifier,
2477            transaction_vote_tracker,
2478            commit_vote_monitor,
2479            context.clone(),
2480            commands_sender,
2481            round_tracker,
2482            "test",
2483        )
2484        .await;
2485
2486        // THEN
2487        assert!(result.is_ok());
2488
2489        // Check blocks were sent to core
2490        let added_blocks = core_dispatcher.get_add_blocks().await;
2491        assert_eq!(
2492            added_blocks
2493                .iter()
2494                .map(|b| b.reference())
2495                .collect::<BTreeSet<_>>(),
2496            expected_block_refs,
2497        );
2498
2499        // Check blocks were unlocked
2500        assert_eq!(inflight_blocks_map.num_of_locked_blocks(), 0);
2501    }
2502}