1use std::{
4 collections::{BTreeMap, BTreeSet},
5 sync::Arc,
6 time::Duration,
7};
8
9use bytes::Bytes;
10use consensus_config::AuthorityIndex;
11use consensus_types::block::{BlockRef, Round, TransactionIndex};
12use futures::{StreamExt as _, stream::FuturesUnordered};
13use itertools::Itertools as _;
14use mysten_common::{ZipDebugEqIteratorExt, debug_fatal};
15use mysten_metrics::{
16 monitored_future,
17 monitored_mpsc::{Receiver, Sender, channel},
18 monitored_scope,
19};
20use parking_lot::{Mutex, RwLock};
21use rand::{prelude::SliceRandom as _, rngs::ThreadRng};
22use sui_macros::fail_point_async;
23use tap::TapFallible;
24use tokio::{
25 sync::{mpsc::error::TrySendError, oneshot},
26 task::{JoinError, JoinSet},
27 time::{Instant, sleep, sleep_until, timeout},
28};
29use tracing::{debug, info, trace, warn};
30
31use crate::{
32 BlockAPI,
33 block::{ExtendedBlock, SignedBlock, VerifiedBlock},
34 block_verifier::BlockVerifier,
35 commit::CommitIndex,
36 commit_vote_monitor::{CommitVoteMonitor, is_commit_lagging},
37 context::Context,
38 core_thread::CoreThreadDispatcher,
39 dag_state::DagState,
40 error::{ConsensusError, ConsensusResult},
41 network::{ObserverNetworkClient, PeerId, SynchronizerClient, ValidatorNetworkClient},
42 peers_pool::PeersPool,
43 round_tracker::RoundTracker,
44 task::{shutdown_join_set, spawn_blocking},
45 transaction_vote_tracker::TransactionVoteTracker,
46};
47
48const FETCH_BLOCKS_CONCURRENCY: usize = 5;
50
51const FETCH_REQUEST_TIMEOUT: Duration = Duration::from_millis(2_000);
53const FETCH_FROM_PEERS_TIMEOUT: Duration = Duration::from_millis(4_000);
54
55const MAX_AUTHORITIES_TO_FETCH_PER_BLOCK: usize = 2;
56
57const MAX_PERIODIC_SYNC_PEERS: usize = 3;
59
60const COMMIT_PROGRESS_TIMEOUT: Duration = Duration::from_secs(10);
62
63struct BlocksGuard {
64 map: Arc<InflightBlocksMap>,
65 block_refs: BTreeSet<BlockRef>,
66 peer: PeerId,
67}
68
69impl Drop for BlocksGuard {
70 fn drop(&mut self) {
71 self.map.unlock_blocks(&self.block_refs, self.peer.clone());
72 }
73}
74
75struct InflightBlocksMap {
80 inner: Mutex<BTreeMap<BlockRef, BTreeSet<PeerId>>>,
81}
82
83impl InflightBlocksMap {
84 fn new() -> Arc<Self> {
85 Arc::new(Self {
86 inner: Mutex::new(BTreeMap::new()),
87 })
88 }
89
90 fn lock_blocks(
96 self: &Arc<Self>,
97 missing_block_refs: BTreeSet<BlockRef>,
98 peer: PeerId,
99 ) -> Option<BlocksGuard> {
100 let mut blocks = BTreeSet::new();
101 let mut inner = self.inner.lock();
102
103 for block_ref in missing_block_refs {
104 let peers = inner.entry(block_ref).or_default();
107 if peers.len() < MAX_AUTHORITIES_TO_FETCH_PER_BLOCK && !peers.contains(&peer) {
108 assert!(peers.insert(peer.clone()));
109 blocks.insert(block_ref);
110 }
111 }
112
113 if blocks.is_empty() {
114 None
115 } else {
116 Some(BlocksGuard {
117 map: self.clone(),
118 block_refs: blocks,
119 peer,
120 })
121 }
122 }
123
124 fn unlock_blocks(self: &Arc<Self>, block_refs: &BTreeSet<BlockRef>, peer: PeerId) {
128 let mut blocks_to_fetch = self.inner.lock();
130 for block_ref in block_refs {
131 let peers = blocks_to_fetch
132 .get_mut(block_ref)
133 .expect("Should have found a non empty map");
134
135 assert!(peers.remove(&peer), "Peer should be present!");
136
137 if peers.is_empty() {
139 blocks_to_fetch.remove(block_ref);
140 }
141 }
142 }
143
144 fn swap_locks(
148 self: &Arc<Self>,
149 blocks_guard: BlocksGuard,
150 peer: PeerId,
151 ) -> Option<BlocksGuard> {
152 let block_refs = blocks_guard.block_refs.clone();
153
154 drop(blocks_guard);
156
157 self.lock_blocks(block_refs, peer)
159 }
160
161 #[cfg(test)]
162 fn num_of_locked_blocks(self: &Arc<Self>) -> usize {
163 let inner = self.inner.lock();
164 inner.len()
165 }
166}
167
168enum Command {
169 FetchBlocks {
170 missing_block_refs: BTreeSet<BlockRef>,
171 peer: PeerId,
172 result: oneshot::Sender<Result<(), ConsensusError>>,
173 },
174 FetchOwnLastBlock,
175 KickOffScheduler,
176 Shutdown {
177 result: oneshot::Sender<()>,
178 },
179}
180
181pub(crate) struct SynchronizerHandle {
182 commands_sender: Sender<Command>,
183 tasks: tokio::sync::Mutex<JoinSet<()>>,
184}
185
186impl SynchronizerHandle {
187 pub(crate) async fn fetch_blocks(
190 &self,
191 missing_block_refs: BTreeSet<BlockRef>,
192 peer: PeerId,
193 ) -> ConsensusResult<()> {
194 let (sender, receiver) = oneshot::channel();
195 self.commands_sender
196 .send(Command::FetchBlocks {
197 missing_block_refs,
198 peer,
199 result: sender,
200 })
201 .await
202 .map_err(|_err| ConsensusError::Shutdown)?;
203 receiver.await.map_err(|_err| ConsensusError::Shutdown)?
204 }
205
206 pub(crate) async fn stop(&self) {
207 let (shutdown_sender, shutdown_receiver) = oneshot::channel();
208 let _ = self
209 .commands_sender
210 .send(Command::Shutdown {
211 result: shutdown_sender,
212 })
213 .await;
214 let _ = shutdown_receiver.await;
215
216 let mut tasks = self.tasks.lock().await;
217 shutdown_join_set(&mut tasks).await;
218 }
219
220 #[cfg(test)]
221 pub(crate) fn new_for_test() -> Arc<Self> {
223 let (tx, _rx) = channel("test_synchronizer", 1);
224 Arc::new(Self {
225 commands_sender: tx,
226 tasks: tokio::sync::Mutex::new(JoinSet::new()),
227 })
228 }
229}
230
231pub(crate) struct Synchronizer<
253 V: BlockVerifier,
254 D: CoreThreadDispatcher,
255 VC: ValidatorNetworkClient,
256 OC: ObserverNetworkClient,
257> {
258 context: Arc<Context>,
259 commands_receiver: Receiver<Command>,
260 fetch_block_senders: BTreeMap<PeerId, Sender<BlocksGuard>>,
261 core_dispatcher: Arc<D>,
262 commit_vote_monitor: Arc<CommitVoteMonitor>,
263 dag_state: Arc<RwLock<DagState>>,
264 fetch_blocks_scheduler_task: JoinSet<()>,
265 fetch_own_last_block_task: JoinSet<()>,
266 network_client: Arc<SynchronizerClient<VC, OC>>,
267 block_verifier: Arc<V>,
268 transaction_vote_tracker: TransactionVoteTracker,
269 round_tracker: Arc<RwLock<RoundTracker>>,
270 inflight_blocks_map: Arc<InflightBlocksMap>,
271 commands_sender: Sender<Command>,
272 last_changed_commit_index: CommitIndex,
273 last_commit_change_time: Instant,
274 commit_sync_failover: bool,
276 peers_pool: Arc<PeersPool>,
277}
278
279impl<V, D, VC, OC> Synchronizer<V, D, VC, OC>
280where
281 V: BlockVerifier,
282 D: CoreThreadDispatcher,
283 VC: ValidatorNetworkClient,
284 OC: ObserverNetworkClient,
285{
286 pub(crate) fn start(
287 network_client: Arc<SynchronizerClient<VC, OC>>,
288 context: Arc<Context>,
289 core_dispatcher: Arc<D>,
290 commit_vote_monitor: Arc<CommitVoteMonitor>,
291 block_verifier: Arc<V>,
292 transaction_vote_tracker: TransactionVoteTracker,
293 round_tracker: Arc<RwLock<RoundTracker>>,
294 dag_state: Arc<RwLock<DagState>>,
295 peers_pool: Arc<PeersPool>,
296 sync_last_known_own_block: bool,
297 ) -> Arc<SynchronizerHandle> {
298 let (commands_sender, commands_receiver) =
299 channel("consensus_synchronizer_commands", 1_000);
300 let inflight_blocks_map = InflightBlocksMap::new();
301
302 let mut fetch_block_senders = BTreeMap::new();
304 let mut tasks = JoinSet::new();
305
306 let known_peers = peers_pool.get_known_peers();
309 for peer in known_peers {
310 let (sender, receiver) =
311 channel("consensus_synchronizer_fetches", FETCH_BLOCKS_CONCURRENCY);
312 let fetch_blocks_from_peer_async = Self::fetch_blocks_from_peer(
313 peer.clone(),
314 network_client.clone(),
315 block_verifier.clone(),
316 transaction_vote_tracker.clone(),
317 commit_vote_monitor.clone(),
318 context.clone(),
319 core_dispatcher.clone(),
320 dag_state.clone(),
321 receiver,
322 commands_sender.clone(),
323 round_tracker.clone(),
324 );
325 tasks.spawn(monitored_future!(fetch_blocks_from_peer_async));
326 fetch_block_senders.insert(peer, sender);
327 }
328
329 let commands_sender_clone = commands_sender.clone();
330
331 if sync_last_known_own_block {
332 commands_sender
333 .try_send(Command::FetchOwnLastBlock)
334 .expect("Failed to sync our last block");
335 }
336
337 tasks.spawn(monitored_future!(async move {
339 let mut s = Self {
340 context,
341 commands_receiver,
342 fetch_block_senders,
343 core_dispatcher,
344 commit_vote_monitor,
345 fetch_blocks_scheduler_task: JoinSet::new(),
346 fetch_own_last_block_task: JoinSet::new(),
347 network_client,
348 block_verifier,
349 transaction_vote_tracker,
350 inflight_blocks_map,
351 commands_sender: commands_sender_clone,
352 dag_state,
353 round_tracker,
354 last_changed_commit_index: 0,
355 last_commit_change_time: Instant::now(),
356 commit_sync_failover: false,
357 peers_pool,
358 };
359 s.run().await;
360 }));
361
362 Arc::new(SynchronizerHandle {
363 commands_sender,
364 tasks: tokio::sync::Mutex::new(tasks),
365 })
366 }
367
368 async fn run(&mut self) {
370 const PERIODIC_FETCH_INTERVAL: Duration = Duration::from_millis(200);
372 let scheduler_timeout = sleep_until(Instant::now() + PERIODIC_FETCH_INTERVAL);
373
374 tokio::pin!(scheduler_timeout);
375
376 loop {
377 tokio::select! {
378 Some(command) = self.commands_receiver.recv() => {
379 match command {
380 Command::FetchBlocks{ missing_block_refs, peer, result } => {
381 if !self.peers_pool.is_peer_known(&peer) {
383 result.send(Err(ConsensusError::PeerUnavailable(format!("{:?}", peer)))).ok();
384 continue;
385 }
386
387 let missing_block_refs = missing_block_refs
391 .into_iter()
392 .take(self.context.parameters.max_blocks_per_sync)
393 .collect();
394
395 let blocks_guard = self.inflight_blocks_map.lock_blocks(missing_block_refs, peer.clone());
396 let Some(blocks_guard) = blocks_guard else {
397 result.send(Ok(())).ok();
398 continue;
399 };
400
401 let r = self
404 .fetch_block_senders
405 .get(&peer)
406 .ok_or(ConsensusError::PeerNotFound(format!("Peer {} not found in fetch_block_senders", peer)))
407 .and_then(|sender| {
408 sender
409 .try_send(blocks_guard)
410 .map_err(|err| {
411 match err {
412 TrySendError::Full(_) => {
413 let peer_name = peer.labelname(&self.context);
414 self.context
415 .metrics
416 .node_metrics
417 .synchronizer_skipped_fetch_requests
418 .with_label_values(&[peer_name])
419 .inc();
420 ConsensusError::SynchronizerSaturated(format!("{:?}", peer))
421 },
422 TrySendError::Closed(_) => ConsensusError::Shutdown
423 }
424 })
425 });
426
427 result.send(r).ok();
428 }
429 Command::FetchOwnLastBlock => {
430 if self.fetch_own_last_block_task.is_empty() {
431 self.start_fetch_own_last_block_task();
432 }
433 }
434 Command::KickOffScheduler => {
435 let timeout = if self.fetch_blocks_scheduler_task.is_empty() {
438 Instant::now()
439 } else {
440 Instant::now() + PERIODIC_FETCH_INTERVAL / 2
441 };
442
443 if timeout < scheduler_timeout.deadline() {
445 scheduler_timeout.as_mut().reset(timeout);
446 }
447 }
448 Command::Shutdown { result } => {
449 self.shutdown_tasks().await;
450 self.fetch_block_senders.clear();
451 let _ = result.send(());
452 return;
453 }
454 }
455 },
456 Some(result) = self.fetch_own_last_block_task.join_next(), if !self.fetch_own_last_block_task.is_empty() => {
457 Self::handle_task_result(result, "fetch our last block");
458 },
459 Some(result) = self.fetch_blocks_scheduler_task.join_next(), if !self.fetch_blocks_scheduler_task.is_empty() => {
460 Self::handle_task_result(result, "fetch blocks scheduler");
461 },
462 () = &mut scheduler_timeout => {
463 if self.fetch_blocks_scheduler_task.is_empty()
466 && let Err(err) = self.start_fetch_missing_blocks_task().await {
467 debug!("Core is shutting down, synchronizer is shutting down: {err:?}");
468 self.shutdown_tasks().await;
469 return;
470 };
471
472 scheduler_timeout
473 .as_mut()
474 .reset(Instant::now() + PERIODIC_FETCH_INTERVAL);
475 }
476 }
477 }
478 }
479
480 fn handle_task_result(result: Result<(), JoinError>, task_name: &str) {
482 let Err(e) = result else {
483 return;
484 };
485 if e.is_cancelled() {
486 } else if e.is_panic() {
487 std::panic::resume_unwind(e.into_panic());
488 } else {
489 panic!("{task_name} task failed: {e}");
490 }
491 }
492
493 async fn shutdown_tasks(&mut self) {
496 shutdown_join_set(&mut self.fetch_own_last_block_task).await;
497 shutdown_join_set(&mut self.fetch_blocks_scheduler_task).await;
498 }
499
500 async fn fetch_blocks_from_peer(
501 peer: PeerId,
502 network_client: Arc<SynchronizerClient<VC, OC>>,
503 block_verifier: Arc<V>,
504 transaction_vote_tracker: TransactionVoteTracker,
505 commit_vote_monitor: Arc<CommitVoteMonitor>,
506 context: Arc<Context>,
507 core_dispatcher: Arc<D>,
508 dag_state: Arc<RwLock<DagState>>,
509 mut receiver: Receiver<BlocksGuard>,
510 commands_sender: Sender<Command>,
511 round_tracker: Arc<RwLock<RoundTracker>>,
512 ) {
513 const MAX_ATTEMPTS: u32 = 3;
514 let mut requests = FuturesUnordered::new();
515
516 loop {
517 tokio::select! {
518 Some(blocks_guard) = receiver.recv(), if requests.len() < FETCH_BLOCKS_CONCURRENCY => {
519 let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
520
521 requests.push(Self::fetch_blocks_request(network_client.clone(), peer.clone(), blocks_guard, fetch_after_rounds, true, FETCH_REQUEST_TIMEOUT, 0))
522 },
523 Some((response, blocks_guard, failed_attempts, _peer, fetch_after_rounds)) = requests.next() => {
524 match response {
525 Ok(blocks) => {
526 if let Err(err) = Self::process_fetched_blocks(blocks,
527 peer.clone(),
528 blocks_guard,
529 core_dispatcher.clone(),
530 block_verifier.clone(),
531 transaction_vote_tracker.clone(),
532 commit_vote_monitor.clone(),
533 context.clone(),
534 commands_sender.clone(),
535 round_tracker.clone(),
536 "live"
537 ).await {
538 warn!("Error while processing fetched blocks from peer {}: {err}", peer.hostname(&context));
539 context.metrics.node_metrics.synchronizer_process_fetched_failures.with_label_values(&[peer.labelname(&context).as_str(), "live"]).inc();
540 }
541 },
542 Err(_) => {
543 context.metrics.node_metrics.synchronizer_fetch_failures.with_label_values(&[peer.labelname(&context).as_str(), "live"]).inc();
544 if failed_attempts < MAX_ATTEMPTS {
545 requests.push(Self::fetch_blocks_request(network_client.clone(), peer.clone(), blocks_guard, fetch_after_rounds, true, FETCH_REQUEST_TIMEOUT, failed_attempts))
546 } else {
547 warn!("Max attempts {MAX_ATTEMPTS} reached while trying to fetch blocks from peer {}.", peer.hostname(&context));
548 drop(blocks_guard);
550 }
551 }
552 }
553 },
554 else => {
555 info!("Fetching blocks from peer {} task will now abort.", peer.hostname(&context));
556 break;
557 }
558 }
559 }
560 }
561
562 async fn process_fetched_blocks(
565 mut serialized_blocks: Vec<Bytes>,
566 peer: PeerId,
567 requested_blocks_guard: BlocksGuard,
568 core_dispatcher: Arc<D>,
569 block_verifier: Arc<V>,
570 transaction_vote_tracker: TransactionVoteTracker,
571 commit_vote_monitor: Arc<CommitVoteMonitor>,
572 context: Arc<Context>,
573 commands_sender: Sender<Command>,
574 round_tracker: Arc<RwLock<RoundTracker>>,
575 sync_method: &str,
576 ) -> ConsensusResult<()> {
577 if serialized_blocks.is_empty() {
578 return Ok(());
579 }
580
581 serialized_blocks.truncate(context.parameters.max_blocks_per_sync);
583
584 let (blocks, voted_blocks) = spawn_blocking({
586 let block_verifier = block_verifier.clone();
587 let context = context.clone();
588 let peer = peer.clone();
589 move || Self::verify_blocks(serialized_blocks, block_verifier, &context, peer)
590 })
591 .await??;
592
593 if context.protocol_config.transaction_voting_enabled() {
594 transaction_vote_tracker.add_voted_blocks(voted_blocks);
595 }
596
597 for block in &blocks {
599 commit_vote_monitor.observe_block(block);
600 }
601
602 {
605 let mut tracker = round_tracker.write();
606 for block in &blocks {
607 tracker.update_from_verified_block(&ExtendedBlock {
608 block: block.clone(),
609 excluded_ancestors: vec![],
610 });
611 }
612 }
613
614 let metrics = &context.metrics.node_metrics;
615 metrics
616 .synchronizer_fetched_blocks_by_peer
617 .with_label_values(&[peer.labelname(&context).as_str(), sync_method])
618 .inc_by(blocks.len() as u64);
619 for block in &blocks {
620 let block_hostname = &context.committee.authority(block.author()).hostname;
621 metrics
622 .synchronizer_fetched_blocks_by_authority
623 .with_label_values(&[block_hostname.as_str(), sync_method])
624 .inc();
625 }
626
627 debug!(
628 "Synced {} missing blocks from peer {:?}: {}",
629 blocks.len(),
630 peer,
631 blocks.iter().map(|b| b.reference().to_string()).join(", "),
632 );
633
634 let missing_blocks = core_dispatcher
638 .add_blocks(blocks)
639 .await
640 .map_err(|_| ConsensusError::Shutdown)?;
641
642 drop(requested_blocks_guard);
644
645 if !missing_blocks.is_empty() {
647 if let Err(TrySendError::Full(_)) = commands_sender.try_send(Command::KickOffScheduler)
649 {
650 warn!("Commands channel is full")
651 }
652 }
653
654 context
655 .metrics
656 .node_metrics
657 .missing_blocks_after_fetch_total
658 .inc_by(missing_blocks.len() as u64);
659
660 Ok(())
661 }
662
663 fn get_fetch_after_rounds(context: &Context, dag_state: &RwLock<DagState>) -> Vec<Round> {
664 let (blocks, gc_round) = {
665 let dag_state = dag_state.read();
666 (
667 dag_state.get_last_cached_block_per_authority(Round::MAX),
668 dag_state.gc_round(),
669 )
670 };
671 assert_eq!(blocks.len(), context.committee.size());
672
673 blocks
674 .into_iter()
675 .map(|(block, _)| block.round().max(gc_round))
676 .collect::<Vec<_>>()
677 }
678
679 fn verify_blocks(
680 serialized_blocks: Vec<Bytes>,
681 block_verifier: Arc<V>,
682 context: &Context,
683 peer: PeerId,
684 ) -> ConsensusResult<(
685 Vec<VerifiedBlock>,
686 Vec<(VerifiedBlock, Vec<TransactionIndex>)>,
687 )> {
688 let mut verified_blocks = Vec::new();
689 let mut voted_blocks = Vec::new();
690 for serialized_block in serialized_blocks {
691 let signed_block: SignedBlock =
692 bcs::from_bytes(&serialized_block).map_err(ConsensusError::MalformedBlock)?;
693
694 let (verified_block, reject_txn_votes) = block_verifier
696 .verify_and_vote(signed_block, serialized_block)
697 .tap_err(|e| {
698 let peer_label = peer.labelname(context);
699 context
700 .metrics
701 .node_metrics
702 .invalid_blocks
703 .with_label_values(&[peer_label.as_str(), "synchronizer", e.name()])
704 .inc();
705 info!("Invalid block received from {}: {}", peer, e);
706 })?;
707
708 let now = context.clock.timestamp_utc_ms();
710 let drift = verified_block.timestamp_ms().saturating_sub(now);
711 if drift > 0 {
712 let author_hostname = &context
713 .committee
714 .authority(verified_block.author())
715 .hostname;
716 context
717 .metrics
718 .node_metrics
719 .block_timestamp_drift_ms
720 .with_label_values(&[author_hostname.as_str(), "synchronizer"])
721 .inc_by(drift);
722
723 trace!(
724 "Synced block {} timestamp {} is in the future (now={}).",
725 verified_block.reference(),
726 verified_block.timestamp_ms(),
727 now
728 );
729 }
730
731 verified_blocks.push(verified_block.clone());
732 voted_blocks.push((verified_block, reject_txn_votes));
733 }
734
735 Ok((verified_blocks, voted_blocks))
736 }
737
738 async fn fetch_blocks_request(
739 network_client: Arc<SynchronizerClient<VC, OC>>,
740 peer: PeerId,
741 blocks_guard: BlocksGuard,
742 fetch_after_rounds: Vec<Round>,
743 fetch_missing_ancestors: bool,
744 request_timeout: Duration,
745 mut failed_attempts: u32,
746 ) -> (
747 ConsensusResult<Vec<Bytes>>,
748 BlocksGuard,
749 u32,
750 PeerId,
751 Vec<Round>,
752 ) {
753 let start = Instant::now();
754 let resp = timeout(
755 request_timeout,
756 network_client.fetch_blocks(
757 peer.clone(),
758 blocks_guard.block_refs.iter().copied().collect(),
759 fetch_after_rounds.clone(),
760 fetch_missing_ancestors,
761 request_timeout,
762 ),
763 )
764 .await;
765
766 fail_point_async!("consensus-delay");
767
768 let resp = match resp {
769 Ok(Err(err)) => {
770 sleep_until(start + request_timeout).await;
773 failed_attempts += 1;
774 Err(err)
775 } Err(err) => {
777 sleep_until(start + request_timeout).await;
779 failed_attempts += 1;
780 Err(ConsensusError::NetworkRequestTimeout(err.to_string()))
781 }
782 Ok(result) => result,
783 };
784 (
785 resp,
786 blocks_guard,
787 failed_attempts,
788 peer,
789 fetch_after_rounds,
790 )
791 }
792
793 fn start_fetch_own_last_block_task(&mut self) {
794 const FETCH_OWN_BLOCK_RETRY_DELAY: Duration = Duration::from_millis(1_000);
795 const MAX_RETRY_DELAY_STEP: Duration = Duration::from_millis(4_000);
796
797 let context = self.context.clone();
798 let dag_state = self.dag_state.clone();
799 let network_client = self.network_client.clone();
800 let block_verifier = self.block_verifier.clone();
801 let core_dispatcher = self.core_dispatcher.clone();
802
803 self.fetch_own_last_block_task
804 .spawn(monitored_future!(async move {
805 let _scope = monitored_scope("FetchOwnLastBlockTask");
806
807 let fetch_own_block = |authority_index: AuthorityIndex, fetch_own_block_delay: Duration| {
808 let network_client_cloned = network_client.clone();
809 let own_index = context.own_index;
810 async move {
811 sleep(fetch_own_block_delay).await;
812 let r = network_client_cloned.fetch_latest_blocks(authority_index, vec![own_index], FETCH_REQUEST_TIMEOUT).await;
813 (r, authority_index)
814 }
815 };
816
817
818 let process_blocks = |blocks: Vec<Bytes>, authority_index: AuthorityIndex| -> ConsensusResult<Vec<VerifiedBlock>> {
819 let mut result = Vec::new();
820 for serialized_block in blocks {
821 let signed_block = bcs::from_bytes(&serialized_block).map_err(ConsensusError::MalformedBlock)?;
822 let (verified_block, _) = block_verifier.verify_and_vote(signed_block, serialized_block).tap_err(|err|{
823 let hostname = context.committee.authority(authority_index).hostname.clone();
824 context
825 .metrics
826 .node_metrics
827 .invalid_blocks
828 .with_label_values(&[hostname.as_str(), "synchronizer_own_block", err.name()])
829 .inc();
830 warn!("Invalid block received from {}: {}", authority_index, err);
831 })?;
832
833 if verified_block.author() != context.own_index {
834 return Err(ConsensusError::UnexpectedLastOwnBlock { index: authority_index, block_ref: verified_block.reference()});
835 }
836 result.push(verified_block);
837 }
838 Ok(result)
839 };
840
841 let mut highest_round;
843 let mut retries = 0;
844 let mut retry_delay_step = Duration::from_millis(500);
845 'main:loop {
846 if context.committee.size() == 1 {
847 highest_round = dag_state.read().get_last_proposed_block().expect("Last proposed block should be returned on validators").round();
848 info!("Only one node in the network, will not try fetching own last block from peers.");
849 break 'main;
850 }
851
852 let mut total_stake = 0;
853 highest_round = 0;
854
855 let mut results = FuturesUnordered::new();
857
858 for (authority_index, _authority) in context.committee.authorities() {
859 if authority_index != context.own_index {
860 results.push(fetch_own_block(authority_index, Duration::from_millis(0)));
861 }
862 }
863
864 let timer = sleep_until(Instant::now() + context.parameters.sync_last_known_own_block_timeout);
866 tokio::pin!(timer);
867
868 'inner: loop {
869 tokio::select! {
870 result = results.next() => {
871 let Some((result, authority_index)) = result else {
872 break 'inner;
873 };
874 match result {
875 Ok(result) => {
876 match process_blocks(result, authority_index) {
877 Ok(blocks) => {
878 let max_round = blocks.into_iter().map(|b|b.round()).max().unwrap_or(0);
879 highest_round = highest_round.max(max_round);
880
881 total_stake += context.committee.stake(authority_index);
882 },
883 Err(err) => {
884 warn!("Invalid result returned from {authority_index} while fetching last own block: {err}");
885 }
886 }
887 },
888 Err(err) => {
889 warn!("Error {err} while fetching our own block from peer {authority_index}. Will retry.");
890 results.push(fetch_own_block(authority_index, FETCH_OWN_BLOCK_RETRY_DELAY));
891 }
892 }
893 },
894 () = &mut timer => {
895 info!("Timeout while trying to sync our own last block from peers");
896 break 'inner;
897 }
898 }
899 }
900
901 if context.committee.reached_validity(total_stake) {
903 info!("{} out of {} total stake returned acceptable results for our own last block with highest round {}, with {retries} retries.", total_stake, context.committee.total_stake(), highest_round);
904 break 'main;
905 }
906
907 retries += 1;
908 context.metrics.node_metrics.sync_last_known_own_block_retries.inc();
909 warn!("Not enough stake: {} out of {} total stake returned acceptable results for our own last block with highest round {}. Will now retry {retries}.", total_stake, context.committee.total_stake(), highest_round);
910
911 sleep(retry_delay_step).await;
912
913 retry_delay_step = Duration::from_secs_f64(retry_delay_step.as_secs_f64() * 1.5);
914 retry_delay_step = retry_delay_step.min(MAX_RETRY_DELAY_STEP);
915 }
916
917 context.metrics.node_metrics.last_known_own_block_round.set(highest_round as i64);
919
920 if let Err(err) = core_dispatcher.set_last_known_proposed_round(highest_round) {
921 warn!("Error received while calling dispatcher, probably dispatcher is shutting down, will now exit: {err:?}");
922 }
923 }));
924 }
925
926 async fn start_fetch_missing_blocks_task(&mut self) -> ConsensusResult<()> {
927 if self.context.committee.size() == 1 {
928 trace!(
929 "Only one node in the network, will not try fetching missing blocks from peers."
930 );
931 return Ok(());
932 }
933
934 if !self.should_run_periodic_sync() {
938 return Ok(());
939 }
940
941 let context = self.context.clone();
942 let network_client = self.network_client.clone();
943 let block_verifier = self.block_verifier.clone();
944 let transaction_vote_tracker = self.transaction_vote_tracker.clone();
945 let commit_vote_monitor = self.commit_vote_monitor.clone();
946 let core_dispatcher = self.core_dispatcher.clone();
947 let blocks_to_fetch = self.inflight_blocks_map.clone();
948 let commands_sender = self.commands_sender.clone();
949 let dag_state = self.dag_state.clone();
950 let round_tracker = self.round_tracker.clone();
951 let peers_pool = self.peers_pool.clone();
952
953 let mut missing_blocks = self
954 .core_dispatcher
955 .get_missing_blocks()
956 .await
957 .map_err(|_err| ConsensusError::Shutdown)?;
958 if self.commit_sync_failover {
959 let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
962 missing_blocks.retain(|block| block.round <= fetch_after_rounds[block.author.value()]);
963 } else if missing_blocks.is_empty() {
964 return Ok(());
965 }
966
967 self.fetch_blocks_scheduler_task
968 .spawn(monitored_future!(async move {
969 let _scope = monitored_scope("FetchMissingBlocksScheduler");
970 context
971 .metrics
972 .node_metrics
973 .fetch_blocks_scheduler_inflight
974 .inc();
975 let total_requested = missing_blocks.len();
976
977 let results = if missing_blocks.is_empty() {
978 let _scope = monitored_scope("BlockSync::Periodic::HighestAcceptedRounds");
979 Self::fetch_blocks_with_fetch_after_rounds(
981 context.clone(),
982 blocks_to_fetch.clone(),
983 network_client,
984 dag_state,
985 peers_pool,
986 )
987 .await
988 } else {
989 let _scope = monitored_scope("BlockSync::Periodic::MissingBlocks");
990 Self::fetch_blocks_from_peers(
992 context.clone(),
993 blocks_to_fetch.clone(),
994 network_client,
995 missing_blocks,
996 dag_state,
997 peers_pool,
998 )
999 .await
1000 };
1001 context
1002 .metrics
1003 .node_metrics
1004 .fetch_blocks_scheduler_inflight
1005 .dec();
1006 if results.is_empty() {
1007 return;
1008 }
1009
1010 fail_point_async!("consensus-delay");
1011
1012 let mut total_fetched = 0;
1014 for (blocks_guard, fetched_blocks, peer) in results {
1015 total_fetched += fetched_blocks.len();
1016
1017 if let Err(err) = Self::process_fetched_blocks(
1018 fetched_blocks,
1019 peer.clone(),
1020 blocks_guard,
1021 core_dispatcher.clone(),
1022 block_verifier.clone(),
1023 transaction_vote_tracker.clone(),
1024 commit_vote_monitor.clone(),
1025 context.clone(),
1026 commands_sender.clone(),
1027 round_tracker.clone(),
1028 "periodic",
1029 )
1030 .await
1031 {
1032 warn!(
1033 "Error occurred while processing fetched blocks from peer {:?}: {err}",
1034 peer
1035 );
1036 let peer_name = peer.labelname(&context);
1037 context
1038 .metrics
1039 .node_metrics
1040 .synchronizer_process_fetched_failures
1041 .with_label_values(&[peer_name.as_str(), "periodic"])
1042 .inc();
1043 }
1044 }
1045
1046 debug!(
1047 "Total blocks requested to fetch: {}, total fetched: {}",
1048 total_requested, total_fetched
1049 );
1050 }));
1051
1052 Ok(())
1053 }
1054
1055 fn should_run_periodic_sync(&mut self) -> bool {
1056 let current_commit_index = self.dag_state.read().last_commit_index();
1057 let quorum_commit_index = self.commit_vote_monitor.quorum_commit_index();
1058 let now = Instant::now();
1059 let metrics = &self.context.metrics.node_metrics;
1060
1061 if !is_commit_lagging(
1063 self.context.as_ref(),
1064 current_commit_index,
1065 quorum_commit_index,
1066 ) {
1067 metrics
1068 .synchronizer_periodic_sync_decision
1069 .with_label_values(&["true", "default"])
1070 .inc();
1071 self.last_changed_commit_index = current_commit_index;
1073 self.last_commit_change_time = now;
1074 self.commit_sync_failover = false;
1075 return true;
1077 }
1078 if self.commit_sync_failover {
1082 if current_commit_index
1083 < self.last_changed_commit_index + self.context.parameters.commit_sync_batch_size
1084 {
1085 metrics
1086 .synchronizer_periodic_sync_decision
1087 .with_label_values(&["true", "commit_catchup::run"])
1088 .inc();
1089 return true;
1093 } else {
1094 metrics
1095 .synchronizer_periodic_sync_decision
1096 .with_label_values(&["false", "commit_catchup::end"])
1097 .inc();
1098 self.last_changed_commit_index = current_commit_index;
1100 self.last_commit_change_time = now;
1101 self.commit_sync_failover = false;
1102 return false;
1104 }
1105 }
1106
1107 if current_commit_index == self.last_changed_commit_index {
1109 if now.duration_since(self.last_commit_change_time) >= COMMIT_PROGRESS_TIMEOUT {
1111 metrics
1112 .synchronizer_periodic_sync_decision
1113 .with_label_values(&["true", "commit_catchup::start"])
1114 .inc();
1115 self.commit_sync_failover = true;
1116 return true;
1118 }
1119 } else {
1120 self.last_changed_commit_index = current_commit_index;
1122 self.last_commit_change_time = now;
1123 }
1124
1125 metrics
1126 .synchronizer_periodic_sync_decision
1127 .with_label_values(&["false", "commit_lag"])
1128 .inc();
1129 false
1130 }
1131
1132 async fn fetch_blocks_with_fetch_after_rounds(
1135 context: Arc<Context>,
1136 inflight_blocks: Arc<InflightBlocksMap>,
1137 network_client: Arc<SynchronizerClient<VC, OC>>,
1138 dag_state: Arc<RwLock<DagState>>,
1139 peers_pool: Arc<PeersPool>,
1140 ) -> Vec<(BlocksGuard, Vec<Bytes>, PeerId)> {
1141 let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
1142
1143 let peers = peers_pool.get_known_peers();
1146
1147 assert!(!peers.is_empty(), "No known peers to fetch blocks from");
1150
1151 let peer = if cfg!(test) {
1152 peers.first()
1153 } else {
1154 peers.choose(&mut ThreadRng::default())
1155 }
1156 .unwrap()
1157 .clone();
1158
1159 let response = timeout(
1160 FETCH_REQUEST_TIMEOUT,
1161 network_client.fetch_blocks(
1162 peer.clone(),
1163 vec![],
1164 fetch_after_rounds,
1165 false,
1166 FETCH_REQUEST_TIMEOUT,
1167 ),
1168 )
1169 .await;
1170
1171 let serialized_blocks = match response {
1172 Ok(Ok(blocks)) => blocks,
1173 Ok(Err(err)) => {
1174 debug!("Failed to fetch blocks with fetch_after_rounds from peer {peer}: {err}");
1175 return vec![];
1176 }
1177 Err(_) => {
1178 debug!("Timed out fetching blocks with fetch_after_rounds from peer {peer}");
1179 return vec![];
1180 }
1181 };
1182
1183 let blocks_guard = BlocksGuard {
1184 map: inflight_blocks,
1185 block_refs: BTreeSet::new(),
1186 peer: peer.clone(),
1187 };
1188
1189 vec![(blocks_guard, serialized_blocks, peer)]
1190 }
1191
1192 async fn fetch_blocks_from_peers(
1198 context: Arc<Context>,
1199 inflight_blocks: Arc<InflightBlocksMap>,
1200 network_client: Arc<SynchronizerClient<VC, OC>>,
1201 missing_blocks: BTreeSet<BlockRef>,
1202 dag_state: Arc<RwLock<DagState>>,
1203 peers_pool: Arc<PeersPool>,
1204 ) -> Vec<(BlocksGuard, Vec<Bytes>, PeerId)> {
1205 let missing_blocks = missing_blocks
1209 .into_iter()
1210 .take(2 * MAX_PERIODIC_SYNC_PEERS * context.parameters.max_blocks_per_fetch)
1211 .collect::<Vec<_>>();
1212
1213 let mut authorities = BTreeMap::<AuthorityIndex, Vec<BlockRef>>::new();
1215 for block_ref in &missing_blocks {
1216 authorities
1217 .entry(block_ref.author)
1218 .or_default()
1219 .push(*block_ref);
1220 }
1221
1222 let mut peers = peers_pool.get_known_peers();
1224
1225 assert!(!peers.is_empty(), "No known peers to fetch blocks from");
1230
1231 let num_authorities_per_peer = authorities
1232 .len()
1233 .div_ceil(peers.len().min(MAX_PERIODIC_SYNC_PEERS));
1234
1235 let mut missing_blocks_per_authority = vec![0; context.committee.size()];
1237 for (authority, blocks) in &authorities {
1238 missing_blocks_per_authority[*authority] += blocks.len();
1239 }
1240 for (missing, (_, authority)) in missing_blocks_per_authority
1241 .into_iter()
1242 .zip_debug_eq(context.committee.authorities())
1243 {
1244 context
1245 .metrics
1246 .node_metrics
1247 .synchronizer_missing_blocks_by_authority
1248 .with_label_values(&[&authority.hostname])
1249 .inc_by(missing as u64);
1250 context
1251 .metrics
1252 .node_metrics
1253 .synchronizer_current_missing_blocks_by_authority
1254 .with_label_values(&[&authority.hostname])
1255 .set(missing as i64);
1256 }
1257
1258 if cfg!(not(test)) {
1260 peers.shuffle(&mut ThreadRng::default());
1262 }
1263
1264 let mut peers = peers.into_iter();
1265 let mut request_futures = FuturesUnordered::new();
1266
1267 let mut authorities = authorities.into_values().collect::<Vec<_>>();
1269 if cfg!(not(test)) {
1270 authorities.shuffle(&mut ThreadRng::default());
1272 }
1273
1274 let fetch_after_rounds = Self::get_fetch_after_rounds(&context, &dag_state);
1275
1276 for batch in authorities.chunks(num_authorities_per_peer) {
1278 let Some(peer) = peers.next() else {
1279 debug_fatal!("No more peers left to fetch blocks!");
1280 break;
1281 };
1282 let peer_name = peer.hostname(&context);
1283 let block_refs = batch
1286 .iter()
1287 .flatten()
1288 .cloned()
1289 .collect::<BTreeSet<_>>()
1290 .into_iter()
1291 .take(context.parameters.max_blocks_per_fetch)
1292 .collect::<BTreeSet<_>>();
1293
1294 if let Some(blocks_guard) = inflight_blocks.lock_blocks(block_refs, peer.clone()) {
1296 info!(
1297 "Periodic sync of {} missing blocks from peer {} {:?}: {}",
1298 blocks_guard.block_refs.len(),
1299 peer_name.as_str(),
1300 peer,
1301 blocks_guard
1302 .block_refs
1303 .iter()
1304 .map(|b| b.to_string())
1305 .collect::<Vec<_>>()
1306 .join(", ")
1307 );
1308 request_futures.push(Self::fetch_blocks_request(
1309 network_client.clone(),
1310 peer,
1311 blocks_guard,
1312 fetch_after_rounds.clone(),
1313 false,
1314 FETCH_REQUEST_TIMEOUT,
1315 0,
1316 ));
1317 }
1318 }
1319
1320 let mut results = Vec::new();
1321 let fetcher_timeout = sleep(FETCH_FROM_PEERS_TIMEOUT);
1322
1323 tokio::pin!(fetcher_timeout);
1324
1325 while !request_futures.is_empty() {
1327 tokio::select! {
1328 Some((response, blocks_guard, _failed_attempts, peer, fetch_after_rounds)) = request_futures.next() => {
1329 match response {
1330 Ok(fetched_blocks) => {
1331 results.push((blocks_guard, fetched_blocks, peer));
1332 },
1333 Err(_) => {
1334 let peer_name = peer.labelname(&context);
1335 context.metrics.node_metrics.synchronizer_fetch_failures.with_label_values(&[peer_name.as_str(), "periodic"]).inc();
1336 if let Some(next_peer) = peers.next() {
1338 if let Some(blocks_guard) = inflight_blocks.swap_locks(blocks_guard, next_peer.clone()) {
1340 info!(
1341 "Retrying syncing {} missing blocks from peer {:?}: {}",
1342 blocks_guard.block_refs.len(),
1343 next_peer,
1344 blocks_guard.block_refs
1345 .iter()
1346 .map(|b| b.to_string())
1347 .collect::<Vec<_>>()
1348 .join(", ")
1349 );
1350 request_futures.push(Self::fetch_blocks_request(
1351 network_client.clone(),
1352 next_peer,
1353 blocks_guard,
1354 fetch_after_rounds,
1355 false,
1356 FETCH_REQUEST_TIMEOUT,
1357 0,
1358 ));
1359 } else {
1360 debug!("Couldn't acquire locks to fetch blocks from peer {:?}.", next_peer)
1361 }
1362 } else {
1363 debug!("No more peers left to fetch blocks");
1364 }
1365 }
1366 }
1367 },
1368 _ = &mut fetcher_timeout => {
1369 debug!("Timed out while fetching missing blocks");
1370 break;
1371 }
1372 }
1373 }
1374
1375 results
1376 }
1377}
1378
1379#[cfg(test)]
1380mod tests {
1381 use std::{
1382 collections::{BTreeMap, BTreeSet},
1383 sync::Arc,
1384 time::Duration,
1385 };
1386
1387 use async_trait::async_trait;
1388 use bytes::Bytes;
1389 use consensus_config::{AuthorityIndex, Parameters};
1390 use consensus_types::block::{BlockDigest, BlockRef, Round};
1391 use mysten_metrics::monitored_mpsc;
1392 use parking_lot::RwLock;
1393 use tokio::{sync::Mutex, time::sleep};
1394
1395 use crate::commit::{CommitVote, TrustedCommit};
1396 use crate::{
1397 CommitDigest, CommitIndex,
1398 block::{TestBlock, VerifiedBlock},
1399 block_verifier::NoopBlockVerifier,
1400 commit_vote_monitor::CommitVoteMonitor,
1401 context::Context,
1402 core_thread::CoreThreadDispatcher,
1403 dag_state::DagState,
1404 error::{ConsensusError, ConsensusResult},
1405 network::{
1406 BlockStream, ObserverNetworkClient, PeerId, SynchronizerClient, ValidatorNetworkClient,
1407 },
1408 storage::mem_store::MemStore,
1409 synchronizer::{
1410 COMMIT_PROGRESS_TIMEOUT, FETCH_BLOCKS_CONCURRENCY, FETCH_FROM_PEERS_TIMEOUT,
1411 FETCH_REQUEST_TIMEOUT, InflightBlocksMap, Synchronizer,
1412 },
1413 };
1414 use crate::{
1415 commit_vote_monitor::COMMIT_LAG_MULTIPLIER, core_thread::MockCoreThreadDispatcher,
1416 peers_pool::PeersPool, round_tracker::RoundTracker,
1417 transaction_vote_tracker::TransactionVoteTracker,
1418 };
1419
1420 type FetchRequestKey = (Vec<BlockRef>, AuthorityIndex);
1421 type FetchRequestResponse = (Vec<VerifiedBlock>, Option<Duration>);
1422 type FetchLatestBlockKey = (AuthorityIndex, Vec<AuthorityIndex>);
1423 type FetchLatestBlockResponse = (Vec<VerifiedBlock>, Option<Duration>);
1424
1425 #[derive(Default)]
1426 struct MockNetworkClient {
1427 fetch_blocks_requests: Mutex<BTreeMap<FetchRequestKey, FetchRequestResponse>>,
1428 fetch_latest_blocks_requests:
1429 Mutex<BTreeMap<FetchLatestBlockKey, Vec<FetchLatestBlockResponse>>>,
1430 }
1431
1432 impl MockNetworkClient {
1433 async fn stub_fetch_blocks(
1434 &self,
1435 blocks: Vec<VerifiedBlock>,
1436 peer: AuthorityIndex,
1437 latency: Option<Duration>,
1438 ) {
1439 let mut lock = self.fetch_blocks_requests.lock().await;
1440 let block_refs = blocks
1441 .iter()
1442 .map(|block| block.reference())
1443 .collect::<Vec<_>>();
1444 lock.insert((block_refs, peer), (blocks, latency));
1445 }
1446
1447 async fn stub_fetch_blocks_for_key(
1448 &self,
1449 key_refs: Vec<BlockRef>,
1450 response_blocks: Vec<VerifiedBlock>,
1451 peer: AuthorityIndex,
1452 latency: Option<Duration>,
1453 ) {
1454 let mut lock = self.fetch_blocks_requests.lock().await;
1455 lock.insert((key_refs, peer), (response_blocks, latency));
1456 }
1457
1458 async fn stub_fetch_latest_blocks(
1459 &self,
1460 blocks: Vec<VerifiedBlock>,
1461 peer: AuthorityIndex,
1462 authorities: Vec<AuthorityIndex>,
1463 latency: Option<Duration>,
1464 ) {
1465 let mut lock = self.fetch_latest_blocks_requests.lock().await;
1466 lock.entry((peer, authorities))
1467 .or_default()
1468 .push((blocks, latency));
1469 }
1470
1471 async fn fetch_latest_blocks_pending_calls(&self) -> usize {
1472 let lock = self.fetch_latest_blocks_requests.lock().await;
1473 lock.len()
1474 }
1475 }
1476
1477 #[async_trait]
1478 impl ValidatorNetworkClient for MockNetworkClient {
1479 async fn subscribe_blocks(
1480 &self,
1481 _peer: AuthorityIndex,
1482 _last_received: Round,
1483 _timeout: Duration,
1484 ) -> ConsensusResult<BlockStream> {
1485 unimplemented!("subscribe_blocks not implemented in mock")
1486 }
1487
1488 async fn fetch_blocks(
1489 &self,
1490 peer: AuthorityIndex,
1491 block_refs: Vec<BlockRef>,
1492 _fetch_after_rounds: Vec<Round>,
1493 _fetch_missing_ancestors: bool,
1494 _timeout: Duration,
1495 ) -> ConsensusResult<Vec<Bytes>> {
1496 let mut lock = self.fetch_blocks_requests.lock().await;
1497 let response = lock.remove(&(block_refs.clone(), peer)).unwrap_or_else(|| {
1498 panic!(
1499 "Unexpected fetch blocks request made: {:?} {}. Current lock: {:?}",
1500 block_refs, peer, lock
1501 );
1502 });
1503
1504 let serialised = response
1505 .0
1506 .into_iter()
1507 .map(|block| block.serialized().clone())
1508 .collect::<Vec<_>>();
1509
1510 drop(lock);
1511
1512 if let Some(latency) = response.1 {
1513 sleep(latency).await;
1514 }
1515
1516 Ok(serialised)
1517 }
1518
1519 async fn fetch_commits(
1520 &self,
1521 _peer: AuthorityIndex,
1522 _commit_range: crate::commit::CommitRange,
1523 _timeout: Duration,
1524 ) -> ConsensusResult<(Vec<Bytes>, Vec<Bytes>)> {
1525 unimplemented!("fetch_commits not implemented in mock")
1526 }
1527
1528 async fn fetch_latest_blocks(
1529 &self,
1530 peer: AuthorityIndex,
1531 authorities: Vec<AuthorityIndex>,
1532 _timeout: Duration,
1533 ) -> ConsensusResult<Vec<Bytes>> {
1534 let mut lock = self.fetch_latest_blocks_requests.lock().await;
1535 let mut responses = lock
1536 .remove(&(peer, authorities.clone()))
1537 .expect("Unexpected fetch blocks request made");
1538
1539 let response = responses.remove(0);
1540 let serialised = response
1541 .0
1542 .into_iter()
1543 .map(|block| block.serialized().clone())
1544 .collect::<Vec<_>>();
1545
1546 if !responses.is_empty() {
1547 lock.insert((peer, authorities), responses);
1548 }
1549
1550 drop(lock);
1551
1552 if let Some(latency) = response.1 {
1553 sleep(latency).await;
1554 }
1555
1556 Ok(serialised)
1557 }
1558
1559 async fn get_latest_rounds(
1560 &self,
1561 _peer: AuthorityIndex,
1562 _timeout: Duration,
1563 ) -> ConsensusResult<(Vec<Round>, Vec<Round>)> {
1564 unimplemented!("get_latest_rounds not implemented in mock")
1565 }
1566
1567 #[cfg(all(test, not(msim)))]
1568 async fn send_block(
1569 &self,
1570 _peer: AuthorityIndex,
1571 _block: &VerifiedBlock,
1572 _timeout: Duration,
1573 ) -> ConsensusResult<()> {
1574 unimplemented!("send_block not implemented in mock")
1575 }
1576 }
1577
1578 #[async_trait]
1579 impl ObserverNetworkClient for MockNetworkClient {
1580 async fn stream_blocks(
1581 &self,
1582 _peer: crate::network::PeerId,
1583 _highest_round_per_authority: Vec<Round>,
1584 _filter: crate::network::BlockStreamFilter,
1585 _timeout: Duration,
1586 ) -> ConsensusResult<crate::network::ObserverBlockStream> {
1587 unimplemented!("stream_blocks not implemented in mock")
1588 }
1589
1590 async fn fetch_blocks(
1591 &self,
1592 _peer: crate::network::PeerId,
1593 _block_refs: Vec<BlockRef>,
1594 _fetch_after_rounds: Vec<Round>,
1595 _fetch_missing_ancestors: bool,
1596 _timeout: Duration,
1597 ) -> ConsensusResult<Vec<Bytes>> {
1598 unimplemented!("Observer fetch_blocks not implemented in mock")
1599 }
1600
1601 async fn fetch_commits(
1602 &self,
1603 _peer: crate::network::PeerId,
1604 _commit_range: crate::commit::CommitRange,
1605 _timeout: Duration,
1606 ) -> ConsensusResult<(Vec<Bytes>, Vec<Bytes>)> {
1607 unimplemented!("Observer fetch_commits not implemented in mock")
1608 }
1609 }
1610
1611 #[test]
1612 fn test_inflight_blocks_map() {
1613 let map = InflightBlocksMap::new();
1615 let some_block_refs = [
1616 BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN),
1617 BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN),
1618 BlockRef::new(12, AuthorityIndex::new_for_test(3), BlockDigest::MIN),
1619 BlockRef::new(15, AuthorityIndex::new_for_test(2), BlockDigest::MIN),
1620 ];
1621 let missing_block_refs = some_block_refs.iter().cloned().collect::<BTreeSet<_>>();
1622
1623 {
1625 let mut all_guards = Vec::new();
1626
1627 for i in 1..=2 {
1629 let authority = AuthorityIndex::new_for_test(i);
1630 let peer = PeerId::Validator(authority);
1631
1632 let guard = map.lock_blocks(missing_block_refs.clone(), peer.clone());
1633 let guard = guard.expect("Guard should be created");
1634 assert_eq!(guard.block_refs.len(), 4);
1635
1636 all_guards.push(guard);
1637
1638 let guard = map.lock_blocks(missing_block_refs.clone(), peer);
1640 assert!(guard.is_none());
1641 }
1642
1643 let authority_3 = AuthorityIndex::new_for_test(3);
1645 let peer_3 = PeerId::Validator(authority_3);
1646
1647 let guard = map.lock_blocks(missing_block_refs.clone(), peer_3.clone());
1648 assert!(guard.is_none());
1649
1650 drop(all_guards.remove(0));
1652
1653 let guard = map.lock_blocks(missing_block_refs.clone(), peer_3);
1654 let guard = guard.expect("Guard should be successfully acquired");
1655
1656 assert_eq!(guard.block_refs, missing_block_refs);
1657
1658 drop(guard);
1660 drop(all_guards);
1661
1662 assert_eq!(map.num_of_locked_blocks(), 0);
1663 }
1664
1665 {
1667 let authority_1 = AuthorityIndex::new_for_test(1);
1669 let peer_1 = PeerId::Validator(authority_1);
1670 let guard = map.lock_blocks(missing_block_refs.clone(), peer_1).unwrap();
1671
1672 let authority_2 = AuthorityIndex::new_for_test(2);
1674 let peer_2 = PeerId::Validator(authority_2);
1675 let guard = map.swap_locks(guard, peer_2);
1676
1677 assert_eq!(guard.unwrap().block_refs, missing_block_refs);
1678 }
1679 }
1680
1681 #[tokio::test]
1682 async fn successful_fetch_blocks_from_peer() {
1683 let (context, _) = Context::new_for_test(4);
1685 let context = Arc::new(context);
1686 let block_verifier = Arc::new(NoopBlockVerifier {});
1687 let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
1688 let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
1689 let mock_client = Arc::new(MockNetworkClient::default());
1690 let store = Arc::new(MemStore::new());
1691 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1692 let transaction_vote_tracker =
1693 TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
1694 let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
1695
1696 let network_client = Arc::new(SynchronizerClient::new(
1697 context.clone(),
1698 Some(mock_client.clone()),
1699 Some(mock_client.clone()),
1700 ));
1701 let peers_pool = Arc::new(PeersPool::new(context.clone()));
1702 let handle = Synchronizer::start(
1703 network_client,
1704 context.clone(),
1705 core_dispatcher.clone(),
1706 commit_vote_monitor,
1707 block_verifier,
1708 transaction_vote_tracker,
1709 round_tracker,
1710 dag_state,
1711 peers_pool.clone(),
1712 false,
1713 );
1714
1715 let expected_blocks = (0..10)
1717 .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
1718 .collect::<Vec<_>>();
1719 let missing_blocks = expected_blocks
1720 .iter()
1721 .map(|block| block.reference())
1722 .collect::<BTreeSet<_>>();
1723
1724 let peer = AuthorityIndex::new_for_test(1);
1726 mock_client
1727 .stub_fetch_blocks(expected_blocks.clone(), peer, None)
1728 .await;
1729
1730 assert!(
1732 handle
1733 .fetch_blocks(missing_blocks, PeerId::Validator(peer))
1734 .await
1735 .is_ok()
1736 );
1737
1738 sleep(Duration::from_millis(1_000)).await;
1740
1741 let added_blocks = core_dispatcher.get_add_blocks().await;
1743 assert_eq!(added_blocks, expected_blocks);
1744 }
1745
1746 #[tokio::test]
1747 async fn saturate_fetch_blocks_from_peer() {
1748 let (context, _) = Context::new_for_test(4);
1750 let context = Arc::new(context);
1751 let block_verifier = Arc::new(NoopBlockVerifier {});
1752 let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
1753 let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
1754 let mock_client = Arc::new(MockNetworkClient::default());
1755 let store = Arc::new(MemStore::new());
1756 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1757 let transaction_vote_tracker =
1758 TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
1759 let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
1760
1761 let network_client = Arc::new(SynchronizerClient::new(
1762 context.clone(),
1763 Some(mock_client.clone()),
1764 Some(mock_client.clone()),
1765 ));
1766 let peers_pool = Arc::new(PeersPool::new(context.clone()));
1767 let handle = Synchronizer::start(
1768 network_client,
1769 context.clone(),
1770 core_dispatcher.clone(),
1771 commit_vote_monitor,
1772 block_verifier,
1773 transaction_vote_tracker,
1774 round_tracker,
1775 dag_state,
1776 peers_pool.clone(),
1777 false,
1778 );
1779
1780 let expected_blocks = (0..=2 * FETCH_BLOCKS_CONCURRENCY)
1782 .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round as Round, 0).build()))
1783 .collect::<Vec<_>>();
1784
1785 let peer = AuthorityIndex::new_for_test(1);
1787 let mut iter = expected_blocks.iter().peekable();
1788 while let Some(block) = iter.next() {
1789 mock_client
1792 .stub_fetch_blocks(
1793 vec![block.clone()],
1794 peer,
1795 Some(Duration::from_millis(5_000)),
1796 )
1797 .await;
1798
1799 let mut missing_blocks = BTreeSet::new();
1800 missing_blocks.insert(block.reference());
1801
1802 if iter.peek().is_none() {
1805 match handle
1806 .fetch_blocks(missing_blocks, PeerId::Validator(peer))
1807 .await
1808 {
1809 Err(ConsensusError::SynchronizerSaturated(peer_str)) => {
1810 assert_eq!(peer_str, format!("{:?}", PeerId::Validator(peer)));
1811 }
1812 _ => panic!("A saturated synchronizer error was expected"),
1813 }
1814 } else {
1815 assert!(
1816 handle
1817 .fetch_blocks(missing_blocks, PeerId::Validator(peer))
1818 .await
1819 .is_ok()
1820 );
1821 }
1822 }
1823 }
1824
1825 #[tokio::test(flavor = "current_thread", start_paused = true)]
1826 async fn synchronizer_periodic_task_fetch_blocks() {
1827 let (context, _) = Context::new_for_test(4);
1829 let context = Arc::new(context);
1830 let block_verifier = Arc::new(NoopBlockVerifier {});
1831 let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
1832 let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
1833 let mock_client = Arc::new(MockNetworkClient::default());
1834 let store = Arc::new(MemStore::new());
1835 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1836 let transaction_vote_tracker =
1837 TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
1838 let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
1839
1840 let expected_blocks = (0..10)
1842 .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
1843 .collect::<Vec<_>>();
1844 let missing_blocks = expected_blocks
1845 .iter()
1846 .map(|block| block.reference())
1847 .collect::<BTreeSet<_>>();
1848
1849 core_dispatcher
1851 .stub_missing_blocks(missing_blocks.clone())
1852 .await;
1853
1854 mock_client
1858 .stub_fetch_blocks(
1859 expected_blocks.clone(),
1860 AuthorityIndex::new_for_test(1),
1861 Some(FETCH_REQUEST_TIMEOUT),
1862 )
1863 .await;
1864 mock_client
1865 .stub_fetch_blocks(
1866 expected_blocks.clone(),
1867 AuthorityIndex::new_for_test(2),
1868 None,
1869 )
1870 .await;
1871
1872 let network_client = Arc::new(SynchronizerClient::new(
1874 context.clone(),
1875 Some(mock_client.clone()),
1876 Some(mock_client.clone()),
1877 ));
1878 let peers_pool = Arc::new(PeersPool::new(context.clone()));
1879 let _handle = Synchronizer::start(
1880 network_client,
1881 context.clone(),
1882 core_dispatcher.clone(),
1883 commit_vote_monitor,
1884 block_verifier,
1885 transaction_vote_tracker,
1886 round_tracker,
1887 dag_state,
1888 peers_pool.clone(),
1889 false,
1890 );
1891
1892 sleep(2 * FETCH_REQUEST_TIMEOUT).await;
1893
1894 let added_blocks = core_dispatcher.get_add_blocks().await;
1896 assert_eq!(added_blocks, expected_blocks);
1897
1898 assert!(
1900 core_dispatcher
1901 .get_missing_blocks()
1902 .await
1903 .unwrap()
1904 .is_empty()
1905 );
1906 }
1907
1908 #[tokio::test(flavor = "current_thread", start_paused = true)]
1909 async fn fetch_blocks_from_peers_returns_when_no_requests_are_pending() {
1910 type TestSynchronizer = Synchronizer<
1911 NoopBlockVerifier,
1912 MockCoreThreadDispatcher,
1913 MockNetworkClient,
1914 MockNetworkClient,
1915 >;
1916
1917 let (context, _) = Context::new_for_test(4);
1919 let context = Arc::new(context);
1920 let mock_client = Arc::new(MockNetworkClient::default());
1921 let store = Arc::new(MemStore::new());
1922 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
1923 let network_client = Arc::new(SynchronizerClient::new(
1924 context.clone(),
1925 Some(mock_client.clone()),
1926 Some(mock_client.clone()),
1927 ));
1928 let peers_pool = Arc::new(PeersPool::new(context.clone()));
1929 let inflight_blocks = InflightBlocksMap::new();
1930
1931 let blocks = (1..=3)
1933 .map(|author| VerifiedBlock::new_for_test(TestBlock::new(1, author).build()))
1934 .collect::<Vec<_>>();
1935 let missing_blocks = blocks
1936 .iter()
1937 .map(|block| block.reference())
1938 .collect::<BTreeSet<_>>();
1939
1940 for (i, block) in blocks.iter().enumerate() {
1942 let latency = (i == 2).then_some(2 * FETCH_REQUEST_TIMEOUT);
1943 mock_client
1944 .stub_fetch_blocks(
1945 vec![block.clone()],
1946 AuthorityIndex::new_for_test(i as u32 + 1),
1947 latency,
1948 )
1949 .await;
1950 }
1951
1952 let start = tokio::time::Instant::now();
1954 let results = TestSynchronizer::fetch_blocks_from_peers(
1955 context.clone(),
1956 inflight_blocks.clone(),
1957 network_client.clone(),
1958 missing_blocks.clone(),
1959 dag_state.clone(),
1960 peers_pool.clone(),
1961 )
1962 .await;
1963
1964 assert_eq!(results.len(), 2);
1967 assert_eq!(start.elapsed(), FETCH_REQUEST_TIMEOUT);
1968 assert!(start.elapsed() < FETCH_FROM_PEERS_TIMEOUT);
1969 drop(results);
1970 assert_eq!(inflight_blocks.num_of_locked_blocks(), 0);
1971
1972 let _guards = (1..=2)
1974 .map(|i| {
1975 let peer = PeerId::Validator(AuthorityIndex::new_for_test(i));
1976 inflight_blocks
1977 .lock_blocks(missing_blocks.clone(), peer)
1978 .unwrap()
1979 })
1980 .collect::<Vec<_>>();
1981
1982 let start = tokio::time::Instant::now();
1983 let results = TestSynchronizer::fetch_blocks_from_peers(
1984 context,
1985 inflight_blocks.clone(),
1986 network_client,
1987 missing_blocks,
1988 dag_state,
1989 peers_pool,
1990 )
1991 .await;
1992
1993 assert!(results.is_empty());
1995 assert_eq!(start.elapsed(), Duration::ZERO);
1996 }
1997
1998 #[tokio::test(flavor = "current_thread", start_paused = true)]
1999 async fn synchronizer_periodic_task_when_commit_lagging_gets_disabled() {
2000 let (context, _) = Context::new_for_test(4);
2002 let context = Arc::new(context);
2003 let block_verifier = Arc::new(NoopBlockVerifier {});
2004 let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2005 let mock_client = Arc::new(MockNetworkClient::default());
2006 let store = Arc::new(MemStore::new());
2007 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2008 let transaction_vote_tracker =
2009 TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2010 let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2011 let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2012
2013 let sync_missing_block_round_threshold = context.parameters.commit_sync_batch_size;
2015 let stub_blocks = (sync_missing_block_round_threshold * 2
2016 ..sync_missing_block_round_threshold * 3)
2017 .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
2018 .collect::<Vec<_>>();
2019 let missing_blocks = stub_blocks
2020 .iter()
2021 .map(|block| block.reference())
2022 .collect::<BTreeSet<_>>();
2023 core_dispatcher
2024 .stub_missing_blocks(missing_blocks.clone())
2025 .await;
2026
2027 mock_client
2032 .stub_fetch_blocks(
2033 stub_blocks.clone(),
2034 AuthorityIndex::new_for_test(1),
2035 Some(FETCH_REQUEST_TIMEOUT),
2036 )
2037 .await;
2038 mock_client
2039 .stub_fetch_blocks(stub_blocks.clone(), AuthorityIndex::new_for_test(2), None)
2040 .await;
2041 let mut expected_blocks = stub_blocks
2042 .iter()
2043 .take(context.parameters.max_blocks_per_sync)
2044 .cloned()
2045 .collect::<Vec<_>>();
2046
2047 let round = context.parameters.commit_sync_batch_size * COMMIT_LAG_MULTIPLIER * 2;
2049 let commit_index: CommitIndex = round - 1;
2050 let blocks = (0..4)
2051 .map(|authority| {
2052 let commit_votes = vec![CommitVote::new(commit_index, CommitDigest::MIN)];
2053 let block = TestBlock::new(round, authority)
2054 .set_commit_votes(commit_votes)
2055 .build();
2056
2057 VerifiedBlock::new_for_test(block)
2058 })
2059 .collect::<Vec<_>>();
2060
2061 for block in blocks {
2064 commit_vote_monitor.observe_block(&block);
2065 }
2066
2067 let network_client = Arc::new(SynchronizerClient::new(
2069 context.clone(),
2070 Some(mock_client.clone()),
2071 Some(mock_client.clone()),
2072 ));
2073 let peers_pool = Arc::new(PeersPool::new(context.clone()));
2074 let _handle = Synchronizer::start(
2075 network_client,
2076 context.clone(),
2077 core_dispatcher.clone(),
2078 commit_vote_monitor.clone(),
2079 block_verifier,
2080 transaction_vote_tracker,
2081 round_tracker,
2082 dag_state.clone(),
2083 peers_pool.clone(),
2084 false,
2085 );
2086
2087 sleep(COMMIT_PROGRESS_TIMEOUT / 2).await;
2090
2091 let added_blocks = core_dispatcher.get_add_blocks().await;
2094 assert_eq!(added_blocks, vec![]);
2095
2096 {
2099 let mut d = dag_state.write();
2100 for index in 1..=commit_index {
2101 let commit =
2102 TrustedCommit::new_for_test(index, CommitDigest::MIN, 0, BlockRef::MIN, vec![]);
2103
2104 d.add_commit(commit);
2105 }
2106
2107 assert_eq!(
2108 d.last_commit_index(),
2109 commit_vote_monitor.quorum_commit_index()
2110 );
2111 }
2112
2113 core_dispatcher
2115 .stub_missing_blocks(missing_blocks.clone())
2116 .await;
2117
2118 sleep(2 * FETCH_REQUEST_TIMEOUT).await;
2119
2120 let mut added_blocks = core_dispatcher.get_add_blocks().await;
2122
2123 added_blocks.sort_by_key(|block| block.reference());
2124 expected_blocks.sort_by_key(|block| block.reference());
2125
2126 assert_eq!(added_blocks, expected_blocks);
2127 }
2128
2129 #[tokio::test(flavor = "current_thread", start_paused = true)]
2130 async fn synchronizer_periodic_sync_resumes_when_commit_sync_stalled() {
2131 let (context, _) = Context::new_for_test(4);
2133 let context = Arc::new(context);
2134 let block_verifier = Arc::new(NoopBlockVerifier {});
2135 let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2136 let mock_client = Arc::new(MockNetworkClient::default());
2137 let store = Arc::new(MemStore::new());
2138 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2139 let transaction_vote_tracker =
2140 TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2141 let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2142 let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2143
2144 let sync_missing_block_round_threshold = context.parameters.commit_sync_batch_size;
2146 let stub_blocks = (sync_missing_block_round_threshold * 2
2147 ..sync_missing_block_round_threshold * 3)
2148 .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
2149 .collect::<Vec<_>>();
2150 let missing_blocks = stub_blocks
2151 .iter()
2152 .map(|block| block.reference())
2153 .collect::<BTreeSet<_>>();
2154 core_dispatcher
2155 .stub_missing_blocks(missing_blocks.clone())
2156 .await;
2157
2158 mock_client
2161 .stub_fetch_blocks(
2162 stub_blocks.clone(),
2163 AuthorityIndex::new_for_test(1),
2164 Some(FETCH_REQUEST_TIMEOUT),
2165 )
2166 .await;
2167 mock_client
2168 .stub_fetch_blocks(stub_blocks.clone(), AuthorityIndex::new_for_test(2), None)
2169 .await;
2170 let mut expected_blocks = stub_blocks
2171 .iter()
2172 .take(context.parameters.max_blocks_per_sync)
2173 .cloned()
2174 .collect::<Vec<_>>();
2175
2176 let round = context.parameters.commit_sync_batch_size * COMMIT_LAG_MULTIPLIER * 2;
2178 let commit_index: CommitIndex = round - 1;
2179 let blocks = (0..4)
2180 .map(|authority| {
2181 let commit_votes = vec![CommitVote::new(commit_index, CommitDigest::MIN)];
2182 let block = TestBlock::new(round, authority)
2183 .set_commit_votes(commit_votes)
2184 .build();
2185 VerifiedBlock::new_for_test(block)
2186 })
2187 .collect::<Vec<_>>();
2188 for block in blocks {
2189 commit_vote_monitor.observe_block(&block);
2190 }
2191
2192 let network_client = Arc::new(SynchronizerClient::new(
2194 context.clone(),
2195 Some(mock_client.clone()),
2196 Some(mock_client.clone()),
2197 ));
2198 let peers_pool = Arc::new(PeersPool::new(context.clone()));
2199 let _handle = Synchronizer::start(
2200 network_client,
2201 context.clone(),
2202 core_dispatcher.clone(),
2203 commit_vote_monitor.clone(),
2204 block_verifier,
2205 transaction_vote_tracker,
2206 round_tracker,
2207 dag_state.clone(),
2208 peers_pool.clone(),
2209 false,
2210 );
2211
2212 sleep(COMMIT_PROGRESS_TIMEOUT - Duration::from_millis(100)).await;
2214 let added_blocks = core_dispatcher.get_add_blocks().await;
2215 assert_eq!(added_blocks, vec![]);
2216
2217 core_dispatcher
2219 .stub_missing_blocks(missing_blocks.clone())
2220 .await;
2221 mock_client
2224 .stub_fetch_blocks_for_key(
2225 vec![],
2226 expected_blocks.clone(),
2227 AuthorityIndex::new_for_test(1),
2228 Some(Duration::from_millis(500)),
2229 )
2230 .await;
2231
2232 sleep(Duration::from_millis(200) + FETCH_REQUEST_TIMEOUT).await;
2234
2235 let mut added_blocks = core_dispatcher.get_add_blocks().await;
2236 assert!(
2237 !added_blocks.is_empty(),
2238 "Expected periodic sync to resume after commit sync stall"
2239 );
2240 added_blocks.sort_by_key(|block| block.reference());
2241 expected_blocks.sort_by_key(|block| block.reference());
2242 assert_eq!(added_blocks, expected_blocks);
2243
2244 core_dispatcher.get_add_blocks().await;
2247 {
2248 let current = dag_state.read().last_commit_index();
2249 let mut d = dag_state.write();
2250 for index in (current + 1)..=(current + context.parameters.commit_sync_batch_size) {
2252 let commit =
2253 TrustedCommit::new_for_test(index, CommitDigest::MIN, 0, BlockRef::MIN, vec![]);
2254 d.add_commit(commit);
2255 }
2256 }
2257
2258 core_dispatcher
2260 .stub_missing_blocks(missing_blocks.clone())
2261 .await;
2262 mock_client
2263 .stub_fetch_blocks(
2264 stub_blocks
2265 .iter()
2266 .take(context.parameters.max_blocks_per_sync)
2267 .cloned()
2268 .collect::<Vec<_>>(),
2269 AuthorityIndex::new_for_test(2),
2270 None,
2271 )
2272 .await;
2273
2274 sleep(2 * FETCH_REQUEST_TIMEOUT).await;
2277 let added_blocks = core_dispatcher.get_add_blocks().await;
2278 assert_eq!(
2279 added_blocks,
2280 vec![],
2281 "Expected periodic sync to be skipped after stall resolved"
2282 );
2283 }
2284
2285 #[tokio::test(flavor = "current_thread", start_paused = true)]
2286 async fn synchronizer_fetch_own_last_block() {
2287 let (context, _) = Context::new_for_test(4);
2289 let context = Arc::new(context.with_parameters(Parameters {
2290 sync_last_known_own_block_timeout: Duration::from_millis(2_000),
2291 ..Default::default()
2292 }));
2293 let block_verifier = Arc::new(NoopBlockVerifier {});
2294 let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2295 let mock_client = Arc::new(MockNetworkClient::default());
2296 let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2297 let store = Arc::new(MemStore::new());
2298 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2299 let transaction_vote_tracker =
2300 TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2301 let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2302 let our_index = AuthorityIndex::new_for_test(0);
2303
2304 let mut expected_blocks = (9..=10)
2306 .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()))
2307 .collect::<Vec<_>>();
2308
2309 let block_1 = expected_blocks.pop().unwrap();
2312 mock_client
2313 .stub_fetch_latest_blocks(
2314 vec![block_1.clone()],
2315 AuthorityIndex::new_for_test(1),
2316 vec![our_index],
2317 None,
2318 )
2319 .await;
2320 mock_client
2321 .stub_fetch_latest_blocks(
2322 vec![block_1],
2323 AuthorityIndex::new_for_test(1),
2324 vec![our_index],
2325 None,
2326 )
2327 .await;
2328
2329 let block_2 = expected_blocks.pop().unwrap();
2331 mock_client
2332 .stub_fetch_latest_blocks(
2333 vec![block_2.clone()],
2334 AuthorityIndex::new_for_test(2),
2335 vec![our_index],
2336 Some(Duration::from_secs(10)),
2337 )
2338 .await;
2339 mock_client
2340 .stub_fetch_latest_blocks(
2341 vec![block_2],
2342 AuthorityIndex::new_for_test(2),
2343 vec![our_index],
2344 None,
2345 )
2346 .await;
2347
2348 mock_client
2350 .stub_fetch_latest_blocks(
2351 vec![],
2352 AuthorityIndex::new_for_test(3),
2353 vec![our_index],
2354 Some(Duration::from_secs(10)),
2355 )
2356 .await;
2357 mock_client
2358 .stub_fetch_latest_blocks(
2359 vec![],
2360 AuthorityIndex::new_for_test(3),
2361 vec![our_index],
2362 None,
2363 )
2364 .await;
2365
2366 let network_client = Arc::new(SynchronizerClient::new(
2368 context.clone(),
2369 Some(mock_client.clone()),
2370 Some(mock_client.clone()),
2371 ));
2372 let peers_pool = Arc::new(PeersPool::new(context.clone()));
2373 let handle = Synchronizer::start(
2374 network_client,
2375 context.clone(),
2376 core_dispatcher.clone(),
2377 commit_vote_monitor,
2378 block_verifier,
2379 transaction_vote_tracker,
2380 round_tracker,
2381 dag_state,
2382 peers_pool.clone(),
2383 true,
2384 );
2385
2386 sleep(context.parameters.sync_last_known_own_block_timeout * 2).await;
2388
2389 assert_eq!(
2391 core_dispatcher.get_last_own_proposed_round().await,
2392 vec![10]
2393 );
2394
2395 assert_eq!(mock_client.fetch_latest_blocks_pending_calls().await, 0);
2397
2398 assert_eq!(
2400 context
2401 .metrics
2402 .node_metrics
2403 .sync_last_known_own_block_retries
2404 .get(),
2405 1
2406 );
2407
2408 handle.stop().await;
2410 }
2411
2412 #[tokio::test]
2413 async fn test_process_fetched_blocks() {
2414 let (context, _) = Context::new_for_test(4);
2416 let context = Arc::new(context);
2417 let block_verifier = Arc::new(NoopBlockVerifier {});
2418 let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default());
2419 let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone()));
2420 let store = Arc::new(MemStore::new());
2421 let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
2422 let transaction_vote_tracker =
2423 TransactionVoteTracker::new(context.clone(), block_verifier.clone(), dag_state.clone());
2424 let round_tracker = Arc::new(RwLock::new(RoundTracker::new(context.clone(), vec![])));
2425 let (commands_sender, _commands_receiver) =
2426 monitored_mpsc::channel("consensus_synchronizer_commands", 1000);
2427
2428 let mut expected_blocks = vec![VerifiedBlock::new_for_test(TestBlock::new(60, 0).build())];
2432 expected_blocks.extend(
2433 (30..=60).map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 1).build())),
2434 );
2435 assert_eq!(
2436 expected_blocks.len(),
2437 context.parameters.max_blocks_per_sync
2438 );
2439
2440 let expected_serialized_blocks = expected_blocks
2441 .iter()
2442 .map(|b| b.serialized().clone())
2443 .collect::<Vec<_>>();
2444
2445 let expected_block_refs = expected_blocks
2446 .iter()
2447 .map(|b| b.reference())
2448 .collect::<BTreeSet<_>>();
2449
2450 let peer_index = AuthorityIndex::new_for_test(2);
2452 let peer = PeerId::Validator(peer_index);
2453
2454 let inflight_blocks_map = InflightBlocksMap::new();
2456 let blocks_guard = inflight_blocks_map
2457 .lock_blocks(expected_block_refs.clone(), peer.clone())
2458 .expect("Failed to lock blocks");
2459
2460 assert_eq!(
2461 inflight_blocks_map.num_of_locked_blocks(),
2462 expected_block_refs.len()
2463 );
2464
2465 let result = Synchronizer::<
2467 NoopBlockVerifier,
2468 MockCoreThreadDispatcher,
2469 MockNetworkClient,
2470 MockNetworkClient,
2471 >::process_fetched_blocks(
2472 expected_serialized_blocks,
2473 peer,
2474 blocks_guard, core_dispatcher.clone(),
2476 block_verifier,
2477 transaction_vote_tracker,
2478 commit_vote_monitor,
2479 context.clone(),
2480 commands_sender,
2481 round_tracker,
2482 "test",
2483 )
2484 .await;
2485
2486 assert!(result.is_ok());
2488
2489 let added_blocks = core_dispatcher.get_add_blocks().await;
2491 assert_eq!(
2492 added_blocks
2493 .iter()
2494 .map(|b| b.reference())
2495 .collect::<BTreeSet<_>>(),
2496 expected_block_refs,
2497 );
2498
2499 assert_eq!(inflight_blocks_map.num_of_locked_blocks(), 0);
2501 }
2502}