Skip to main content

consensus_core/
subscriber.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{
5    sync::{Arc, Weak},
6    time::Duration,
7};
8
9use consensus_config::AuthorityIndex;
10use consensus_types::block::Round;
11use futures::StreamExt;
12use mysten_metrics::spawn_monitored_task;
13use parking_lot::{Mutex, RwLock};
14use tokio::{
15    task::JoinHandle,
16    time::{sleep, timeout},
17};
18use tracing::{debug, error, info};
19
20use crate::{
21    block::BlockAPI as _,
22    context::Context,
23    dag_state::DagState,
24    error::ConsensusError,
25    network::{SerializedBlockForm, ValidatorNetworkClient, ValidatorNetworkService},
26    task::{join_and_propagate_panic, reap_finished_task},
27};
28
29/// Bounds both establishing a subscription and waiting for the next block on it, so the
30/// subscription is abandoned and retried when either makes no progress for this long. A healthy
31/// peer proposes blocks multiple times per second, and (re)subscribing to a peer that has proposed
32/// before immediately yields at least its last proposed block, so timeouts and reconnections stay
33/// rare unless the peer is not proposing. This primarily guards against subscriptions that stop
34/// making progress without surfacing a transport error, e.g. a peer whose runtime stalls while its
35/// connections stay open. Kept well above the expected gap between proposals, because a peer that
36/// is reachable but not proposing gets resubscribed to on every timeout.
37const SUBSCRIPTION_TIMEOUT: Duration = Duration::from_secs(30);
38
39/// Subscriber manages the block stream subscriptions to other peers, taking care of retrying
40/// when subscription streams break. Blocks returned from the peer are sent to the authority
41/// service for processing.
42/// Currently subscription management for individual peer is not exposed, but it could become
43/// useful in future.
44pub(crate) struct Subscriber<C: ValidatorNetworkClient, S: ValidatorNetworkService> {
45    context: Arc<Context>,
46    network_client: Arc<C>,
47    authority_service: Arc<S>,
48    dag_state: Arc<RwLock<DagState>>,
49    subscriptions: Arc<Mutex<Box<[Option<JoinHandle<()>>]>>>,
50    // Retain replaced subscription tasks so stop() can await them and propagate panics.
51    retired_subscriptions: Arc<Mutex<Vec<JoinHandle<()>>>>,
52}
53
54impl<C: ValidatorNetworkClient, S: ValidatorNetworkService> Subscriber<C, S> {
55    pub(crate) fn new(
56        context: Arc<Context>,
57        network_client: Arc<C>,
58        authority_service: Arc<S>,
59        dag_state: Arc<RwLock<DagState>>,
60    ) -> Self {
61        let subscriptions = (0..context.committee.size())
62            .map(|_| None)
63            .collect::<Vec<_>>();
64        Self {
65            context,
66            network_client,
67            authority_service,
68            dag_state,
69            subscriptions: Arc::new(Mutex::new(subscriptions.into_boxed_slice())),
70            retired_subscriptions: Arc::new(Mutex::new(Vec::new())),
71        }
72    }
73
74    pub(crate) fn subscribe(&self, peer: AuthorityIndex) {
75        if peer == self.context.own_index {
76            error!("Attempt to subscribe to own validator {peer} is ignored!");
77            return;
78        }
79        let context = self.context.clone();
80        let network_client = self.network_client.clone();
81        // Subscriber already holds these resources strongly. Give subscription tasks weak
82        // references so they do not become additional owners during shutdown.
83        let authority_service = Arc::downgrade(&self.authority_service);
84        let dag_state = Arc::downgrade(&self.dag_state);
85
86        let mut subscriptions = self.subscriptions.lock();
87        self.unsubscribe_locked(peer, &mut subscriptions[peer.value()]);
88        subscriptions[peer.value()] = Some(spawn_monitored_task!(Self::subscription_loop(
89            context,
90            network_client,
91            authority_service,
92            dag_state,
93            peer,
94        )));
95    }
96
97    pub(crate) async fn stop(&self) {
98        {
99            let mut subscriptions = self.subscriptions.lock();
100            for (peer, _) in self.context.committee.authorities() {
101                self.unsubscribe_locked(peer, &mut subscriptions[peer.value()]);
102            }
103        }
104
105        // All retired subscriptions have already been aborted by unsubscribe_locked().
106        let subscriptions = std::mem::take(&mut *self.retired_subscriptions.lock());
107        for subscription in subscriptions {
108            join_and_propagate_panic(subscription).await;
109        }
110    }
111
112    fn unsubscribe_locked(&self, peer: AuthorityIndex, subscription: &mut Option<JoinHandle<()>>) {
113        let peer_hostname = &self.context.committee.authority(peer).hostname;
114        if let Some(subscription) = subscription.take() {
115            subscription.abort();
116            let mut retired_subscriptions = self.retired_subscriptions.lock();
117            // Reap retired subscriptions that have finished, so the list stays bounded under
118            // repeated resubscriptions.
119            retired_subscriptions.retain_mut(|task| !reap_finished_task(task));
120            retired_subscriptions.push(subscription);
121        }
122        // There is a race between shutting down the subscription task and clearing the metric here.
123        // TODO: fix the race when unsubscribe_locked() gets called outside of stop().
124        self.context
125            .metrics
126            .node_metrics
127            .subscribed_to
128            .with_label_values(&[peer_hostname])
129            .set(0);
130    }
131
132    async fn subscription_loop(
133        context: Arc<Context>,
134        network_client: Arc<C>,
135        authority_service: Weak<S>,
136        dag_state: Weak<RwLock<DagState>>,
137        peer: AuthorityIndex,
138    ) {
139        const IMMEDIATE_RETRIES: i64 = 3;
140        const MIN_TIMEOUT: Duration = Duration::from_millis(500);
141        // When not immediately retrying, limit retry delay between 100ms and 10s.
142        let mut backoff = mysten_common::backoff::ExponentialBackoff::new(
143            Duration::from_millis(100),
144            Duration::from_secs(10),
145        );
146
147        let peer_hostname = &context.committee.authority(peer).hostname;
148        let mut retries: i64 = 0;
149        'subscription: loop {
150            context
151                .metrics
152                .node_metrics
153                .subscribed_to
154                .with_label_values(&[peer_hostname])
155                .set(0);
156
157            let mut delay = Duration::ZERO;
158            if retries > IMMEDIATE_RETRIES {
159                delay = backoff.next().unwrap();
160                debug!(
161                    "Delaying retry {} of peer {} subscription, in {} seconds",
162                    retries,
163                    peer_hostname,
164                    delay.as_secs_f32(),
165                );
166                sleep(delay).await;
167            } else if retries > 0 {
168                // Retry immediately, but still yield to avoid monopolizing the thread.
169                tokio::task::yield_now().await;
170            }
171            retries += 1;
172
173            let last_accepted: Round = {
174                let Some(dag_state) = dag_state.upgrade() else {
175                    return;
176                };
177                let dag_state = dag_state.read();
178                let gc_round = dag_state.gc_round();
179                dag_state
180                    .get_last_block_for_authority(peer)
181                    .round()
182                    .max(gc_round)
183            };
184
185            // Use longer timeout when retry delay is long, to adapt to slow network.
186            let request_timeout = MIN_TIMEOUT.max(delay);
187            // `request_timeout` only bounds acquiring the channel, and the channel is usually
188            // cached, so establishing the stream can otherwise block indefinitely waiting for the
189            // peer's response headers, e.g. when the peer accepts connections but its runtime is
190            // stalled. Bound it here rather than with a gRPC deadline on the request, which would
191            // cap the lifetime of the whole subscription.
192            let subscribe = timeout(
193                SUBSCRIPTION_TIMEOUT,
194                network_client.subscribe_blocks(peer, last_accepted, request_timeout),
195            )
196            .await;
197            let mut blocks = match subscribe {
198                Ok(Ok(blocks)) => {
199                    debug!(
200                        "Subscribed to peer {} {} after {} attempts",
201                        peer, peer_hostname, retries
202                    );
203                    context
204                        .metrics
205                        .node_metrics
206                        .subscriber_connection_attempts
207                        .with_label_values(&[peer_hostname.as_str(), "success"])
208                        .inc();
209                    blocks
210                }
211                Ok(Err(e)) => {
212                    debug!(
213                        "Failed to subscribe to blocks from peer {} {}: {}",
214                        peer, peer_hostname, e
215                    );
216                    context
217                        .metrics
218                        .node_metrics
219                        .subscriber_connection_attempts
220                        .with_label_values(&[peer_hostname.as_str(), "failure"])
221                        .inc();
222                    continue 'subscription;
223                }
224                Err(_) => {
225                    debug!(
226                        "Timed out subscribing to blocks from peer {} {} after {:?}",
227                        peer, peer_hostname, SUBSCRIPTION_TIMEOUT
228                    );
229                    context
230                        .metrics
231                        .node_metrics
232                        .subscriber_connection_attempts
233                        .with_label_values(&[peer_hostname.as_str(), "failure"])
234                        .inc();
235                    continue 'subscription;
236                }
237            };
238
239            // Now can consider the subscription successful
240            context
241                .metrics
242                .node_metrics
243                .subscribed_to
244                .with_label_values(&[peer_hostname])
245                .set(1);
246
247            'stream: loop {
248                match timeout(SUBSCRIPTION_TIMEOUT, blocks.next()).await {
249                    Ok(Some(block)) => {
250                        context
251                            .metrics
252                            .node_metrics
253                            .subscribed_blocks
254                            .with_label_values(&[peer_hostname])
255                            .inc();
256                        let Some(authority_service) = authority_service.upgrade() else {
257                            return;
258                        };
259                        // Nothing emits the slim form yet; the decoder that turns it
260                        // back into a full block lands with the codec. Until then a
261                        // slim payload can only come from a misbehaving peer: drop it
262                        // here rather than hand downstream a form it cannot parse.
263                        if matches!(block.block, SerializedBlockForm::Slim(_)) {
264                            let reason: &'static str =
265                                (&ConsensusError::UnexpectedBlockForm).into();
266                            context
267                                .metrics
268                                .node_metrics
269                                .subscribe_stream_form_failures
270                                .with_label_values(&[peer_hostname, reason])
271                                .inc();
272                            retries = 0;
273                            backoff.reset();
274                            continue 'stream;
275                        }
276                        let result = authority_service.handle_send_block(peer, block).await;
277                        if let Err(e) = result {
278                            match e {
279                                ConsensusError::BlockRejected { block_ref, reason } => {
280                                    debug!(
281                                        "Failed to process block from peer {} {} for block {:?}: {}",
282                                        peer, peer_hostname, block_ref, reason
283                                    );
284                                }
285                                _ => {
286                                    info!(
287                                        "Invalid block received from peer {} {}: {}",
288                                        peer, peer_hostname, e
289                                    );
290                                }
291                            }
292                        }
293                        // Reset the retry counter and backoff when a block is received, so a peer
294                        // that recovers after flapping reconnects promptly instead of inheriting
295                        // the previously escalated delay.
296                        retries = 0;
297                        backoff.reset();
298                    }
299                    Ok(None) => {
300                        debug!(
301                            "Subscription to blocks from peer {} {} ended",
302                            peer, peer_hostname
303                        );
304                        retries += 1;
305                        break 'stream;
306                    }
307                    Err(_) => {
308                        info!(
309                            "Subscription to blocks from peer {} {} made no progress for {:?}",
310                            peer, peer_hostname, SUBSCRIPTION_TIMEOUT
311                        );
312                        retries += 1;
313                        break 'stream;
314                    }
315                }
316            }
317        }
318    }
319}
320
321#[cfg(test)]
322mod test {
323    use async_trait::async_trait;
324    use bytes::Bytes;
325    use consensus_types::block::BlockRef;
326    use futures::stream;
327
328    use super::*;
329    use crate::{
330        VerifiedBlock,
331        commit::CommitRange,
332        error::ConsensusResult,
333        network::{BlockStream, ExtendedSerializedBlock, test_network::TestService},
334        storage::mem_store::MemStore,
335    };
336
337    struct SubscriberTestClient {
338        // Records the `last_received` round passed to each subscribe_blocks() call.
339        subscribe_calls: Mutex<Vec<Round>>,
340        // Interval between blocks on the returned stream. None keeps the stream open
341        // forever without yielding any block.
342        block_interval: Option<Duration>,
343        // When true, subscribe_blocks() itself never returns.
344        hang_on_subscribe: bool,
345        emit_slim: bool,
346    }
347
348    impl SubscriberTestClient {
349        fn new() -> Self {
350            Self::new_with_block_interval(Duration::from_millis(1))
351        }
352
353        fn new_pending() -> Self {
354            Self {
355                subscribe_calls: Mutex::new(Vec::new()),
356                block_interval: None,
357                hang_on_subscribe: false,
358                emit_slim: false,
359            }
360        }
361
362        fn new_with_block_interval(interval: Duration) -> Self {
363            Self {
364                subscribe_calls: Mutex::new(Vec::new()),
365                block_interval: Some(interval),
366                hang_on_subscribe: false,
367                emit_slim: false,
368            }
369        }
370
371        fn new_hanging_subscribe() -> Self {
372            Self {
373                subscribe_calls: Mutex::new(Vec::new()),
374                block_interval: None,
375                hang_on_subscribe: true,
376                emit_slim: false,
377            }
378        }
379
380        fn subscribe_calls(&self) -> Vec<Round> {
381            self.subscribe_calls.lock().clone()
382        }
383    }
384
385    #[async_trait]
386    impl ValidatorNetworkClient for SubscriberTestClient {
387        #[cfg(not(msim))]
388        async fn send_block(
389            &self,
390            _peer: AuthorityIndex,
391            _block: &VerifiedBlock,
392            _timeout: Duration,
393        ) -> ConsensusResult<()> {
394            unimplemented!("Unimplemented")
395        }
396
397        async fn subscribe_blocks(
398            &self,
399            _peer: AuthorityIndex,
400            last_received: Round,
401            _timeout: Duration,
402        ) -> ConsensusResult<BlockStream> {
403            self.subscribe_calls.lock().push(last_received);
404            if self.hang_on_subscribe {
405                std::future::pending::<()>().await;
406            }
407            let Some(interval) = self.block_interval else {
408                return Ok(Box::pin(stream::pending()));
409            };
410            let emit_slim = self.emit_slim;
411            let block_stream = stream::unfold(0u8, move |i| async move {
412                sleep(interval).await;
413                // With emit_slim set, every other payload is the slim form, which the
414                // subscriber must drop without delivering.
415                let block = if emit_slim && i % 2 == 0 {
416                    ExtendedSerializedBlock {
417                        block: SerializedBlockForm::Slim(Bytes::from(vec![2u8; 8])),
418                        excluded_ancestors: vec![],
419                    }
420                } else {
421                    ExtendedSerializedBlock {
422                        block: SerializedBlockForm::Full(Bytes::from(vec![1u8; 8])),
423                        excluded_ancestors: vec![],
424                    }
425                };
426                Some((block, i.wrapping_add(1)))
427            })
428            .take(10);
429            Ok(Box::pin(block_stream))
430        }
431
432        async fn fetch_blocks(
433            &self,
434            _peer: AuthorityIndex,
435            _block_refs: Vec<BlockRef>,
436            _fetch_after_rounds: Vec<Round>,
437            _fetch_missing_ancestors: bool,
438            _timeout: Duration,
439        ) -> ConsensusResult<Vec<Bytes>> {
440            unimplemented!("Unimplemented")
441        }
442
443        async fn fetch_commits(
444            &self,
445            _peer: AuthorityIndex,
446            _commit_range: CommitRange,
447            _timeout: Duration,
448        ) -> ConsensusResult<(Vec<Bytes>, Vec<Bytes>)> {
449            unimplemented!("Unimplemented")
450        }
451
452        async fn fetch_latest_blocks(
453            &self,
454            _peer: AuthorityIndex,
455            _authorities: Vec<AuthorityIndex>,
456            _timeout: Duration,
457        ) -> ConsensusResult<Vec<Bytes>> {
458            unimplemented!("Unimplemented")
459        }
460
461        async fn get_latest_rounds(
462            &self,
463            _peer: AuthorityIndex,
464            _timeout: Duration,
465        ) -> ConsensusResult<(Vec<Round>, Vec<Round>)> {
466            unimplemented!("Unimplemented")
467        }
468    }
469
470    #[tokio::test(flavor = "current_thread", start_paused = true)]
471    async fn subscriber_retries() {
472        let (context, _keys) = Context::new_for_test(4);
473        let context = Arc::new(context);
474        let authority_service = Arc::new(Mutex::new(TestService::new()));
475        let network_client = Arc::new(SubscriberTestClient::new());
476        let store = Arc::new(MemStore::new());
477        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
478        let subscriber = Subscriber::new(
479            context.clone(),
480            network_client,
481            authority_service.clone(),
482            dag_state,
483        );
484
485        let peer = context.committee.to_authority_index(2).unwrap();
486        subscriber.subscribe(peer);
487
488        // Wait for enough blocks received.
489        for _ in 0..10 {
490            tokio::time::sleep(Duration::from_secs(1)).await;
491            let service = authority_service.lock();
492            if service.handle_send_block.len() >= 100 {
493                break;
494            }
495        }
496
497        // Even if the stream ends after 10 blocks, the subscriber should retry and get enough
498        // blocks eventually.
499        let service = authority_service.lock();
500        assert!(service.handle_send_block.len() >= 100);
501        for (p, block) in service.handle_send_block.iter() {
502            assert_eq!(*p, peer);
503            assert_eq!(
504                *block,
505                ExtendedSerializedBlock {
506                    block: SerializedBlockForm::Full(Bytes::from(vec![1u8; 8])),
507                    excluded_ancestors: vec![],
508                }
509            );
510        }
511    }
512
513    /// Slim payloads are dropped at the subscriber: nothing can decode them until the
514    /// codec lands, so they must never reach the authority service, while full payloads
515    /// on the same stream keep flowing.
516    #[tokio::test(flavor = "current_thread", start_paused = true)]
517    async fn subscriber_drops_slim_payloads_without_delivering() {
518        let (context, _keys) = Context::new_for_test(4);
519        let context = Arc::new(context);
520        let authority_service = Arc::new(Mutex::new(TestService::new()));
521        let network_client = Arc::new(SubscriberTestClient {
522            subscribe_calls: Mutex::new(Vec::new()),
523            block_interval: Some(Duration::from_millis(1)),
524            hang_on_subscribe: false,
525            emit_slim: true,
526        });
527        let store = Arc::new(MemStore::new());
528        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
529        let subscriber = Subscriber::new(
530            context.clone(),
531            network_client,
532            authority_service.clone(),
533            dag_state,
534        );
535
536        let peer = context.committee.to_authority_index(2).unwrap();
537        subscriber.subscribe(peer);
538
539        for _ in 0..10 {
540            tokio::time::sleep(Duration::from_secs(1)).await;
541            if authority_service.lock().handle_send_block.len() >= 20 {
542                break;
543            }
544        }
545
546        let service = authority_service.lock();
547        assert!(!service.handle_send_block.is_empty());
548        for (_, block) in service.handle_send_block.iter() {
549            assert!(
550                matches!(block.block, SerializedBlockForm::Full(_)),
551                "a slim payload must never be delivered"
552            );
553        }
554        assert!(
555            context
556                .metrics
557                .node_metrics
558                .subscribe_stream_form_failures
559                .with_label_values(&[
560                    context.committee.authority(peer).hostname.as_str(),
561                    "UnexpectedBlockForm",
562                ])
563                .get()
564                > 0,
565            "dropped slim payloads must be counted"
566        );
567    }
568
569    #[tokio::test(flavor = "current_thread", start_paused = true)]
570    async fn subscriber_reconnects_when_stream_makes_no_progress() {
571        let (context, _keys) = Context::new_for_test(4);
572        let context = Arc::new(context);
573        let authority_service = Arc::new(Mutex::new(TestService::new()));
574        let network_client = Arc::new(SubscriberTestClient::new_pending());
575        let store = Arc::new(MemStore::new());
576        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
577        let subscriber = Subscriber::new(
578            context.clone(),
579            network_client.clone(),
580            authority_service,
581            dag_state,
582        );
583
584        let peer = context.committee.to_authority_index(2).unwrap();
585        subscriber.subscribe(peer);
586
587        tokio::time::sleep(SUBSCRIPTION_TIMEOUT + Duration::from_millis(1)).await;
588
589        assert!(
590            network_client.subscribe_calls().len() >= 2,
591            "an idle subscription should be re-established"
592        );
593    }
594
595    #[tokio::test(flavor = "current_thread", start_paused = true)]
596    async fn subscriber_retries_when_subscribing_makes_no_progress() {
597        let (context, _keys) = Context::new_for_test(4);
598        let context = Arc::new(context);
599        let authority_service = Arc::new(Mutex::new(TestService::new()));
600        // The peer accepts the subscription but never responds, so the request to establish the
601        // stream never completes.
602        let network_client = Arc::new(SubscriberTestClient::new_hanging_subscribe());
603        let store = Arc::new(MemStore::new());
604        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
605        let subscriber = Subscriber::new(
606            context.clone(),
607            network_client.clone(),
608            authority_service,
609            dag_state,
610        );
611
612        let peer = context.committee.to_authority_index(2).unwrap();
613        subscriber.subscribe(peer);
614
615        tokio::time::sleep(SUBSCRIPTION_TIMEOUT + Duration::from_millis(1)).await;
616
617        assert!(
618            network_client.subscribe_calls().len() >= 2,
619            "subscribing should be abandoned and retried when the peer never responds"
620        );
621    }
622
623    #[tokio::test(flavor = "current_thread", start_paused = true)]
624    async fn subscriber_stays_subscribed_when_stream_progresses_within_idle_timeout() {
625        let (context, _keys) = Context::new_for_test(4);
626        let context = Arc::new(context);
627        let authority_service = Arc::new(Mutex::new(TestService::new()));
628        // Blocks arrive slower than from a healthy peer but within the idle timeout, so the
629        // timeout must reset on every received block and never tear down the subscription.
630        let network_client = Arc::new(SubscriberTestClient::new_with_block_interval(
631            SUBSCRIPTION_TIMEOUT - Duration::from_secs(1),
632        ));
633        let store = Arc::new(MemStore::new());
634        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
635        let subscriber = Subscriber::new(
636            context.clone(),
637            network_client.clone(),
638            authority_service.clone(),
639            dag_state,
640        );
641
642        let peer = context.committee.to_authority_index(2).unwrap();
643        subscriber.subscribe(peer);
644
645        tokio::time::sleep(SUBSCRIPTION_TIMEOUT * 4).await;
646
647        assert_eq!(
648            network_client.subscribe_calls().len(),
649            1,
650            "a stream that keeps delivering blocks within the idle timeout should not be re-established"
651        );
652        assert!(
653            !authority_service.lock().handle_send_block.is_empty(),
654            "blocks from the slow stream should have been processed"
655        );
656    }
657
658    // Regression test: `last_received` must be recomputed from DagState before each connection
659    // attempt. Previously it was captured once at subscribe() time and reused for every reconnect,
660    // causing already-accepted blocks to be re-streamed and re-verified.
661    #[tokio::test(flavor = "current_thread", start_paused = true)]
662    async fn subscriber_recomputes_resume_round_on_reconnect() {
663        use crate::block::TestBlock;
664
665        let (context, _keys) = Context::new_for_test(4);
666        let context = Arc::new(context);
667        let store = Arc::new(MemStore::new());
668        let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store)));
669        let network_client = Arc::new(SubscriberTestClient::new());
670        let authority_service = Arc::new(Mutex::new(TestService::new()));
671        let subscriber = Subscriber::new(
672            context.clone(),
673            network_client.clone(),
674            authority_service,
675            dag_state.clone(),
676        );
677
678        let peer = context.committee.to_authority_index(2).unwrap();
679        subscriber.subscribe(peer);
680
681        // Before any block from the peer is accepted, every reconnect resumes from genesis (0).
682        tokio::time::sleep(Duration::from_secs(3)).await;
683        {
684            let recorded = network_client.subscribe_calls();
685            assert!(
686                !recorded.is_empty() && recorded.iter().all(|&r| r == 0),
687                "before a block is accepted, every reconnect should resume from round 0: {recorded:?}"
688            );
689        }
690
691        // Advance the locally accepted round for the peer.
692        const RESUME_ROUND: Round = 10;
693        dag_state.write().accept_block(VerifiedBlock::new_for_test(
694            TestBlock::new(RESUME_ROUND, peer.value() as u32).build(),
695        ));
696
697        // After the block is accepted, reconnects must resume from the advanced round. With the
698        // bug, `last_received` would stay at 0 forever.
699        let mut observed_resume = false;
700        for _ in 0..10 {
701            tokio::time::sleep(Duration::from_secs(1)).await;
702            if network_client.subscribe_calls().last() == Some(&RESUME_ROUND) {
703                observed_resume = true;
704                break;
705            }
706        }
707        assert!(
708            observed_resume,
709            "after accepting a block at round {RESUME_ROUND}, the subscriber should resume from it; \
710             recorded resume rounds: {:?}",
711            network_client.subscribe_calls()
712        );
713    }
714}