Skip to main content

sui_rpc/client/ledger_streams/
stream.rs

1use std::time::Duration;
2
3use futures::StreamExt;
4use prost::bytes::Bytes;
5use tonic::Request;
6use tonic::Status;
7use tonic::codegen::BoxStream;
8
9use super::super::Client;
10use super::super::Result;
11use super::adapter::LedgerFrame;
12use super::adapter::ListScanDirection;
13use super::adapter::Position;
14use super::adapter::Progress;
15use super::adapter::RecoveryGap;
16use super::adapter::SubscriptionAdapter;
17use super::adapter::set_start;
18use super::list::ExpectedEnd;
19use super::list::LedgerTipPolicy;
20use super::list::ListAction;
21use super::list::ListMachine;
22use super::list::build_initial_list_request;
23use super::list::build_live_tip_baseline_list_request;
24use super::list::build_polling_list_request;
25use super::list::build_recovery_list_request;
26use super::list::determine_expected_end;
27use super::observability::LedgerStreamEvent;
28use super::observability::LedgerStreamObservability;
29use super::observability::LedgerStreamOperation;
30use super::observability::LedgerStreamStage;
31use super::retry::FailurePhase;
32use super::retry::RequestBounds;
33use super::retry::RetryState;
34use super::subscription::BufferedSubscriptionDrain;
35use super::subscription::BufferedSubscriptionState;
36use super::subscription::GapReplay;
37use super::subscription::LiveSubscription;
38use super::types::Delivery;
39use super::types::LedgerStreamConfig;
40use crate::proto::sui::rpc::v2::GetServiceInfoRequest;
41
42enum Phase<A: SubscriptionAdapter> {
43    /// Lists one match after a bare resume cursor to learn its position, then resumes there.
44    ResolveCursor(Box<ListMachine<A>>),
45    /// Waits until the ledger reaches the start checkpoint, then lists history through its tip.
46    ReadInitialTip,
47    /// Lists through a tip, then follows the ledger with the stream's delivery.
48    // `Phase` moves by value; each replay allocates its boxed machine once.
49    Replay(Box<Replay<A>>),
50    Subscribing(SubscribingPhase<A>),
51    Polling(PollingPhase),
52    Terminal(Status),
53    Done,
54}
55
56pub(super) enum Start {
57    Tip,
58    Checkpoint(u64),
59    /// At `position`, which the opaque `cursor` also names.
60    After {
61        cursor: Bytes,
62        position: Position,
63    },
64    /// Strictly after the opaque `cursor`, at a position the stream learns by listing from it.
65    AfterCursor(Bytes),
66}
67
68/// Lists one match after a bare resume `cursor`, sent as `options.after`, to learn its position.
69fn cursor_resolution<A: SubscriptionAdapter>(
70    client: &Client,
71    list_template: &A::ListRequest,
72    cursor: Bytes,
73    observability: &LedgerStreamObservability,
74) -> Box<ListMachine<A>> {
75    let mut payload = list_template.clone();
76    let options = A::options_mut(&mut payload);
77    options.after = Some(cursor);
78    options.limit = Some(1);
79    let expected_end = determine_expected_end::<A>(&payload, ListScanDirection::Ascending);
80    Box::new(ListMachine::new(
81        client.clone(),
82        payload,
83        expected_end,
84        ListScanDirection::Ascending,
85        LedgerTipPolicy::WaitForExpectedBound,
86        observability.clone(),
87        LedgerStreamStage::InitialReplay,
88    ))
89}
90
91struct Replay<A: SubscriptionAdapter> {
92    list_machine: ListMachine<A>,
93    output: ReplayOutput,
94}
95
96#[derive(Clone, Copy)]
97enum ReplayOutput {
98    Suppress,
99    Yield,
100}
101
102enum SubscribingPhase<A: SubscriptionAdapter> {
103    Attempt,
104    RetryDelay(Duration),
105    ReadRecoveryTip,
106    GapReplay(Box<GapReplay<A>>),
107    DrainGapBuffer(BufferedSubscriptionDrain<A>),
108    Live(LiveSubscription<A>),
109}
110
111enum PollingPhase {
112    Sleep,
113    ReadTip,
114}
115
116enum Step<A: SubscriptionAdapter> {
117    Next(Phase<A>),
118    Yield(Phase<A>, A::Output),
119}
120
121fn terminal<A: SubscriptionAdapter>(status: Status) -> Step<A> {
122    Step::Next(Phase::Terminal(status))
123}
124
125/// Keys a List response by its frame's progress.
126fn list_frame<A: SubscriptionAdapter>(
127    response: A::ListResponse,
128    progress: Progress<Bytes>,
129) -> Result<LedgerFrame<A::Item, A::Cursor>> {
130    let Progress {
131        position,
132        cursor,
133        checkpoint,
134    } = progress;
135    LedgerFrame::new(
136        A::into_list_item(response),
137        Progress {
138            position,
139            cursor: A::list_cursor(cursor),
140            checkpoint,
141        },
142        A::GRANULARITY,
143    )
144}
145
146/// The List interval between `committed` progress and a new subscription's `first` frame, or
147/// `None` when the subscription starts at or before committed progress.
148fn recovery_gap<A: SubscriptionAdapter>(
149    committed: &Progress<A::Cursor>,
150    first: &LedgerFrame<A::Item, A::Cursor>,
151) -> Option<RecoveryGap<A::Cursor>> {
152    // An item frame's cursor names its item's key as `options.before`.
153    let end = first.leading_boundary();
154    (end > committed.position).then(|| RecoveryGap {
155        start: committed.position,
156        after: committed.cursor.clone(),
157        end,
158        before: first.progress.cursor.clone(),
159    })
160}
161
162enum SubscriptionAttempt<A: SubscriptionAdapter> {
163    Established {
164        stream: BoxStream<A::SubscribeResponse>,
165        first_frame: LedgerFrame<A::Item, A::Cursor>,
166    },
167    Failed {
168        status: Status,
169        failure_phase: FailurePhase,
170    },
171}
172
173async fn dispatch_subscription<A: SubscriptionAdapter>(
174    client: &Client,
175    observability: &LedgerStreamObservability,
176    request: Request<A::SubscribeRequest>,
177    observability_stage: LedgerStreamStage,
178) -> Result<BoxStream<A::SubscribeResponse>> {
179    let future = A::dispatch_subscribe(client.clone(), request);
180    let started_at = observability.start_timer();
181    let result = future.await;
182    observability.emit_rpc_response(
183        started_at,
184        A::FAMILY,
185        LedgerStreamOperation::Subscribe,
186        observability_stage,
187        &result,
188    );
189    result
190}
191
192pub(super) struct Driver<A: SubscriptionAdapter> {
193    client: Client,
194    list_template: A::ListRequest,
195    subscribe_payload: A::SubscribeRequest,
196    item_required: bool,
197    config: LedgerStreamConfig,
198    observability: LedgerStreamObservability,
199    service_info_retry: RetryState,
200    subscription_retry: RetryState,
201    delivery: Delivery,
202    phase: Option<Phase<A>>,
203    /// The position a stream started at a checkpoint begins at. Until progress is committed,
204    /// frames at or before it deliver nothing.
205    start_position: Option<Position>,
206    /// Every match before `position` has been delivered, or precedes the stream's start.
207    committed_progress: Option<Progress<A::Cursor>>,
208}
209
210impl<A: SubscriptionAdapter> Driver<A> {
211    pub(super) fn new_stream(
212        client: Client,
213        subscribe_payload: A::SubscribeRequest,
214        mut list_template: A::ListRequest,
215        start: Start,
216        delivery: Delivery,
217        config: LedgerStreamConfig,
218    ) -> Self {
219        let observability = LedgerStreamObservability::new(config.observer());
220        let item_required = A::item_required(&subscribe_payload);
221        let mut start_position = None;
222        let mut committed_progress = None;
223        let requested_phase = match start {
224            Start::Tip => match delivery {
225                Delivery::Subscribe => Phase::Subscribing(SubscribingPhase::Attempt),
226                Delivery::Poll => Phase::Polling(PollingPhase::ReadTip),
227            },
228            Start::Checkpoint(checkpoint) => {
229                let position = Position::checkpoint_start(checkpoint);
230                set_start::<A>(&mut list_template, position, None);
231                start_position = Some(position);
232                Phase::ReadInitialTip
233            }
234            Start::After { cursor, position } => {
235                let position = A::GRANULARITY.normalize(position);
236                set_start::<A>(&mut list_template, position, Some(cursor.clone()));
237                committed_progress = Some(Progress {
238                    position,
239                    cursor: A::list_cursor(cursor),
240                    checkpoint: None,
241                });
242                Phase::ReadInitialTip
243            }
244            Start::AfterCursor(cursor) => Phase::ResolveCursor(cursor_resolution(
245                &client,
246                &list_template,
247                cursor,
248                &observability,
249            )),
250        };
251        let phase = if config.ledger_tip_poll_interval == Duration::ZERO {
252            Phase::Terminal(Status::invalid_argument(
253                "ledger_tip_poll_interval must be greater than zero",
254            ))
255        } else {
256            requested_phase
257        };
258        Self {
259            client,
260            list_template,
261            subscribe_payload,
262            item_required,
263            config,
264            observability,
265            service_info_retry: RetryState::new(A::FAMILY, LedgerStreamOperation::GetServiceInfo),
266            subscription_retry: RetryState::new(A::FAMILY, LedgerStreamOperation::Subscribe),
267            delivery,
268            phase: Some(phase),
269            start_position,
270            committed_progress,
271        }
272    }
273
274    pub(super) async fn next(&mut self) -> Option<Result<A::Output>> {
275        loop {
276            let step = match self.phase.take().unwrap_or(Phase::Done) {
277                Phase::Done => {
278                    self.phase = Some(Phase::Done);
279                    return None;
280                }
281                Phase::Terminal(status) => {
282                    self.observability
283                        .emit(|| LedgerStreamEvent::TerminalError {
284                            family: A::FAMILY,
285                            status: status.clone(),
286                        });
287                    self.phase = Some(Phase::Done);
288                    return Some(Err(status));
289                }
290                Phase::ResolveCursor(resolution) => self.step_resolve_cursor(resolution).await,
291                Phase::ReadInitialTip => self.read_initial_tip().await,
292                Phase::Replay(replay) => self.step_replay(replay).await,
293                Phase::Subscribing(phase) => self.step_subscribing(phase).await,
294                Phase::Polling(phase) => self.step_polling(phase).await,
295            };
296            match step {
297                Step::Next(next) => self.phase = Some(next),
298                Step::Yield(next, output) => {
299                    self.phase = Some(next);
300                    return Some(Ok(output));
301                }
302            }
303        }
304    }
305
306    /// A replay phase over an internally built ascending request.
307    fn replay(
308        &self,
309        payload: A::ListRequest,
310        expected_end: ExpectedEnd,
311        stage: LedgerStreamStage,
312        output: ReplayOutput,
313    ) -> Phase<A> {
314        Phase::Replay(Box::new(Replay {
315            list_machine: self.list_machine(payload, expected_end, stage),
316            output,
317        }))
318    }
319
320    /// An ascending List machine for an internally built request, paged by `list_page_limit`.
321    fn list_machine(
322        &self,
323        mut payload: A::ListRequest,
324        expected_end: ExpectedEnd,
325        stage: LedgerStreamStage,
326    ) -> ListMachine<A> {
327        A::options_mut(&mut payload).limit = self.config.list_page_limit;
328        ListMachine::new(
329            self.client.clone(),
330            payload,
331            expected_end,
332            ListScanDirection::Ascending,
333            LedgerTipPolicy::WaitForExpectedBound,
334            self.observability.clone(),
335            stage,
336        )
337    }
338
339    /// Delivers `frame` unless it lies behind committed progress, then continues with `next`.
340    fn deliver(&mut self, next: Phase<A>, frame: LedgerFrame<A::Item, A::Cursor>) -> Step<A> {
341        match self.accept_frame(frame) {
342            Some(output) => Step::Yield(next, output),
343            None => Step::Next(next),
344        }
345    }
346
347    /// Resumes at the first frame of a bare cursor's List request: delivers that frame, then
348    /// starts the stream at its position.
349    async fn step_resolve_cursor(&mut self, mut list_machine: Box<ListMachine<A>>) -> Step<A> {
350        let rpc_event = list_machine.poll_rpc().await;
351        match list_machine.process_event(rpc_event, &self.config) {
352            ListAction::Frame {
353                response, progress, ..
354            } => {
355                let frame = match list_frame::<A>(response, progress) {
356                    Ok(frame) => frame,
357                    Err(status) => return terminal(status),
358                };
359                set_start::<A>(
360                    &mut self.list_template,
361                    frame.progress.position,
362                    A::cursor_bytes(&frame.progress.cursor),
363                );
364                self.deliver(Phase::ReadInitialTip, frame)
365            }
366            ListAction::Continue => Step::Next(Phase::ResolveCursor(list_machine)),
367            ListAction::Terminal(status) => terminal(status),
368        }
369    }
370
371    async fn read_initial_tip(&mut self) -> Step<A> {
372        match self
373            .read_service_checkpoint_height(LedgerStreamStage::InitialReplay)
374            .await
375        {
376            Err(status) => terminal(status),
377            // Wait for `start_checkpoint`; otherwise `[start_checkpoint, checkpoint_height + 1)`
378            // is invalid.
379            Ok(checkpoint_height)
380                if A::start_checkpoint(&self.list_template)
381                    .is_some_and(|start_checkpoint| checkpoint_height < start_checkpoint) =>
382            {
383                tokio::time::sleep(self.config.ledger_tip_poll_interval).await;
384                Step::Next(Phase::ReadInitialTip)
385            }
386            Ok(checkpoint_height) => {
387                match build_initial_list_request::<A>(&self.list_template, checkpoint_height) {
388                    Ok((payload, expected_end)) => Step::Next(self.replay(
389                        payload,
390                        expected_end,
391                        LedgerStreamStage::InitialReplay,
392                        ReplayOutput::Yield,
393                    )),
394                    Err(status) => terminal(status),
395                }
396            }
397        }
398    }
399
400    async fn step_replay(&mut self, mut replay: Box<Replay<A>>) -> Step<A> {
401        let rpc_event = replay.list_machine.poll_rpc().await;
402        match replay.list_machine.process_event(rpc_event, &self.config) {
403            ListAction::Frame {
404                response,
405                progress,
406                complete,
407            } => {
408                let frame = match list_frame::<A>(response, progress) {
409                    Ok(frame) => frame,
410                    Err(status) => return terminal(status),
411                };
412                let output = replay.output;
413                let next = if complete {
414                    self.finish_replay()
415                } else {
416                    Phase::Replay(replay)
417                };
418                match (self.accept_frame(frame), output) {
419                    (Some(frame_output), ReplayOutput::Yield) => Step::Yield(next, frame_output),
420                    _ => Step::Next(next),
421                }
422            }
423            ListAction::Continue => Step::Next(Phase::Replay(replay)),
424            ListAction::Terminal(status) => terminal(status),
425        }
426    }
427
428    /// Follows the ledger with the stream's delivery once a replay reaches its tip.
429    fn finish_replay(&self) -> Phase<A> {
430        match self.delivery {
431            Delivery::Subscribe => Phase::Subscribing(SubscribingPhase::Attempt),
432            Delivery::Poll => Phase::Polling(PollingPhase::Sleep),
433        }
434    }
435
436    /// The replay from committed progress through `checkpoint_height`, or `None` when committed
437    /// progress already passes it. With nothing committed, lists only the tip checkpoint, without
438    /// yielding its frames, to establish a resume point.
439    fn catch_up(
440        &self,
441        checkpoint_height: u64,
442        stage: LedgerStreamStage,
443    ) -> Result<Option<Phase<A>>> {
444        match &self.committed_progress {
445            None => {
446                let (payload, expected_end) = build_live_tip_baseline_list_request::<A>(
447                    &self.list_template,
448                    checkpoint_height,
449                )?;
450                Ok(Some(self.replay(
451                    payload,
452                    expected_end,
453                    stage,
454                    ReplayOutput::Suppress,
455                )))
456            }
457            // The position lies at or past the start of checkpoint `checkpoint_height + 1`.
458            Some(committed) if committed.position.checkpoint > checkpoint_height => Ok(None),
459            Some(committed) => {
460                let (payload, expected_end) = build_polling_list_request::<A>(
461                    &self.list_template,
462                    committed,
463                    checkpoint_height,
464                )?;
465                Ok(Some(self.replay(
466                    payload,
467                    expected_end,
468                    stage,
469                    ReplayOutput::Yield,
470                )))
471            }
472        }
473    }
474
475    async fn step_subscribing(&mut self, phase: SubscribingPhase<A>) -> Step<A> {
476        match phase {
477            SubscribingPhase::Attempt => {
478                let observability_stage = if self.committed_progress.is_some() {
479                    LedgerStreamStage::LiveSubscription
480                } else {
481                    LedgerStreamStage::LiveTipStartup
482                };
483                match self.subscription_attempt(observability_stage).await {
484                    Err(status) => terminal(status),
485                    Ok(SubscriptionAttempt::Failed {
486                        status,
487                        failure_phase,
488                    }) => self.schedule_connect_retry(status, failure_phase, observability_stage),
489                    Ok(SubscriptionAttempt::Established {
490                        stream,
491                        first_frame,
492                    }) => {
493                        self.subscription_retry.reset(&self.observability);
494                        let gap = self
495                            .committed_progress
496                            .as_ref()
497                            .and_then(|committed| recovery_gap::<A>(committed, &first_frame));
498                        match gap {
499                            Some(gap) => self.enter_gap_replay(gap, stream, first_frame),
500                            None => {
501                                let live = LiveSubscription {
502                                    stream,
503                                    furthest: first_frame.progress.clone(),
504                                };
505                                self.deliver(
506                                    Phase::Subscribing(SubscribingPhase::Live(live)),
507                                    first_frame,
508                                )
509                            }
510                        }
511                    }
512                }
513            }
514            SubscribingPhase::RetryDelay(delay) => {
515                tokio::time::sleep(delay).await;
516                Step::Next(Phase::Subscribing(SubscribingPhase::ReadRecoveryTip))
517            }
518            SubscribingPhase::ReadRecoveryTip => match self
519                .read_service_checkpoint_height(LedgerStreamStage::GapRecovery)
520                .await
521            {
522                Err(status) => terminal(status),
523                Ok(checkpoint_height) => {
524                    match self.catch_up(checkpoint_height, LedgerStreamStage::GapRecovery) {
525                        Err(status) => terminal(status),
526                        Ok(None) => Step::Next(Phase::Subscribing(SubscribingPhase::Attempt)),
527                        Ok(Some(replay)) => {
528                            self.observability
529                                .emit(|| LedgerStreamEvent::GapRecoveryStarted {
530                                    family: A::FAMILY,
531                                });
532                            Step::Next(replay)
533                        }
534                    }
535                }
536            },
537            SubscribingPhase::GapReplay(mut gap) => {
538                let rpc_event =
539                    if let BufferedSubscriptionState::Active(live) = &mut gap.subscription_state {
540                        tokio::select! {
541                            event = gap.list_machine.poll_rpc() => Some(event),
542                            result = live.stream.next() => {
543                                if gap.buffer_subscription_result(
544                                    result,
545                                    self.item_required,
546                                    &self.config,
547                                ) {
548                                    self.subscription_retry.reset(&self.observability);
549                                }
550                                None
551                            }
552                        }
553                    } else {
554                        Some(gap.list_machine.poll_rpc().await)
555                    };
556                let Some(rpc_event) = rpc_event else {
557                    return Step::Next(Phase::Subscribing(SubscribingPhase::GapReplay(gap)));
558                };
559                match gap.list_machine.process_event(rpc_event, &self.config) {
560                    ListAction::Frame {
561                        response,
562                        progress,
563                        complete,
564                    } => {
565                        let frame = match list_frame::<A>(response, progress) {
566                            Ok(frame) => frame,
567                            Err(status) => return terminal(status),
568                        };
569                        let next = if complete {
570                            SubscribingPhase::DrainGapBuffer(
571                                (*gap).into_buffered_subscription_drain(),
572                            )
573                        } else {
574                            SubscribingPhase::GapReplay(gap)
575                        };
576                        self.deliver(Phase::Subscribing(next), frame)
577                    }
578                    ListAction::Continue => {
579                        Step::Next(Phase::Subscribing(SubscribingPhase::GapReplay(gap)))
580                    }
581                    ListAction::Terminal(status) => terminal(status),
582                }
583            }
584            SubscribingPhase::DrainGapBuffer(mut delivery) => {
585                if let Some(frame) = delivery.buffered_subscription_frames.pop_front() {
586                    self.deliver(
587                        Phase::Subscribing(SubscribingPhase::DrainGapBuffer(delivery)),
588                        frame,
589                    )
590                } else {
591                    match delivery.subscription_state {
592                        BufferedSubscriptionState::Failed(status) => self.schedule_connect_retry(
593                            status,
594                            FailurePhase::Body,
595                            LedgerStreamStage::GapRecovery,
596                        ),
597                        BufferedSubscriptionState::DroppedAtBufferLimit => {
598                            Step::Next(Phase::Subscribing(SubscribingPhase::Attempt))
599                        }
600                        BufferedSubscriptionState::Active(live) => {
601                            Step::Next(Phase::Subscribing(SubscribingPhase::Live(live)))
602                        }
603                    }
604                }
605            }
606            SubscribingPhase::Live(mut live) => match live.stream.next().await {
607                Some(Ok(response)) => match A::parse_live(response, self.item_required) {
608                    Err(status) => terminal(status),
609                    Ok(frame) => match live.furthest.advance(
610                        &frame.progress,
611                        ListScanDirection::Ascending,
612                        frame.item.is_some(),
613                        "subscription",
614                    ) {
615                        Err(status) => terminal(status),
616                        Ok((position_advanced, coverage_advanced)) => {
617                            if position_advanced || coverage_advanced {
618                                self.subscription_retry.reset(&self.observability);
619                            }
620                            // A subscription that started behind committed progress repeats
621                            // frames until it passes that progress.
622                            self.deliver(Phase::Subscribing(SubscribingPhase::Live(live)), frame)
623                        }
624                    },
625                },
626                Some(Err(status)) => {
627                    self.emit_subscription_interruption(
628                        &status,
629                        LedgerStreamStage::LiveSubscription,
630                    );
631                    self.schedule_connect_retry(
632                        status,
633                        FailurePhase::Body,
634                        LedgerStreamStage::LiveSubscription,
635                    )
636                }
637                None => {
638                    let status = Status::unavailable("subscription stream ended unexpectedly");
639                    self.emit_subscription_interruption(
640                        &status,
641                        LedgerStreamStage::LiveSubscription,
642                    );
643                    self.schedule_connect_retry(
644                        status,
645                        FailurePhase::Body,
646                        LedgerStreamStage::LiveSubscription,
647                    )
648                }
649            },
650        }
651    }
652
653    async fn step_polling(&mut self, phase: PollingPhase) -> Step<A> {
654        match phase {
655            PollingPhase::Sleep => {
656                tokio::time::sleep(self.config.ledger_tip_poll_interval).await;
657                Step::Next(Phase::Polling(PollingPhase::ReadTip))
658            }
659            PollingPhase::ReadTip => {
660                let stage = if self.committed_progress.is_some() {
661                    LedgerStreamStage::PollingTail
662                } else {
663                    LedgerStreamStage::PollingBaseline
664                };
665                match self.read_service_checkpoint_height(stage).await {
666                    Err(status) => terminal(status),
667                    Ok(checkpoint_height) => match self.catch_up(checkpoint_height, stage) {
668                        Err(status) => terminal(status),
669                        Ok(None) => Step::Next(Phase::Polling(PollingPhase::Sleep)),
670                        Ok(Some(replay)) => Step::Next(replay),
671                    },
672                }
673            }
674        }
675    }
676
677    async fn read_service_checkpoint_height(
678        &mut self,
679        observability_stage: LedgerStreamStage,
680    ) -> Result<u64> {
681        loop {
682            let request = Request::new(GetServiceInfoRequest::default());
683            let mut client = self.client.ledger_client();
684            let future = client.get_service_info(request);
685            let started_at = self.observability.start_timer();
686            let result = future.await;
687            self.observability.emit_rpc_response(
688                started_at,
689                A::FAMILY,
690                LedgerStreamOperation::GetServiceInfo,
691                observability_stage,
692                &result,
693            );
694            match result {
695                Ok(response) => {
696                    let checkpoint_height =
697                        response.into_inner().checkpoint_height.ok_or_else(|| {
698                            Status::data_loss(
699                                "GetServiceInfo response is missing checkpoint_height",
700                            )
701                        })?;
702                    self.service_info_retry.reset(&self.observability);
703                    return Ok(checkpoint_height);
704                }
705                Err(status) => {
706                    let Some(delay) = self.service_info_retry.retry_delay(
707                        &status,
708                        // Unary dispatch and body share this await. Remote statuses retain dispatch
709                        // classification; local tonic transport sources allow reset-code retry.
710                        FailurePhase::Dispatch,
711                        RequestBounds::CallerSupplied,
712                        &self.config,
713                        &self.observability,
714                        observability_stage,
715                    ) else {
716                        return Err(status);
717                    };
718                    tokio::time::sleep(delay).await;
719                }
720            }
721        }
722    }
723
724    fn emit_subscription_interruption(
725        &self,
726        status: &Status,
727        observability_stage: LedgerStreamStage,
728    ) {
729        self.observability
730            .emit(|| LedgerStreamEvent::SubscriptionStreamInterrupted {
731                family: A::FAMILY,
732                stage: observability_stage,
733                status: status.clone(),
734            });
735    }
736
737    fn enter_gap_replay(
738        &mut self,
739        gap: RecoveryGap<A::Cursor>,
740        new_subscription_stream: BoxStream<A::SubscribeResponse>,
741        new_subscription_frame: LedgerFrame<A::Item, A::Cursor>,
742    ) -> Step<A> {
743        // List fills the interval before this frame before buffered live data is delivered.
744        let (payload, expected_end) =
745            match build_recovery_list_request::<A>(self.list_template.clone(), &gap) {
746                Ok(request) => request,
747                Err(status) => return terminal(status),
748            };
749        self.observability
750            .emit(|| LedgerStreamEvent::GapRecoveryStarted { family: A::FAMILY });
751        let new_live_subscription = LiveSubscription {
752            stream: new_subscription_stream,
753            furthest: new_subscription_frame.progress.clone(),
754        };
755        let recovery_list_machine =
756            self.list_machine(payload, expected_end, LedgerStreamStage::GapRecovery);
757        Step::Next(Phase::Subscribing(SubscribingPhase::GapReplay(Box::new(
758            GapReplay::new(
759                recovery_list_machine,
760                new_live_subscription,
761                new_subscription_frame,
762                &self.config,
763            ),
764        ))))
765    }
766
767    async fn subscription_attempt(
768        &mut self,
769        observability_stage: LedgerStreamStage,
770    ) -> Result<SubscriptionAttempt<A>> {
771        let request = Request::new(self.subscribe_payload.clone());
772        let mut stream = match dispatch_subscription::<A>(
773            &self.client,
774            &self.observability,
775            request,
776            observability_stage,
777        )
778        .await
779        {
780            Ok(stream) => stream,
781            Err(status) => {
782                return Ok(SubscriptionAttempt::Failed {
783                    status,
784                    failure_phase: FailurePhase::Dispatch,
785                });
786            }
787        };
788        let first_frame = match stream.next().await {
789            Some(Ok(response)) => A::parse_live(response, self.item_required)?,
790            Some(Err(status)) => {
791                self.emit_subscription_interruption(&status, observability_stage);
792                return Ok(SubscriptionAttempt::Failed {
793                    status,
794                    failure_phase: FailurePhase::Body,
795                });
796            }
797            None => {
798                let status = Status::unavailable("subscription stream ended unexpectedly");
799                self.emit_subscription_interruption(&status, observability_stage);
800                return Ok(SubscriptionAttempt::Failed {
801                    status,
802                    failure_phase: FailurePhase::Body,
803                });
804            }
805        };
806        Ok(SubscriptionAttempt::Established {
807            stream,
808            first_frame,
809        })
810    }
811
812    fn schedule_connect_retry(
813        &mut self,
814        status: Status,
815        failure_phase: FailurePhase,
816        observability_stage: LedgerStreamStage,
817    ) -> Step<A> {
818        match self.subscription_retry.retry_delay(
819            &status,
820            failure_phase,
821            RequestBounds::CallerSupplied,
822            &self.config,
823            &self.observability,
824            observability_stage,
825        ) {
826            Some(delay) => Step::Next(Phase::Subscribing(SubscribingPhase::RetryDelay(delay))),
827            None => terminal(status),
828        }
829    }
830
831    /// Commits `frame` and returns its output when it lies past committed progress, or past the
832    /// stream's start while nothing is committed. An item frame's position is its item's key with
833    /// the finest index incremented, so the item lies before that point exactly when the frame's
834    /// position does not pass it.
835    fn accept_frame(&mut self, frame: LedgerFrame<A::Item, A::Cursor>) -> Option<A::Output> {
836        let LedgerFrame { item, progress } = frame;
837        let floor = self
838            .committed_progress
839            .as_ref()
840            .map(|committed| committed.position)
841            .or(self.start_position);
842        if floor.is_some_and(|floor| progress.position <= floor) {
843            return None;
844        }
845        self.committed_progress = Some(progress.clone());
846        Some(A::into_output(item.map(|item| item.payload), progress))
847    }
848}