Skip to main content

sui_rpc/client/ledger_streams/
list.rs

1use std::pin::Pin;
2use std::time::Duration;
3
4use futures::StreamExt;
5use prost::bytes::Bytes;
6use tonic::Request;
7use tonic::Status;
8use tonic::codegen::BoxStream;
9
10use super::super::Client;
11use super::super::Result;
12use super::adapter::ListScanDirection;
13use super::adapter::Position;
14use super::adapter::Progress;
15use super::adapter::RecoveryGap;
16use super::adapter::RpcFuture;
17use super::adapter::SubscriptionAdapter;
18use super::adapter::missing_cursor_position;
19use super::adapter::set_end;
20use super::adapter::set_start;
21use super::observability::LedgerStreamEvent;
22use super::observability::LedgerStreamObservability;
23use super::observability::LedgerStreamOperation;
24use super::observability::LedgerStreamStage;
25use super::retry::FailurePhase;
26use super::retry::RequestBounds;
27use super::retry::RetryState;
28use super::retry::backoff_delay;
29use super::types::LedgerStreamConfig;
30use crate::proto::sui::rpc::v2::QueryEnd;
31use crate::proto::sui::rpc::v2::QueryEndReason;
32use crate::proto::sui::rpc::v2::Watermark;
33
34/// Permitted `CheckpointBound` and `CursorBound` endings, with or without prior progress.
35#[derive(Clone, Copy)]
36pub(super) struct ExpectedEnd {
37    checkpoint_after_progress: bool,
38    cursor_after_progress: bool,
39    checkpoint_without_prior_progress: bool,
40    cursor_without_prior_progress: bool,
41}
42
43impl ExpectedEnd {
44    fn from_bounds(checkpoint: bool, cursor: bool) -> Self {
45        Self {
46            checkpoint_after_progress: checkpoint,
47            cursor_after_progress: cursor,
48            checkpoint_without_prior_progress: checkpoint,
49            cursor_without_prior_progress: cursor,
50        }
51    }
52
53    fn accepts(
54        self,
55        reason: QueryEndReason,
56        received_prior_frame: bool,
57        request_has_resume_cursor: bool,
58    ) -> bool {
59        let (checkpoint, cursor) = if received_prior_frame {
60            (self.checkpoint_after_progress, self.cursor_after_progress)
61        } else {
62            // A resume frontier may immediately return CursorBound when nothing follows it.
63            (
64                self.checkpoint_without_prior_progress,
65                self.cursor_without_prior_progress || request_has_resume_cursor,
66            )
67        };
68        (checkpoint && reason == QueryEndReason::CheckpointBound)
69            || (cursor && reason == QueryEndReason::CursorBound)
70    }
71}
72
73fn validate_query_end_item(reason: QueryEndReason, item_present: bool) -> Result<()> {
74    if reason == QueryEndReason::ItemLimit && !item_present {
75        Err(Status::data_loss("ItemLimit QueryEnd is missing its item"))
76    } else if reason != QueryEndReason::ItemLimit && item_present {
77        Err(Status::data_loss(
78            "non-ItemLimit QueryEnd unexpectedly contains an item",
79        ))
80    } else {
81        Ok(())
82    }
83}
84
85/// The boundary before the first checkpoint past `checkpoint_height`.
86fn past_checkpoint_height(checkpoint_height: u64) -> Result<Position> {
87    checkpoint_height
88        .checked_add(1)
89        .map(Position::checkpoint_start)
90        .ok_or_else(|| {
91            Status::out_of_range("checkpoint height cannot be converted to an exclusive end bound")
92        })
93}
94
95/// Lists from the template's start through `checkpoint_height`.
96pub(super) fn build_initial_list_request<A: SubscriptionAdapter>(
97    template: &A::ListRequest,
98    checkpoint_height: u64,
99) -> Result<(A::ListRequest, ExpectedEnd)> {
100    let mut request = template.clone();
101    set_end::<A>(
102        &mut request,
103        past_checkpoint_height(checkpoint_height)?,
104        None,
105    )?;
106    Ok((request, ExpectedEnd::from_bounds(true, false)))
107}
108
109/// Queries a single tip checkpoint to establish baseline watermark progress (used when starting in
110/// Poll mode, or recovering baseline progress after an initial Subscribe connection failure).
111pub(super) fn build_live_tip_baseline_list_request<A: SubscriptionAdapter>(
112    template: &A::ListRequest,
113    checkpoint_height: u64,
114) -> Result<(A::ListRequest, ExpectedEnd)> {
115    let mut request = template.clone();
116    set_start::<A>(
117        &mut request,
118        Position::checkpoint_start(checkpoint_height),
119        None,
120    );
121    set_end::<A>(
122        &mut request,
123        past_checkpoint_height(checkpoint_height)?,
124        None,
125    )?;
126    Ok((request, ExpectedEnd::from_bounds(true, false)))
127}
128
129/// Lists from committed progress through `checkpoint_height`.
130pub(super) fn build_polling_list_request<A: SubscriptionAdapter>(
131    template: &A::ListRequest,
132    committed_progress: &Progress<A::Cursor>,
133    checkpoint_height: u64,
134) -> Result<(A::ListRequest, ExpectedEnd)> {
135    let mut request = template.clone();
136    set_start::<A>(
137        &mut request,
138        committed_progress.position,
139        A::cursor_bytes(&committed_progress.cursor),
140    );
141    set_end::<A>(
142        &mut request,
143        past_checkpoint_height(checkpoint_height)?,
144        None,
145    )?;
146    Ok((request, ExpectedEnd::from_bounds(true, false)))
147}
148
149/// Lists exactly the interval `gap` names.
150pub(super) fn build_recovery_list_request<A: SubscriptionAdapter>(
151    template: A::ListRequest,
152    gap: &RecoveryGap<A::Cursor>,
153) -> Result<(A::ListRequest, ExpectedEnd)> {
154    let mut request = template;
155    set_start::<A>(&mut request, gap.start, A::cursor_bytes(&gap.after));
156    let before = A::cursor_bytes(&gap.before);
157    let ends_at_cursor = before.is_some();
158    set_end::<A>(&mut request, gap.end, before)?;
159    // The end fields and the `before` cursor name the same point, so either may end the scan.
160    Ok((request, ExpectedEnd::from_bounds(true, ends_at_cursor)))
161}
162
163pub(super) fn determine_expected_end<A: SubscriptionAdapter>(
164    request: &A::ListRequest,
165    direction: ListScanDirection,
166) -> ExpectedEnd {
167    let options = A::options(request);
168    let checkpoint_without_prior_progress = A::start_checkpoint(request).is_some()
169        || A::end_checkpoint(request).is_some()
170        || direction == ListScanDirection::Descending;
171    let cursor_without_prior_progress =
172        options.is_some_and(|options| options.after.is_some() || options.before.is_some());
173    let (checkpoint_after_progress, cursor_after_progress) = match direction {
174        ListScanDirection::Ascending => (
175            A::end_checkpoint(request).is_some(),
176            options.is_some_and(|options| options.before.is_some()),
177        ),
178        ListScanDirection::Descending => {
179            (true, options.is_some_and(|options| options.after.is_some()))
180        }
181    };
182    ExpectedEnd {
183        checkpoint_after_progress,
184        cursor_after_progress,
185        checkpoint_without_prior_progress,
186        cursor_without_prior_progress,
187    }
188}
189
190enum RpcState<A: SubscriptionAdapter> {
191    Idle,
192    Dispatch {
193        future: RpcFuture<A::ListResponse>,
194        started_at: Option<tokio::time::Instant>,
195    },
196    Stream(BoxStream<A::ListResponse>),
197    Sleep(Pin<Box<tokio::time::Sleep>>),
198}
199
200/// Raw I/O event produced by [`ListMachine::poll_rpc`].
201pub(super) enum RpcEvent<R> {
202    Frame(R),
203    Status(Status, FailurePhase),
204    Eof,
205    Wake,
206}
207
208/// Driver action returned by [`ListMachine::process_event`].
209pub(super) enum ListAction<R> {
210    Frame {
211        response: R,
212        progress: Progress<Bytes>,
213        complete: bool,
214    },
215    Continue,
216    Terminal(Status),
217}
218
219/// How an ascending scan handles a `LedgerTip` end.
220#[derive(Clone, Copy)]
221pub(super) enum LedgerTipPolicy {
222    /// Complete at the indexed tip.
223    Complete,
224    /// Retry from the served frontier until the fixed bound is indexed.
225    WaitForExpectedBound,
226}
227
228pub(super) struct ListMachine<A: SubscriptionAdapter> {
229    client: Client,
230    payload: A::ListRequest,
231    expected_end: ExpectedEnd,
232    direction: ListScanDirection,
233    ledger_tip_policy: LedgerTipPolicy,
234    rpc_state: RpcState<A>,
235    /// Position the active request resumed from, used to detect pagination that does not advance.
236    request_resume_position: Option<Position>,
237    /// Whether the active request carries an opaque cursor in its resume slot.
238    request_has_resume_cursor: bool,
239    /// Most recent frame progress, used to resume and to validate ordering.
240    latest_progress: Option<Progress<Bytes>>,
241    polling_attempt: u32,
242    received_frame_in_request: bool,
243    request_has_checkpoint_coverage: bool,
244    retry: RetryState,
245    pub(super) observability: LedgerStreamObservability,
246    stage: LedgerStreamStage,
247}
248
249impl<A: SubscriptionAdapter> ListMachine<A> {
250    pub(super) fn new(
251        client: Client,
252        payload: A::ListRequest,
253        expected_end: ExpectedEnd,
254        direction: ListScanDirection,
255        ledger_tip_policy: LedgerTipPolicy,
256        observability: LedgerStreamObservability,
257        stage: LedgerStreamStage,
258    ) -> Self {
259        let mut machine = Self {
260            client,
261            payload,
262            expected_end,
263            direction,
264            ledger_tip_policy,
265            rpc_state: RpcState::Idle,
266            request_resume_position: None,
267            request_has_resume_cursor: false,
268            latest_progress: None,
269            polling_attempt: 0,
270            received_frame_in_request: false,
271            request_has_checkpoint_coverage: false,
272            retry: RetryState::new(A::FAMILY, LedgerStreamOperation::List),
273            observability,
274            stage,
275        };
276        machine.start_dispatch();
277        machine
278    }
279
280    fn start_dispatch(&mut self) {
281        self.received_frame_in_request = false;
282        self.request_has_checkpoint_coverage = false;
283        self.request_has_resume_cursor = A::options(&self.payload)
284            .is_some_and(|options| self.direction.has_resume_cursor(options));
285        let request = Request::new(self.payload.clone());
286        self.rpc_state = RpcState::Dispatch {
287            future: A::dispatch_list(self.client.clone(), request),
288            started_at: None,
289        };
290    }
291
292    /// Points the next request at the most recent frame's resume point.
293    fn resume_from_latest_progress(&mut self) -> Result<()> {
294        if let Some(progress) = &self.latest_progress {
295            self.direction
296                .set_resume::<A>(&mut self.payload, progress)?;
297            self.request_resume_position = Some(progress.position);
298        }
299        Ok(())
300    }
301
302    fn sleep(&mut self, delay: Duration) {
303        self.rpc_state = RpcState::Sleep(Box::pin(tokio::time::sleep(delay)));
304    }
305
306    /// Dropping this future preserves `self.rpc_state`, so the next call resumes rather than
307    /// restarts in-flight work.
308    pub(super) async fn poll_rpc(&mut self) -> RpcEvent<A::ListResponse> {
309        loop {
310            match &mut self.rpc_state {
311                RpcState::Idle => {
312                    return RpcEvent::Status(
313                        Status::internal("idle RPC state polled"),
314                        FailurePhase::Dispatch,
315                    );
316                }
317                RpcState::Dispatch { future, started_at } => {
318                    // The timer starts on the first poll and survives cancellation by gap buffering.
319                    if started_at.is_none() {
320                        *started_at = self.observability.start_timer();
321                    }
322                    let rpc_started_at = started_at.as_ref().cloned();
323                    let result = future.await;
324                    self.observability.emit_rpc_response(
325                        rpc_started_at,
326                        A::FAMILY,
327                        LedgerStreamOperation::List,
328                        self.stage,
329                        &result,
330                    );
331                    match result {
332                        Ok(stream) => self.rpc_state = RpcState::Stream(stream),
333                        Err(status) => {
334                            self.rpc_state = RpcState::Idle;
335                            return RpcEvent::Status(status, FailurePhase::Dispatch);
336                        }
337                    }
338                }
339                RpcState::Stream(stream) => match stream.next().await {
340                    Some(Ok(response)) => return RpcEvent::Frame(response),
341                    Some(Err(status)) => {
342                        self.rpc_state = RpcState::Idle;
343                        return RpcEvent::Status(status, FailurePhase::Body);
344                    }
345                    None => {
346                        self.rpc_state = RpcState::Idle;
347                        return RpcEvent::Eof;
348                    }
349                },
350                RpcState::Sleep(sleep) => {
351                    sleep.as_mut().await;
352                    self.rpc_state = RpcState::Idle;
353                    return RpcEvent::Wake;
354                }
355            }
356        }
357    }
358
359    /// Advances validation and pagination state with one RPC event.
360    pub(super) fn process_event(
361        &mut self,
362        event: RpcEvent<A::ListResponse>,
363        config: &LedgerStreamConfig,
364    ) -> ListAction<A::ListResponse> {
365        match event {
366            RpcEvent::Wake => {
367                self.start_dispatch();
368                ListAction::Continue
369            }
370            // Successful List streams end with QueryEnd; bare EOF has no completion boundary.
371            RpcEvent::Eof => ListAction::Terminal(Status::data_loss(
372                "List stream ended before its QueryEnd frame",
373            )),
374            RpcEvent::Status(status, phase) => match self.process_status(status, phase, config) {
375                Ok(()) => ListAction::Continue,
376                Err(status) => ListAction::Terminal(status),
377            },
378            RpcEvent::Frame(response) => {
379                let (item_present, watermark, end) = A::extract_metadata(&response);
380                match self.process_frame_metadata(item_present, watermark, end, config) {
381                    Ok((progress, complete)) => ListAction::Frame {
382                        response,
383                        progress,
384                        complete,
385                    },
386                    Err(status) => ListAction::Terminal(status),
387                }
388            }
389        }
390    }
391
392    fn process_status(
393        &mut self,
394        status: Status,
395        phase: FailurePhase,
396        config: &LedgerStreamConfig,
397    ) -> Result<()> {
398        self.resume_from_latest_progress()?;
399        // A request that resumes from a frame has bounds a server reported.
400        let bounds = if self.request_resume_position.is_some() {
401            RequestBounds::ServerReported
402        } else {
403            RequestBounds::CallerSupplied
404        };
405        if let Some(delay) = self.retry.retry_delay(
406            &status,
407            phase,
408            bounds,
409            config,
410            &self.observability,
411            self.stage,
412        ) {
413            self.sleep(delay);
414            Ok(())
415        } else {
416            Err(status)
417        }
418    }
419
420    fn process_frame_metadata(
421        &mut self,
422        item_present: bool,
423        watermark: Option<&Watermark>,
424        end: Option<&QueryEnd>,
425        config: &LedgerStreamConfig,
426    ) -> Result<(Progress<Bytes>, bool)> {
427        let watermark =
428            watermark.ok_or_else(|| Status::data_loss("List frame is missing its watermark"))?;
429        let cursor = watermark
430            .cursor
431            .clone()
432            .ok_or_else(|| Status::data_loss("List watermark is missing its cursor"))?;
433        let end_reason = end
434            .map(|end| {
435                end.reason
436                    .and_then(|reason| QueryEndReason::try_from(reason).ok())
437                    .filter(|reason| *reason != QueryEndReason::Unknown)
438                    .ok_or_else(|| {
439                        Status::data_loss("List QueryEnd has a missing or unknown reason")
440                    })
441            })
442            .transpose()?;
443
444        let checkpoint = watermark.checkpoint;
445        if checkpoint.is_none() && self.request_has_checkpoint_coverage {
446            return Err(Status::data_loss(
447                "List checkpoint coverage became unavailable",
448            ));
449        }
450        if checkpoint.is_some() {
451            self.request_has_checkpoint_coverage = true;
452        }
453
454        let position = watermark
455            .cursor_position
456            .as_ref()
457            .ok_or_else(|| missing_cursor_position("List"))?;
458        let mut frame_progress = Progress {
459            position: A::GRANULARITY.position(position),
460            cursor,
461            checkpoint,
462        };
463        let advanced = match &mut self.latest_progress {
464            Some(latest) => {
465                frame_progress.inherit_checkpoint_coverage(latest);
466                let (position_advanced, _) =
467                    latest.advance(&frame_progress, self.direction, item_present, "List")?;
468                item_present || position_advanced
469            }
470            None => {
471                self.latest_progress = Some(frame_progress.clone());
472                true
473            }
474        };
475        let received_prior_frame_in_request = self.received_frame_in_request;
476        self.received_frame_in_request = true;
477        if advanced {
478            self.polling_attempt = 0;
479            self.retry.reset(&self.observability);
480        }
481
482        let complete = match end_reason {
483            None => false,
484            Some(reason) => {
485                validate_query_end_item(reason, item_present)?;
486                if reason == QueryEndReason::CheckpointBound {
487                    A::validate_checkpoint_bound(&self.payload, self.direction, checkpoint)?;
488                }
489
490                match reason {
491                    QueryEndReason::ItemLimit | QueryEndReason::ScanLimit => {
492                        // An ItemLimit item already advanced past the previous frame.
493                        if reason == QueryEndReason::ScanLimit
494                            && let Some(resumed_from) = self.request_resume_position
495                            && !self.direction.beyond(frame_progress.position, resumed_from)
496                        {
497                            return Err(Status::data_loss(
498                                "ScanLimit QueryEnd did not advance its cursor_position",
499                            ));
500                        }
501                        self.resume_from_latest_progress()?;
502                        self.start_dispatch();
503                        false
504                    }
505                    QueryEndReason::CheckpointBound | QueryEndReason::CursorBound => {
506                        if !self.expected_end.accepts(
507                            reason,
508                            received_prior_frame_in_request,
509                            self.request_has_resume_cursor,
510                        ) {
511                            return Err(Status::data_loss("List ended at an unexpected bound"));
512                        }
513                        self.rpc_state = RpcState::Idle;
514                        true
515                    }
516                    // Servers end only ascending scans at the ledger tip.
517                    QueryEndReason::LedgerTip
518                        if self.direction == ListScanDirection::Descending =>
519                    {
520                        return Err(Status::data_loss("descending List reached LedgerTip"));
521                    }
522                    QueryEndReason::LedgerTip
523                        if matches!(self.ledger_tip_policy, LedgerTipPolicy::Complete) =>
524                    {
525                        self.rpc_state = RpcState::Idle;
526                        true
527                    }
528                    QueryEndReason::LedgerTip => {
529                        self.resume_from_latest_progress()?;
530                        let delay = backoff_delay(config, self.polling_attempt);
531                        self.polling_attempt = self.polling_attempt.saturating_add(1);
532                        self.sleep(delay);
533                        false
534                    }
535                    QueryEndReason::Unknown => {
536                        return Err(Status::data_loss("List QueryEnd has an unknown reason"));
537                    }
538                }
539            }
540        };
541
542        Ok((frame_progress, complete))
543    }
544}
545
546/// Drives a finite List operation by resuming the request cursor and yielding whole responses.
547pub(super) struct ListDriver<A: SubscriptionAdapter> {
548    machine: Option<ListMachine<A>>,
549    config: LedgerStreamConfig,
550    observability: LedgerStreamObservability,
551    terminal: Option<Status>,
552    done: bool,
553}
554
555impl<A: SubscriptionAdapter> ListDriver<A> {
556    pub(super) fn new(client: Client, request: A::ListRequest, config: LedgerStreamConfig) -> Self {
557        let observability = LedgerStreamObservability::new(config.observer());
558        let payload = request;
559        let direction = ListScanDirection::from_request::<A>(&payload);
560        let terminal = direction.as_ref().err().cloned();
561        let machine = direction.ok().map(|direction| {
562            let expected_end = determine_expected_end::<A>(&payload, direction);
563            ListMachine::new(
564                client,
565                payload,
566                expected_end,
567                direction,
568                LedgerTipPolicy::Complete,
569                observability.clone(),
570                LedgerStreamStage::List,
571            )
572        });
573        Self {
574            machine,
575            config,
576            observability,
577            terminal,
578            done: false,
579        }
580    }
581
582    pub(super) async fn next(&mut self) -> Option<Result<A::ListResponse>> {
583        if self.done {
584            return None;
585        }
586        if let Some(status) = self.terminal.take() {
587            return Some(self.finish_terminal(status));
588        }
589
590        loop {
591            let machine = self.machine.as_mut()?;
592            let event = machine.poll_rpc().await;
593            match machine.process_event(event, &self.config) {
594                ListAction::Frame {
595                    response, complete, ..
596                } => {
597                    if complete {
598                        self.done = true;
599                        self.machine = None;
600                    }
601                    return Some(Ok(response));
602                }
603                ListAction::Continue => {}
604                ListAction::Terminal(status) => return Some(self.finish_terminal(status)),
605            }
606        }
607    }
608
609    fn finish_terminal(&mut self, status: Status) -> Result<A::ListResponse> {
610        self.observability
611            .emit(|| LedgerStreamEvent::TerminalError {
612                family: A::FAMILY,
613                status: status.clone(),
614            });
615        self.done = true;
616        self.machine = None;
617        Err(status)
618    }
619}