1use std::pin::Pin;
2use std::time::Duration;
3
4use futures::StreamExt;
5use prost::bytes::Bytes;
6use tonic::Request;
7use tonic::Status;
8use tonic::codegen::BoxStream;
9
10use super::super::Client;
11use super::super::Result;
12use super::adapter::ListScanDirection;
13use super::adapter::Position;
14use super::adapter::Progress;
15use super::adapter::RecoveryGap;
16use super::adapter::RpcFuture;
17use super::adapter::SubscriptionAdapter;
18use super::adapter::missing_cursor_position;
19use super::adapter::set_end;
20use super::adapter::set_start;
21use super::observability::LedgerStreamEvent;
22use super::observability::LedgerStreamObservability;
23use super::observability::LedgerStreamOperation;
24use super::observability::LedgerStreamStage;
25use super::retry::FailurePhase;
26use super::retry::RequestBounds;
27use super::retry::RetryState;
28use super::retry::backoff_delay;
29use super::types::LedgerStreamConfig;
30use crate::proto::sui::rpc::v2::QueryEnd;
31use crate::proto::sui::rpc::v2::QueryEndReason;
32use crate::proto::sui::rpc::v2::Watermark;
33
34#[derive(Clone, Copy)]
36pub(super) struct ExpectedEnd {
37 checkpoint_after_progress: bool,
38 cursor_after_progress: bool,
39 checkpoint_without_prior_progress: bool,
40 cursor_without_prior_progress: bool,
41}
42
43impl ExpectedEnd {
44 fn from_bounds(checkpoint: bool, cursor: bool) -> Self {
45 Self {
46 checkpoint_after_progress: checkpoint,
47 cursor_after_progress: cursor,
48 checkpoint_without_prior_progress: checkpoint,
49 cursor_without_prior_progress: cursor,
50 }
51 }
52
53 fn accepts(
54 self,
55 reason: QueryEndReason,
56 received_prior_frame: bool,
57 request_has_resume_cursor: bool,
58 ) -> bool {
59 let (checkpoint, cursor) = if received_prior_frame {
60 (self.checkpoint_after_progress, self.cursor_after_progress)
61 } else {
62 (
64 self.checkpoint_without_prior_progress,
65 self.cursor_without_prior_progress || request_has_resume_cursor,
66 )
67 };
68 (checkpoint && reason == QueryEndReason::CheckpointBound)
69 || (cursor && reason == QueryEndReason::CursorBound)
70 }
71}
72
73fn validate_query_end_item(reason: QueryEndReason, item_present: bool) -> Result<()> {
74 if reason == QueryEndReason::ItemLimit && !item_present {
75 Err(Status::data_loss("ItemLimit QueryEnd is missing its item"))
76 } else if reason != QueryEndReason::ItemLimit && item_present {
77 Err(Status::data_loss(
78 "non-ItemLimit QueryEnd unexpectedly contains an item",
79 ))
80 } else {
81 Ok(())
82 }
83}
84
85fn past_checkpoint_height(checkpoint_height: u64) -> Result<Position> {
87 checkpoint_height
88 .checked_add(1)
89 .map(Position::checkpoint_start)
90 .ok_or_else(|| {
91 Status::out_of_range("checkpoint height cannot be converted to an exclusive end bound")
92 })
93}
94
95pub(super) fn build_initial_list_request<A: SubscriptionAdapter>(
97 template: &A::ListRequest,
98 checkpoint_height: u64,
99) -> Result<(A::ListRequest, ExpectedEnd)> {
100 let mut request = template.clone();
101 set_end::<A>(
102 &mut request,
103 past_checkpoint_height(checkpoint_height)?,
104 None,
105 )?;
106 Ok((request, ExpectedEnd::from_bounds(true, false)))
107}
108
109pub(super) fn build_live_tip_baseline_list_request<A: SubscriptionAdapter>(
112 template: &A::ListRequest,
113 checkpoint_height: u64,
114) -> Result<(A::ListRequest, ExpectedEnd)> {
115 let mut request = template.clone();
116 set_start::<A>(
117 &mut request,
118 Position::checkpoint_start(checkpoint_height),
119 None,
120 );
121 set_end::<A>(
122 &mut request,
123 past_checkpoint_height(checkpoint_height)?,
124 None,
125 )?;
126 Ok((request, ExpectedEnd::from_bounds(true, false)))
127}
128
129pub(super) fn build_polling_list_request<A: SubscriptionAdapter>(
131 template: &A::ListRequest,
132 committed_progress: &Progress<A::Cursor>,
133 checkpoint_height: u64,
134) -> Result<(A::ListRequest, ExpectedEnd)> {
135 let mut request = template.clone();
136 set_start::<A>(
137 &mut request,
138 committed_progress.position,
139 A::cursor_bytes(&committed_progress.cursor),
140 );
141 set_end::<A>(
142 &mut request,
143 past_checkpoint_height(checkpoint_height)?,
144 None,
145 )?;
146 Ok((request, ExpectedEnd::from_bounds(true, false)))
147}
148
149pub(super) fn build_recovery_list_request<A: SubscriptionAdapter>(
151 template: A::ListRequest,
152 gap: &RecoveryGap<A::Cursor>,
153) -> Result<(A::ListRequest, ExpectedEnd)> {
154 let mut request = template;
155 set_start::<A>(&mut request, gap.start, A::cursor_bytes(&gap.after));
156 let before = A::cursor_bytes(&gap.before);
157 let ends_at_cursor = before.is_some();
158 set_end::<A>(&mut request, gap.end, before)?;
159 Ok((request, ExpectedEnd::from_bounds(true, ends_at_cursor)))
161}
162
163pub(super) fn determine_expected_end<A: SubscriptionAdapter>(
164 request: &A::ListRequest,
165 direction: ListScanDirection,
166) -> ExpectedEnd {
167 let options = A::options(request);
168 let checkpoint_without_prior_progress = A::start_checkpoint(request).is_some()
169 || A::end_checkpoint(request).is_some()
170 || direction == ListScanDirection::Descending;
171 let cursor_without_prior_progress =
172 options.is_some_and(|options| options.after.is_some() || options.before.is_some());
173 let (checkpoint_after_progress, cursor_after_progress) = match direction {
174 ListScanDirection::Ascending => (
175 A::end_checkpoint(request).is_some(),
176 options.is_some_and(|options| options.before.is_some()),
177 ),
178 ListScanDirection::Descending => {
179 (true, options.is_some_and(|options| options.after.is_some()))
180 }
181 };
182 ExpectedEnd {
183 checkpoint_after_progress,
184 cursor_after_progress,
185 checkpoint_without_prior_progress,
186 cursor_without_prior_progress,
187 }
188}
189
190enum RpcState<A: SubscriptionAdapter> {
191 Idle,
192 Dispatch {
193 future: RpcFuture<A::ListResponse>,
194 started_at: Option<tokio::time::Instant>,
195 },
196 Stream(BoxStream<A::ListResponse>),
197 Sleep(Pin<Box<tokio::time::Sleep>>),
198}
199
200pub(super) enum RpcEvent<R> {
202 Frame(R),
203 Status(Status, FailurePhase),
204 Eof,
205 Wake,
206}
207
208pub(super) enum ListAction<R> {
210 Frame {
211 response: R,
212 progress: Progress<Bytes>,
213 complete: bool,
214 },
215 Continue,
216 Terminal(Status),
217}
218
219#[derive(Clone, Copy)]
221pub(super) enum LedgerTipPolicy {
222 Complete,
224 WaitForExpectedBound,
226}
227
228pub(super) struct ListMachine<A: SubscriptionAdapter> {
229 client: Client,
230 payload: A::ListRequest,
231 expected_end: ExpectedEnd,
232 direction: ListScanDirection,
233 ledger_tip_policy: LedgerTipPolicy,
234 rpc_state: RpcState<A>,
235 request_resume_position: Option<Position>,
237 request_has_resume_cursor: bool,
239 latest_progress: Option<Progress<Bytes>>,
241 polling_attempt: u32,
242 received_frame_in_request: bool,
243 request_has_checkpoint_coverage: bool,
244 retry: RetryState,
245 pub(super) observability: LedgerStreamObservability,
246 stage: LedgerStreamStage,
247}
248
249impl<A: SubscriptionAdapter> ListMachine<A> {
250 pub(super) fn new(
251 client: Client,
252 payload: A::ListRequest,
253 expected_end: ExpectedEnd,
254 direction: ListScanDirection,
255 ledger_tip_policy: LedgerTipPolicy,
256 observability: LedgerStreamObservability,
257 stage: LedgerStreamStage,
258 ) -> Self {
259 let mut machine = Self {
260 client,
261 payload,
262 expected_end,
263 direction,
264 ledger_tip_policy,
265 rpc_state: RpcState::Idle,
266 request_resume_position: None,
267 request_has_resume_cursor: false,
268 latest_progress: None,
269 polling_attempt: 0,
270 received_frame_in_request: false,
271 request_has_checkpoint_coverage: false,
272 retry: RetryState::new(A::FAMILY, LedgerStreamOperation::List),
273 observability,
274 stage,
275 };
276 machine.start_dispatch();
277 machine
278 }
279
280 fn start_dispatch(&mut self) {
281 self.received_frame_in_request = false;
282 self.request_has_checkpoint_coverage = false;
283 self.request_has_resume_cursor = A::options(&self.payload)
284 .is_some_and(|options| self.direction.has_resume_cursor(options));
285 let request = Request::new(self.payload.clone());
286 self.rpc_state = RpcState::Dispatch {
287 future: A::dispatch_list(self.client.clone(), request),
288 started_at: None,
289 };
290 }
291
292 fn resume_from_latest_progress(&mut self) -> Result<()> {
294 if let Some(progress) = &self.latest_progress {
295 self.direction
296 .set_resume::<A>(&mut self.payload, progress)?;
297 self.request_resume_position = Some(progress.position);
298 }
299 Ok(())
300 }
301
302 fn sleep(&mut self, delay: Duration) {
303 self.rpc_state = RpcState::Sleep(Box::pin(tokio::time::sleep(delay)));
304 }
305
306 pub(super) async fn poll_rpc(&mut self) -> RpcEvent<A::ListResponse> {
309 loop {
310 match &mut self.rpc_state {
311 RpcState::Idle => {
312 return RpcEvent::Status(
313 Status::internal("idle RPC state polled"),
314 FailurePhase::Dispatch,
315 );
316 }
317 RpcState::Dispatch { future, started_at } => {
318 if started_at.is_none() {
320 *started_at = self.observability.start_timer();
321 }
322 let rpc_started_at = started_at.as_ref().cloned();
323 let result = future.await;
324 self.observability.emit_rpc_response(
325 rpc_started_at,
326 A::FAMILY,
327 LedgerStreamOperation::List,
328 self.stage,
329 &result,
330 );
331 match result {
332 Ok(stream) => self.rpc_state = RpcState::Stream(stream),
333 Err(status) => {
334 self.rpc_state = RpcState::Idle;
335 return RpcEvent::Status(status, FailurePhase::Dispatch);
336 }
337 }
338 }
339 RpcState::Stream(stream) => match stream.next().await {
340 Some(Ok(response)) => return RpcEvent::Frame(response),
341 Some(Err(status)) => {
342 self.rpc_state = RpcState::Idle;
343 return RpcEvent::Status(status, FailurePhase::Body);
344 }
345 None => {
346 self.rpc_state = RpcState::Idle;
347 return RpcEvent::Eof;
348 }
349 },
350 RpcState::Sleep(sleep) => {
351 sleep.as_mut().await;
352 self.rpc_state = RpcState::Idle;
353 return RpcEvent::Wake;
354 }
355 }
356 }
357 }
358
359 pub(super) fn process_event(
361 &mut self,
362 event: RpcEvent<A::ListResponse>,
363 config: &LedgerStreamConfig,
364 ) -> ListAction<A::ListResponse> {
365 match event {
366 RpcEvent::Wake => {
367 self.start_dispatch();
368 ListAction::Continue
369 }
370 RpcEvent::Eof => ListAction::Terminal(Status::data_loss(
372 "List stream ended before its QueryEnd frame",
373 )),
374 RpcEvent::Status(status, phase) => match self.process_status(status, phase, config) {
375 Ok(()) => ListAction::Continue,
376 Err(status) => ListAction::Terminal(status),
377 },
378 RpcEvent::Frame(response) => {
379 let (item_present, watermark, end) = A::extract_metadata(&response);
380 match self.process_frame_metadata(item_present, watermark, end, config) {
381 Ok((progress, complete)) => ListAction::Frame {
382 response,
383 progress,
384 complete,
385 },
386 Err(status) => ListAction::Terminal(status),
387 }
388 }
389 }
390 }
391
392 fn process_status(
393 &mut self,
394 status: Status,
395 phase: FailurePhase,
396 config: &LedgerStreamConfig,
397 ) -> Result<()> {
398 self.resume_from_latest_progress()?;
399 let bounds = if self.request_resume_position.is_some() {
401 RequestBounds::ServerReported
402 } else {
403 RequestBounds::CallerSupplied
404 };
405 if let Some(delay) = self.retry.retry_delay(
406 &status,
407 phase,
408 bounds,
409 config,
410 &self.observability,
411 self.stage,
412 ) {
413 self.sleep(delay);
414 Ok(())
415 } else {
416 Err(status)
417 }
418 }
419
420 fn process_frame_metadata(
421 &mut self,
422 item_present: bool,
423 watermark: Option<&Watermark>,
424 end: Option<&QueryEnd>,
425 config: &LedgerStreamConfig,
426 ) -> Result<(Progress<Bytes>, bool)> {
427 let watermark =
428 watermark.ok_or_else(|| Status::data_loss("List frame is missing its watermark"))?;
429 let cursor = watermark
430 .cursor
431 .clone()
432 .ok_or_else(|| Status::data_loss("List watermark is missing its cursor"))?;
433 let end_reason = end
434 .map(|end| {
435 end.reason
436 .and_then(|reason| QueryEndReason::try_from(reason).ok())
437 .filter(|reason| *reason != QueryEndReason::Unknown)
438 .ok_or_else(|| {
439 Status::data_loss("List QueryEnd has a missing or unknown reason")
440 })
441 })
442 .transpose()?;
443
444 let checkpoint = watermark.checkpoint;
445 if checkpoint.is_none() && self.request_has_checkpoint_coverage {
446 return Err(Status::data_loss(
447 "List checkpoint coverage became unavailable",
448 ));
449 }
450 if checkpoint.is_some() {
451 self.request_has_checkpoint_coverage = true;
452 }
453
454 let position = watermark
455 .cursor_position
456 .as_ref()
457 .ok_or_else(|| missing_cursor_position("List"))?;
458 let mut frame_progress = Progress {
459 position: A::GRANULARITY.position(position),
460 cursor,
461 checkpoint,
462 };
463 let advanced = match &mut self.latest_progress {
464 Some(latest) => {
465 frame_progress.inherit_checkpoint_coverage(latest);
466 let (position_advanced, _) =
467 latest.advance(&frame_progress, self.direction, item_present, "List")?;
468 item_present || position_advanced
469 }
470 None => {
471 self.latest_progress = Some(frame_progress.clone());
472 true
473 }
474 };
475 let received_prior_frame_in_request = self.received_frame_in_request;
476 self.received_frame_in_request = true;
477 if advanced {
478 self.polling_attempt = 0;
479 self.retry.reset(&self.observability);
480 }
481
482 let complete = match end_reason {
483 None => false,
484 Some(reason) => {
485 validate_query_end_item(reason, item_present)?;
486 if reason == QueryEndReason::CheckpointBound {
487 A::validate_checkpoint_bound(&self.payload, self.direction, checkpoint)?;
488 }
489
490 match reason {
491 QueryEndReason::ItemLimit | QueryEndReason::ScanLimit => {
492 if reason == QueryEndReason::ScanLimit
494 && let Some(resumed_from) = self.request_resume_position
495 && !self.direction.beyond(frame_progress.position, resumed_from)
496 {
497 return Err(Status::data_loss(
498 "ScanLimit QueryEnd did not advance its cursor_position",
499 ));
500 }
501 self.resume_from_latest_progress()?;
502 self.start_dispatch();
503 false
504 }
505 QueryEndReason::CheckpointBound | QueryEndReason::CursorBound => {
506 if !self.expected_end.accepts(
507 reason,
508 received_prior_frame_in_request,
509 self.request_has_resume_cursor,
510 ) {
511 return Err(Status::data_loss("List ended at an unexpected bound"));
512 }
513 self.rpc_state = RpcState::Idle;
514 true
515 }
516 QueryEndReason::LedgerTip
518 if self.direction == ListScanDirection::Descending =>
519 {
520 return Err(Status::data_loss("descending List reached LedgerTip"));
521 }
522 QueryEndReason::LedgerTip
523 if matches!(self.ledger_tip_policy, LedgerTipPolicy::Complete) =>
524 {
525 self.rpc_state = RpcState::Idle;
526 true
527 }
528 QueryEndReason::LedgerTip => {
529 self.resume_from_latest_progress()?;
530 let delay = backoff_delay(config, self.polling_attempt);
531 self.polling_attempt = self.polling_attempt.saturating_add(1);
532 self.sleep(delay);
533 false
534 }
535 QueryEndReason::Unknown => {
536 return Err(Status::data_loss("List QueryEnd has an unknown reason"));
537 }
538 }
539 }
540 };
541
542 Ok((frame_progress, complete))
543 }
544}
545
546pub(super) struct ListDriver<A: SubscriptionAdapter> {
548 machine: Option<ListMachine<A>>,
549 config: LedgerStreamConfig,
550 observability: LedgerStreamObservability,
551 terminal: Option<Status>,
552 done: bool,
553}
554
555impl<A: SubscriptionAdapter> ListDriver<A> {
556 pub(super) fn new(client: Client, request: A::ListRequest, config: LedgerStreamConfig) -> Self {
557 let observability = LedgerStreamObservability::new(config.observer());
558 let payload = request;
559 let direction = ListScanDirection::from_request::<A>(&payload);
560 let terminal = direction.as_ref().err().cloned();
561 let machine = direction.ok().map(|direction| {
562 let expected_end = determine_expected_end::<A>(&payload, direction);
563 ListMachine::new(
564 client,
565 payload,
566 expected_end,
567 direction,
568 LedgerTipPolicy::Complete,
569 observability.clone(),
570 LedgerStreamStage::List,
571 )
572 });
573 Self {
574 machine,
575 config,
576 observability,
577 terminal,
578 done: false,
579 }
580 }
581
582 pub(super) async fn next(&mut self) -> Option<Result<A::ListResponse>> {
583 if self.done {
584 return None;
585 }
586 if let Some(status) = self.terminal.take() {
587 return Some(self.finish_terminal(status));
588 }
589
590 loop {
591 let machine = self.machine.as_mut()?;
592 let event = machine.poll_rpc().await;
593 match machine.process_event(event, &self.config) {
594 ListAction::Frame {
595 response, complete, ..
596 } => {
597 if complete {
598 self.done = true;
599 self.machine = None;
600 }
601 return Some(Ok(response));
602 }
603 ListAction::Continue => {}
604 ListAction::Terminal(status) => return Some(self.finish_terminal(status)),
605 }
606 }
607 }
608
609 fn finish_terminal(&mut self, status: Status) -> Result<A::ListResponse> {
610 self.observability
611 .emit(|| LedgerStreamEvent::TerminalError {
612 family: A::FAMILY,
613 status: status.clone(),
614 });
615 self.done = true;
616 self.machine = None;
617 Err(status)
618 }
619}