Skip to main content

sui_rpc/client/ledger_streams/
retry.rs

1use std::collections::hash_map::RandomState;
2use std::error::Error as _;
3use std::hash::BuildHasher;
4use std::time::Duration;
5
6use prost::Message as _;
7use tonic::Code;
8use tonic::Status;
9
10use crate::proto::google::rpc::ErrorInfo;
11use crate::proto::sui::rpc::v2::ErrorReason;
12
13use super::observability::LedgerStreamEvent;
14use super::observability::LedgerStreamFamily;
15use super::observability::LedgerStreamObservability;
16use super::observability::LedgerStreamOperation;
17use super::observability::LedgerStreamStage;
18use super::types::LedgerStreamConfig;
19
20/// Where an error was encountered during an RPC interaction.
21///
22/// Tonic surfaces errors differently depending on whether the RPC failed while initiating the call
23/// or while reading frames from an already-open response stream.
24#[derive(Clone, Copy, Debug, PartialEq, Eq)]
25pub(super) enum FailurePhase {
26    /// Failed while sending the initial RPC request.
27    Dispatch,
28    /// Failed while reading chunks from the response stream.
29    Body,
30}
31
32/// Where a request's checkpoint and cursor bounds came from.
33#[derive(Clone, Copy, Debug, PartialEq, Eq)]
34pub(super) enum RequestBounds {
35    /// The caller set the bounds, or the request has none.
36    CallerSupplied,
37    /// The request resumes from a position and cursor a server reported. Only descending scans
38    /// receive `ABOVE_LATEST_AVAILABLE` for them.
39    ServerReported,
40}
41
42/// Classifies failures eligible for automatic retry.
43///
44/// `Unavailable`, `DeadlineExceeded`, `ResourceExhausted`, and `Aborted` are always retried.
45///
46/// `Cancelled` and `Unknown` use two heuristics based on where the error was observed:
47/// - During `Dispatch` (initiating the RPC), server-sent `Cancelled` and `Unknown` are
48///   conservatively treated as terminal. We only retry if Tonic attached an underlying error source
49///   (`status.source().is_some()`), treating that as evidence of a lower-layer transport or connection
50///   failure.
51/// - During `Body` (streaming responses), stream interruptions (such as Envoy, ALB, or NAT gateway
52///   timeouts resetting an HTTP/2 stream) surface through Tonic as `Cancelled` or `Unknown`. Because
53///   `stream.next()` surfaces transport drops and server trailers identically, we treat body-phase
54///   `Cancelled` and `Unknown` as retryable based on the observation phase alone, allowing the stream
55///   to reconnect and resume from the latest cursor.
56///
57/// `OutOfRange` with the [`ErrorReason::AboveLatestAvailable`] reason is retried when the bound it
58/// names is servable once the server catches up (see [`above_latest_available_is_transient`]): the
59/// request reached a server, such as a lagging replica, that has not yet indexed a point another
60/// server reported. Every other `OutOfRange` is terminal, including data below the pruned floor
61/// ([`ErrorReason::BelowEarliestAvailable`]), a caller's checkpoint bound above the tip, which can
62/// name a checkpoint the chain has not reached, and an `OutOfRange` without a reason.
63///
64/// All other statuses are terminal.
65fn transient(status: &Status, phase: FailurePhase, bounds: RequestBounds) -> bool {
66    let dispatch_transient = matches!(
67        status.code(),
68        Code::Unavailable | Code::DeadlineExceeded | Code::ResourceExhausted | Code::Aborted
69    );
70    let reset_code = matches!(status.code(), Code::Cancelled | Code::Unknown);
71    let transport_reset = reset_code && (phase == FailurePhase::Body || status.source().is_some());
72    dispatch_transient || transport_reset || above_latest_available_is_transient(status, bounds)
73}
74
75/// Whether `status` is an `OutOfRange` whose status details carry an `ErrorInfo` with reason
76/// [`ErrorReason::AboveLatestAvailable`] for a bound some server reported: any bound of a request
77/// whose bounds are [`RequestBounds::ServerReported`], or the `bound` metadata `"cursor"`, since
78/// every cursor comes from a server. A server reported such a bound after indexing it, so the
79/// wait is bounded by how far replicas lag each other.
80fn above_latest_available_is_transient(status: &Status, bounds: RequestBounds) -> bool {
81    if status.code() != Code::OutOfRange {
82        return false;
83    }
84    let Ok(details) = crate::proto::google::rpc::Status::decode(status.details()) else {
85        return false;
86    };
87    details
88        .details
89        .iter()
90        .filter_map(|detail| detail.to_msg::<ErrorInfo>().ok())
91        .any(|error_info| {
92            error_info.domain == ErrorReason::DOMAIN
93                && error_info.reason == ErrorReason::AboveLatestAvailable.as_str_name()
94                && (bounds == RequestBounds::ServerReported
95                    || error_info
96                        .metadata
97                        .get("bound")
98                        .is_some_and(|bound| bound == "cursor"))
99        })
100}
101
102pub(super) struct RetryState {
103    family: LedgerStreamFamily,
104    operation: LedgerStreamOperation,
105    consecutive_errors: u32,
106    first_failure: Option<tokio::time::Instant>,
107    started_in: Option<LedgerStreamStage>,
108}
109
110impl RetryState {
111    pub(super) fn new(family: LedgerStreamFamily, operation: LedgerStreamOperation) -> Self {
112        Self {
113            family,
114            operation,
115            consecutive_errors: 0,
116            first_failure: None,
117            started_in: None,
118        }
119    }
120
121    pub(super) fn reset(&mut self, observability: &LedgerStreamObservability) {
122        if self.consecutive_errors == 0 {
123            return;
124        }
125        let consecutive_failures = self.consecutive_errors;
126        self.consecutive_errors = 0;
127        let first_failure = self.first_failure.take();
128        let started_in = self.started_in.take();
129        if let (Some(first_failure), Some(started_in)) = (first_failure, started_in) {
130            observability.emit(|| LedgerStreamEvent::RetryRecovered {
131                family: self.family,
132                operation: self.operation,
133                started_in,
134                consecutive_failures,
135                elapsed: first_failure.elapsed(),
136            });
137        }
138    }
139
140    /// Returns the delay before retrying `status`, or `None` when `status` is terminal: it is not
141    /// [`transient`] for a request with `bounds`, or this state has already retried
142    /// `config.max_consecutive_retries` times since the last [`Self::reset`].
143    pub(super) fn retry_delay(
144        &mut self,
145        status: &Status,
146        phase: FailurePhase,
147        bounds: RequestBounds,
148        config: &LedgerStreamConfig,
149        observability: &LedgerStreamObservability,
150        stage: LedgerStreamStage,
151    ) -> Option<Duration> {
152        if !transient(status, phase, bounds) {
153            return None;
154        }
155        if config
156            .max_consecutive_retries
157            .is_some_and(|max_retries| self.consecutive_errors >= max_retries)
158        {
159            return None;
160        }
161
162        let attempt = self.consecutive_errors;
163        self.consecutive_errors = self.consecutive_errors.saturating_add(1);
164        if attempt == 0 {
165            let first_failure = observability.start_timer();
166            self.started_in = first_failure.as_ref().map(|_| stage);
167            self.first_failure = first_failure;
168        }
169        let delay = backoff_delay(config, attempt);
170        let consecutive_failures = self.consecutive_errors;
171        observability.emit(|| LedgerStreamEvent::RetryScheduled {
172            family: self.family,
173            operation: self.operation,
174            stage,
175            status: status.clone(),
176            consecutive_failures,
177            delay,
178        });
179        Some(delay)
180    }
181}
182
183pub(super) fn backoff_delay(config: &LedgerStreamConfig, attempt: u32) -> Duration {
184    let multiplier = 2_u32.saturating_pow(attempt);
185    let exponential = config.base_retry_delay.saturating_mul(multiplier);
186    let capped = exponential.min(config.max_retry_delay);
187    capped.saturating_add(random_jitter(config.retry_jitter))
188}
189
190fn random_jitter(maximum: Duration) -> Duration {
191    if maximum.is_zero() {
192        return Duration::ZERO;
193    }
194    let maximum_nanos = u64::try_from(maximum.as_nanos()).unwrap_or(u64::MAX);
195    // Hashing with a fresh `RandomState`, randomly seeded per instance, yields an unpredictable
196    // sample without a `rand` dependency.
197    let sample = RandomState::new().hash_one(());
198    let jitter_nanos = if maximum_nanos == u64::MAX {
199        sample
200    } else {
201        sample % (maximum_nanos + 1)
202    };
203    Duration::from_nanos(jitter_nanos)
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209
210    const CALLER: RequestBounds = RequestBounds::CallerSupplied;
211    const SERVER: RequestBounds = RequestBounds::ServerReported;
212
213    #[test]
214    fn internal_is_terminal_in_both_phases() {
215        let status = Status::internal("client idle-body watchdog task panicked");
216        assert!(!transient(&status, FailurePhase::Body, SERVER));
217        assert!(!transient(&status, FailurePhase::Dispatch, SERVER));
218    }
219
220    #[test]
221    fn server_reported_stream_reset_codes_are_transient_only_mid_body() {
222        for status in [
223            Status::cancelled("server cancelled request"),
224            Status::unknown("server failed request"),
225        ] {
226            assert!(transient(&status, FailurePhase::Body, CALLER));
227            assert!(!transient(&status, FailurePhase::Dispatch, CALLER));
228        }
229    }
230
231    #[test]
232    fn transport_sourced_unknown_is_transient_during_dispatch() {
233        let status = Status::from_error(Box::new(std::io::Error::from(
234            std::io::ErrorKind::ConnectionReset,
235        )));
236        assert_eq!(status.code(), Code::Unknown);
237        assert!(status.source().is_some());
238        assert!(transient(&status, FailurePhase::Dispatch, CALLER));
239    }
240
241    #[test]
242    fn dispatch_set_is_transient_in_both_phases() {
243        let status = Status::unavailable("connection refused");
244        assert!(transient(&status, FailurePhase::Dispatch, CALLER));
245        assert!(transient(&status, FailurePhase::Body, CALLER));
246    }
247
248    fn out_of_range_with_reason(reason: ErrorReason, bound: Option<&str>) -> Status {
249        let mut error_info = ErrorInfo::default()
250            .with_domain(ErrorReason::DOMAIN)
251            .with_reason(reason.as_str_name());
252        if let Some(bound) = bound {
253            error_info
254                .metadata
255                .insert("bound".to_owned(), bound.to_owned());
256        }
257        let details = crate::proto::google::rpc::Status {
258            code: Code::OutOfRange.into(),
259            message: "requested data out of range".to_owned(),
260            details: vec![prost_types::Any::from_msg(&error_info).unwrap()],
261        };
262        Status::with_details(
263            Code::OutOfRange,
264            details.message.clone(),
265            details.encode_to_vec().into(),
266        )
267    }
268
269    fn above_latest_available(bound: &str) -> Status {
270        out_of_range_with_reason(ErrorReason::AboveLatestAvailable, Some(bound))
271    }
272
273    #[test]
274    fn cursor_above_latest_available_is_transient_in_both_phases() {
275        let status = above_latest_available("cursor");
276        for bounds in [CALLER, SERVER] {
277            assert!(transient(&status, FailurePhase::Dispatch, bounds));
278            assert!(transient(&status, FailurePhase::Body, bounds));
279        }
280    }
281
282    #[test]
283    fn caller_checkpoint_bound_above_latest_available_is_terminal() {
284        for bound in ["start_checkpoint", "end_checkpoint"] {
285            let status = above_latest_available(bound);
286            assert!(
287                !transient(&status, FailurePhase::Dispatch, CALLER),
288                "{bound}"
289            );
290            assert!(!transient(&status, FailurePhase::Body, CALLER), "{bound}");
291        }
292    }
293
294    #[test]
295    fn server_reported_checkpoint_bound_above_latest_available_is_transient() {
296        for bound in ["start_checkpoint", "end_checkpoint"] {
297            let status = above_latest_available(bound);
298            assert!(
299                transient(&status, FailurePhase::Dispatch, SERVER),
300                "{bound}"
301            );
302            assert!(transient(&status, FailurePhase::Body, SERVER), "{bound}");
303        }
304    }
305
306    #[test]
307    fn out_of_range_without_a_server_reported_bound_past_the_tip_is_terminal() {
308        let below_earliest_available =
309            out_of_range_with_reason(ErrorReason::BelowEarliestAvailable, None);
310        let missing_bound = out_of_range_with_reason(ErrorReason::AboveLatestAvailable, None);
311        let without_reason = Status::out_of_range("requested data above latest available");
312        for status in [
313            below_earliest_available.clone(),
314            missing_bound,
315            without_reason.clone(),
316        ] {
317            assert!(
318                !transient(&status, FailurePhase::Dispatch, CALLER),
319                "{status:?}"
320            );
321            assert!(
322                !transient(&status, FailurePhase::Body, CALLER),
323                "{status:?}"
324            );
325        }
326        for status in [below_earliest_available, without_reason] {
327            assert!(
328                !transient(&status, FailurePhase::Dispatch, SERVER),
329                "{status:?}"
330            );
331            assert!(
332                !transient(&status, FailurePhase::Body, SERVER),
333                "{status:?}"
334            );
335        }
336    }
337}