sui_rpc/client/ledger_streams/
retry.rs1use std::collections::hash_map::RandomState;
2use std::error::Error as _;
3use std::hash::BuildHasher;
4use std::time::Duration;
5
6use prost::Message as _;
7use tonic::Code;
8use tonic::Status;
9
10use crate::proto::google::rpc::ErrorInfo;
11use crate::proto::sui::rpc::v2::ErrorReason;
12
13use super::observability::LedgerStreamEvent;
14use super::observability::LedgerStreamFamily;
15use super::observability::LedgerStreamObservability;
16use super::observability::LedgerStreamOperation;
17use super::observability::LedgerStreamStage;
18use super::types::LedgerStreamConfig;
19
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
25pub(super) enum FailurePhase {
26 Dispatch,
28 Body,
30}
31
32#[derive(Clone, Copy, Debug, PartialEq, Eq)]
34pub(super) enum RequestBounds {
35 CallerSupplied,
37 ServerReported,
40}
41
42fn transient(status: &Status, phase: FailurePhase, bounds: RequestBounds) -> bool {
66 let dispatch_transient = matches!(
67 status.code(),
68 Code::Unavailable | Code::DeadlineExceeded | Code::ResourceExhausted | Code::Aborted
69 );
70 let reset_code = matches!(status.code(), Code::Cancelled | Code::Unknown);
71 let transport_reset = reset_code && (phase == FailurePhase::Body || status.source().is_some());
72 dispatch_transient || transport_reset || above_latest_available_is_transient(status, bounds)
73}
74
75fn above_latest_available_is_transient(status: &Status, bounds: RequestBounds) -> bool {
81 if status.code() != Code::OutOfRange {
82 return false;
83 }
84 let Ok(details) = crate::proto::google::rpc::Status::decode(status.details()) else {
85 return false;
86 };
87 details
88 .details
89 .iter()
90 .filter_map(|detail| detail.to_msg::<ErrorInfo>().ok())
91 .any(|error_info| {
92 error_info.domain == ErrorReason::DOMAIN
93 && error_info.reason == ErrorReason::AboveLatestAvailable.as_str_name()
94 && (bounds == RequestBounds::ServerReported
95 || error_info
96 .metadata
97 .get("bound")
98 .is_some_and(|bound| bound == "cursor"))
99 })
100}
101
102pub(super) struct RetryState {
103 family: LedgerStreamFamily,
104 operation: LedgerStreamOperation,
105 consecutive_errors: u32,
106 first_failure: Option<tokio::time::Instant>,
107 started_in: Option<LedgerStreamStage>,
108}
109
110impl RetryState {
111 pub(super) fn new(family: LedgerStreamFamily, operation: LedgerStreamOperation) -> Self {
112 Self {
113 family,
114 operation,
115 consecutive_errors: 0,
116 first_failure: None,
117 started_in: None,
118 }
119 }
120
121 pub(super) fn reset(&mut self, observability: &LedgerStreamObservability) {
122 if self.consecutive_errors == 0 {
123 return;
124 }
125 let consecutive_failures = self.consecutive_errors;
126 self.consecutive_errors = 0;
127 let first_failure = self.first_failure.take();
128 let started_in = self.started_in.take();
129 if let (Some(first_failure), Some(started_in)) = (first_failure, started_in) {
130 observability.emit(|| LedgerStreamEvent::RetryRecovered {
131 family: self.family,
132 operation: self.operation,
133 started_in,
134 consecutive_failures,
135 elapsed: first_failure.elapsed(),
136 });
137 }
138 }
139
140 pub(super) fn retry_delay(
144 &mut self,
145 status: &Status,
146 phase: FailurePhase,
147 bounds: RequestBounds,
148 config: &LedgerStreamConfig,
149 observability: &LedgerStreamObservability,
150 stage: LedgerStreamStage,
151 ) -> Option<Duration> {
152 if !transient(status, phase, bounds) {
153 return None;
154 }
155 if config
156 .max_consecutive_retries
157 .is_some_and(|max_retries| self.consecutive_errors >= max_retries)
158 {
159 return None;
160 }
161
162 let attempt = self.consecutive_errors;
163 self.consecutive_errors = self.consecutive_errors.saturating_add(1);
164 if attempt == 0 {
165 let first_failure = observability.start_timer();
166 self.started_in = first_failure.as_ref().map(|_| stage);
167 self.first_failure = first_failure;
168 }
169 let delay = backoff_delay(config, attempt);
170 let consecutive_failures = self.consecutive_errors;
171 observability.emit(|| LedgerStreamEvent::RetryScheduled {
172 family: self.family,
173 operation: self.operation,
174 stage,
175 status: status.clone(),
176 consecutive_failures,
177 delay,
178 });
179 Some(delay)
180 }
181}
182
183pub(super) fn backoff_delay(config: &LedgerStreamConfig, attempt: u32) -> Duration {
184 let multiplier = 2_u32.saturating_pow(attempt);
185 let exponential = config.base_retry_delay.saturating_mul(multiplier);
186 let capped = exponential.min(config.max_retry_delay);
187 capped.saturating_add(random_jitter(config.retry_jitter))
188}
189
190fn random_jitter(maximum: Duration) -> Duration {
191 if maximum.is_zero() {
192 return Duration::ZERO;
193 }
194 let maximum_nanos = u64::try_from(maximum.as_nanos()).unwrap_or(u64::MAX);
195 let sample = RandomState::new().hash_one(());
198 let jitter_nanos = if maximum_nanos == u64::MAX {
199 sample
200 } else {
201 sample % (maximum_nanos + 1)
202 };
203 Duration::from_nanos(jitter_nanos)
204}
205
206#[cfg(test)]
207mod tests {
208 use super::*;
209
210 const CALLER: RequestBounds = RequestBounds::CallerSupplied;
211 const SERVER: RequestBounds = RequestBounds::ServerReported;
212
213 #[test]
214 fn internal_is_terminal_in_both_phases() {
215 let status = Status::internal("client idle-body watchdog task panicked");
216 assert!(!transient(&status, FailurePhase::Body, SERVER));
217 assert!(!transient(&status, FailurePhase::Dispatch, SERVER));
218 }
219
220 #[test]
221 fn server_reported_stream_reset_codes_are_transient_only_mid_body() {
222 for status in [
223 Status::cancelled("server cancelled request"),
224 Status::unknown("server failed request"),
225 ] {
226 assert!(transient(&status, FailurePhase::Body, CALLER));
227 assert!(!transient(&status, FailurePhase::Dispatch, CALLER));
228 }
229 }
230
231 #[test]
232 fn transport_sourced_unknown_is_transient_during_dispatch() {
233 let status = Status::from_error(Box::new(std::io::Error::from(
234 std::io::ErrorKind::ConnectionReset,
235 )));
236 assert_eq!(status.code(), Code::Unknown);
237 assert!(status.source().is_some());
238 assert!(transient(&status, FailurePhase::Dispatch, CALLER));
239 }
240
241 #[test]
242 fn dispatch_set_is_transient_in_both_phases() {
243 let status = Status::unavailable("connection refused");
244 assert!(transient(&status, FailurePhase::Dispatch, CALLER));
245 assert!(transient(&status, FailurePhase::Body, CALLER));
246 }
247
248 fn out_of_range_with_reason(reason: ErrorReason, bound: Option<&str>) -> Status {
249 let mut error_info = ErrorInfo::default()
250 .with_domain(ErrorReason::DOMAIN)
251 .with_reason(reason.as_str_name());
252 if let Some(bound) = bound {
253 error_info
254 .metadata
255 .insert("bound".to_owned(), bound.to_owned());
256 }
257 let details = crate::proto::google::rpc::Status {
258 code: Code::OutOfRange.into(),
259 message: "requested data out of range".to_owned(),
260 details: vec![prost_types::Any::from_msg(&error_info).unwrap()],
261 };
262 Status::with_details(
263 Code::OutOfRange,
264 details.message.clone(),
265 details.encode_to_vec().into(),
266 )
267 }
268
269 fn above_latest_available(bound: &str) -> Status {
270 out_of_range_with_reason(ErrorReason::AboveLatestAvailable, Some(bound))
271 }
272
273 #[test]
274 fn cursor_above_latest_available_is_transient_in_both_phases() {
275 let status = above_latest_available("cursor");
276 for bounds in [CALLER, SERVER] {
277 assert!(transient(&status, FailurePhase::Dispatch, bounds));
278 assert!(transient(&status, FailurePhase::Body, bounds));
279 }
280 }
281
282 #[test]
283 fn caller_checkpoint_bound_above_latest_available_is_terminal() {
284 for bound in ["start_checkpoint", "end_checkpoint"] {
285 let status = above_latest_available(bound);
286 assert!(
287 !transient(&status, FailurePhase::Dispatch, CALLER),
288 "{bound}"
289 );
290 assert!(!transient(&status, FailurePhase::Body, CALLER), "{bound}");
291 }
292 }
293
294 #[test]
295 fn server_reported_checkpoint_bound_above_latest_available_is_transient() {
296 for bound in ["start_checkpoint", "end_checkpoint"] {
297 let status = above_latest_available(bound);
298 assert!(
299 transient(&status, FailurePhase::Dispatch, SERVER),
300 "{bound}"
301 );
302 assert!(transient(&status, FailurePhase::Body, SERVER), "{bound}");
303 }
304 }
305
306 #[test]
307 fn out_of_range_without_a_server_reported_bound_past_the_tip_is_terminal() {
308 let below_earliest_available =
309 out_of_range_with_reason(ErrorReason::BelowEarliestAvailable, None);
310 let missing_bound = out_of_range_with_reason(ErrorReason::AboveLatestAvailable, None);
311 let without_reason = Status::out_of_range("requested data above latest available");
312 for status in [
313 below_earliest_available.clone(),
314 missing_bound,
315 without_reason.clone(),
316 ] {
317 assert!(
318 !transient(&status, FailurePhase::Dispatch, CALLER),
319 "{status:?}"
320 );
321 assert!(
322 !transient(&status, FailurePhase::Body, CALLER),
323 "{status:?}"
324 );
325 }
326 for status in [below_earliest_available, without_reason] {
327 assert!(
328 !transient(&status, FailurePhase::Dispatch, SERVER),
329 "{status:?}"
330 );
331 assert!(
332 !transient(&status, FailurePhase::Body, SERVER),
333 "{status:?}"
334 );
335 }
336 }
337}