sui_rpc_api/service/health.rs
1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use axum::extract::{Query, State};
5use std::time::Duration;
6use std::time::SystemTime;
7use sui_types::storage::error::Result as StorageResult;
8
9use crate::Result;
10use crate::RpcService;
11
12/// The largest gap, in checkpoints, between the latest executed checkpoint and
13/// the highest checkpoint the live-object index has committed while still
14/// considered healthy.
15///
16/// The embedded indexer follows the tip asynchronously, so the live-object
17/// index always trails the executed tip by a little (roughly the indexer's
18/// snapshot window). A gap larger than this means the live index has fallen
19/// behind -- e.g. the indexer has stalled -- and cannot serve current
20/// live-object reads. The ledger-history cohort backfills separately after a
21/// restore and is excluded from this gap, so by default a node is healthy as
22/// soon as its live-object reads are caught up; callers that also need
23/// history caught up opt in with `max_history_lag_seconds` (see [`Threshold`]).
24const MAX_HEALTHY_INDEX_LAG: u64 = 60;
25
26impl RpcService {
27 /// Perform a simple health check on the service.
28 ///
29 /// `threshold_seconds` is the threshold, or delta, between the server's
30 /// system time and the timestamp in the most recently executed checkpoint
31 /// for which the server is considered to be healthy. If not provided, the
32 /// server's tip is not subject to a staleness check.
33 ///
34 /// Independent of the threshold, when indexing is enabled the server is
35 /// only considered healthy once its live-object indexes have caught up to
36 /// within `MAX_HEALTHY_INDEX_LAG` checkpoints of the latest executed
37 /// checkpoint.
38 ///
39 /// `max_history_lag_seconds`, when provided, additionally requires every
40 /// index pipeline -- including the ledger-history cohort, which backfills
41 /// separately after a restore -- to have caught up to within that many
42 /// seconds of chain time of the live-object index: the chain timestamps
43 /// of the checkpoints their watermarks have reached are compared, so the
44 /// bound means the same on every network and at every checkpoint rate. A
45 /// history index with no watermark yet is never healthy, and neither is a
46 /// node without rpc indexes: its reported tip is its executed checkpoint,
47 /// which can run ahead of what indexed replicas have reached.
48 ///
49 /// When indexing is disabled and `max_history_lag_seconds` is not
50 /// provided, the index checks are skipped.
51 pub fn health_check(
52 &self,
53 threshold_seconds: Option<u32>,
54 max_history_lag_seconds: Option<u32>,
55 ) -> Result<()> {
56 let latest = self.reader.inner().get_latest_checkpoint()?;
57
58 // If we have a provided threshold, check that it's close to the current
59 // time.
60 if let Some(threshold_seconds) = threshold_seconds {
61 let latest_chain_time = latest.timestamp();
62
63 let threshold = SystemTime::now() - Duration::from_secs(threshold_seconds as u64);
64
65 if latest_chain_time < threshold {
66 return Err(anyhow::anyhow!(
67 "The latest checkpoint timestamp is less than the provided threshold"
68 )
69 .into());
70 }
71 }
72
73 // When indexing is enabled, the node is only healthy once its
74 // live-object indexes (owned objects, types, balances) have kept up
75 // with the executed tip. Those indexes are restored to the tip and
76 // follow it, so a healthy node's live frontier trails execution by at
77 // most the indexer's snapshot window. The ledger-history cohort
78 // backfills independently after a restore and is only checked when the
79 // caller passes `max_history_lag_seconds`: gating on it by default
80 // would report a node unhealthy for the whole backfill even though its
81 // live-object reads are already caught up. The executed tip is read
82 // unbounded (rather than via `get_latest_checkpoint`, which is itself
83 // bounded to the live frontier) so a stalled live indexer, whose
84 // frontier falls behind ongoing execution, is still detected.
85 //
86 // The reported tip is bounded to the live frontier, so a replica
87 // whose history trails it advertises checkpoints its ledger-history
88 // reads cannot serve yet. Measure the history gap against that
89 // frontier rather than the executed tip, so the live index's own
90 // (separately bounded) lag behind execution is not counted twice. A
91 // node without an index surface (indexing disabled) reports its
92 // executed checkpoint as the tip, so it fails the history check
93 // whenever `max_history_lag_seconds` is set and skips the index checks
94 // otherwise.
95 let history_ok = if let Some(indexes) = self.reader.inner().indexes() {
96 let executed = self
97 .reader
98 .inner()
99 .get_highest_executed_checkpoint_seq_number()?;
100 let highest_live_indexed = indexes.get_highest_live_indexed_checkpoint_seq_number()?;
101
102 if !index_caught_up(executed, highest_live_indexed, MAX_HEALTHY_INDEX_LAG) {
103 return Err(anyhow::anyhow!(
104 "the live-object index is not caught up to within {MAX_HEALTHY_INDEX_LAG} \
105 checkpoints of the latest executed checkpoint"
106 )
107 .into());
108 }
109
110 history_caught_up(
111 max_history_lag_seconds,
112 Some(IndexTimestamps {
113 read_live_indexed_ms: &|| {
114 indexes.get_highest_live_indexed_checkpoint_timestamp_ms()
115 },
116 read_all_indexed_ms: &|| indexes.get_highest_indexed_checkpoint_timestamp_ms(),
117 }),
118 )?
119 } else {
120 history_caught_up(max_history_lag_seconds, None)?
121 };
122
123 if !history_ok {
124 return Err(anyhow::anyhow!(
125 "`max_history_lag_seconds` is set and the ledger-history indexes are absent or \
126 not caught up to within that many seconds of the live-object index"
127 )
128 .into());
129 }
130
131 Ok(())
132 }
133}
134
135/// Whether the highest live-indexed checkpoint is close enough to the executed
136/// tip to be considered healthy.
137///
138/// `highest_live_indexed` is `None` when the live-object index has not committed
139/// any checkpoint yet, which is never healthy. The live frontier never runs
140/// ahead of execution (it indexes executed checkpoints), but an equal frontier
141/// saturates to a zero lag rather than underflowing.
142fn index_caught_up(executed_seq: u64, highest_live_indexed: Option<u64>, max_lag: u64) -> bool {
143 match highest_live_indexed {
144 Some(indexed) => executed_seq.saturating_sub(indexed) <= max_lag,
145 None => false,
146 }
147}
148
149/// The rpc index surface as the history check sees it: the chain timestamps,
150/// in milliseconds, of the checkpoints its watermarks have reached.
151struct IndexTimestamps<'a> {
152 /// Reads the live-object frontier's timestamp; `None` before the live
153 /// cohort's first commit.
154 read_live_indexed_ms: &'a dyn Fn() -> StorageResult<Option<u64>>,
155 /// Reads the timestamp of the lowest watermark across all pipelines (live
156 /// and ledger-history); `None` while any pipeline has no watermark yet.
157 read_all_indexed_ms: &'a dyn Fn() -> StorageResult<Option<u64>>,
158}
159
160/// Whether every index pipeline (live and ledger-history) has caught up to
161/// within `max_lag_seconds` of chain time of the live-object frontier.
162///
163/// With no `max_lag_seconds` the history cohort is not checked: the result is
164/// `true` and nothing is read. Otherwise `indexes` is `None` for a node without
165/// rpc indexes, which is never healthy. The all-pipelines minimum includes the
166/// live pipelines, but it is read after the live frontier, so a live cohort
167/// that advanced in between can leave it slightly above; that counts as zero
168/// lag. A `None` minimum (in practice a history pipeline after a restore,
169/// before its seed) is never healthy, and neither is a missing live frontier.
170///
171/// A timestamp of 0 means the watermark was written without one, not a real
172/// chain time: the embedded restore anchors the live cohort with only its
173/// checkpoint set, so until the live indexer commits its first checkpoint both
174/// timestamps read 0 while history may still sit at its seed far below. A 0 on
175/// either side is therefore never healthy either.
176fn history_caught_up(
177 max_lag_seconds: Option<u32>,
178 indexes: Option<IndexTimestamps<'_>>,
179) -> StorageResult<bool> {
180 let Some(max_lag_seconds) = max_lag_seconds else {
181 return Ok(true);
182 };
183 let Some(indexes) = indexes else {
184 return Ok(false);
185 };
186 let max_lag_ms = u64::from(max_lag_seconds) * 1_000;
187 let live = (indexes.read_live_indexed_ms)()?;
188 let all = (indexes.read_all_indexed_ms)()?;
189 Ok(match (live, all) {
190 (Some(live), Some(all)) if live > 0 && all > 0 => live.saturating_sub(all) <= max_lag_ms,
191 _ => false,
192 })
193}
194
195#[derive(Debug, serde::Serialize, serde::Deserialize)]
196pub struct Threshold {
197 /// The threshold, or delta, between the server's system time and the timestamp in the most
198 /// recently executed checkpoint for which the server is considered to be healthy.
199 ///
200 /// If not provided, the server will be considered healthy if it can simply fetch the latest
201 /// checkpoint from its store and, when indexing is enabled, its live-object indexes have
202 /// caught up to it.
203 pub threshold_seconds: Option<u32>,
204
205 /// The largest gap, in seconds of chain time, between the live-object index and the slowest
206 /// index pipeline (in practice the ledger-history indexes, which backfill separately after a
207 /// restore) for which the server is considered to be healthy. The gap is measured between
208 /// the timestamps of the checkpoints their watermarks have reached. A history index with no
209 /// watermark yet is unhealthy, and so is a node without rpc indexes (indexing disabled),
210 /// whose reported tip is its executed checkpoint.
211 ///
212 /// If not provided, ledger-history progress and the presence of rpc indexes do not affect
213 /// health.
214 pub max_history_lag_seconds: Option<u32>,
215}
216
217pub async fn health(
218 Query(Threshold {
219 threshold_seconds,
220 max_history_lag_seconds,
221 }): Query<Threshold>,
222 State(state): State<RpcService>,
223) -> impl axum::response::IntoResponse {
224 match state.health_check(threshold_seconds, max_history_lag_seconds) {
225 Ok(()) => (axum::http::StatusCode::OK, "up"),
226 Err(_) => (axum::http::StatusCode::SERVICE_UNAVAILABLE, "down"),
227 }
228}
229
230#[cfg(test)]
231mod tests {
232 use super::*;
233 use sui_types::storage::error::Error as StorageError;
234
235 // The live-object index has not committed any checkpoint yet: never
236 // healthy.
237 #[test]
238 fn not_caught_up_when_unindexed() {
239 assert!(!index_caught_up(100, None, MAX_HEALTHY_INDEX_LAG));
240 assert!(!index_caught_up(0, None, MAX_HEALTHY_INDEX_LAG));
241 }
242
243 // The live frontier is within (or at) the allowed lag of the executed tip:
244 // healthy.
245 #[test]
246 fn caught_up_within_lag() {
247 assert!(index_caught_up(100, Some(100), 60)); // no lag
248 assert!(index_caught_up(100, Some(40), 60)); // exactly at the bound
249 assert!(index_caught_up(100, Some(41), 60)); // inside the bound
250 }
251
252 // The live frontier trails the executed tip by more than the allowed lag
253 // (e.g. the live indexer stalled while execution advanced): unhealthy. A
254 // lagging ledger-history backfill does not reach this path -- it is not part
255 // of the live frontier.
256 #[test]
257 fn not_caught_up_beyond_lag() {
258 assert!(!index_caught_up(100, Some(39), 60)); // one past the bound
259 assert!(!index_caught_up(1_000, Some(0), 60)); // live index far behind
260 }
261
262 // A live frontier level with the executed tip saturates to zero lag rather
263 // than underflowing.
264 #[test]
265 fn caught_up_when_index_at_tip() {
266 assert!(index_caught_up(100, Some(100), 60));
267 assert!(index_caught_up(100, Some(200), 60));
268 }
269
270 /// Evaluate the history check on a node with rpc indexes, with fixed
271 /// live-frontier and all-pipelines timestamps in milliseconds.
272 fn history_ok(max_lag_seconds: Option<u32>, live_ms: Option<u64>, all_ms: Option<u64>) -> bool {
273 let read_live = || Ok(live_ms);
274 let read_all = || Ok(all_ms);
275 let indexes = IndexTimestamps {
276 read_live_indexed_ms: &read_live,
277 read_all_indexed_ms: &read_all,
278 };
279 history_caught_up(max_lag_seconds, Some(indexes)).unwrap()
280 }
281
282 // Without `max_history_lag_seconds` the history cohort does not affect
283 // health and no index timestamp is read, so neither a mid-backfill history
284 // cohort nor a failing read can take the node out of rotation. A node
285 // without rpc indexes is likewise unaffected.
286 #[test]
287 fn history_ignored_when_unset() {
288 let read = || -> StorageResult<Option<u64>> {
289 panic!("index timestamp read with max_history_lag_seconds unset")
290 };
291 let indexes = IndexTimestamps {
292 read_live_indexed_ms: &read,
293 read_all_indexed_ms: &read,
294 };
295 assert!(history_caught_up(None, Some(indexes)).unwrap());
296 assert!(history_caught_up(None, None).unwrap());
297 }
298
299 // A node without rpc indexes reports its executed checkpoint as the tip,
300 // which can run ahead of what indexed replicas have reached: unhealthy
301 // whenever `max_history_lag_seconds` is set, however large.
302 #[test]
303 fn history_not_caught_up_without_indexes() {
304 assert!(!history_caught_up(Some(u32::MAX), None).unwrap());
305 assert!(!history_caught_up(Some(0), None).unwrap());
306 }
307
308 // Every pipeline is within (or at) the allowed lag of the live frontier:
309 // healthy. The bound is in seconds and the timestamps in milliseconds.
310 #[test]
311 fn history_caught_up_within_lag() {
312 assert!(history_ok(Some(10), Some(100_000), Some(100_000))); // no lag
313 assert!(history_ok(Some(10), Some(100_000), Some(90_000))); // exactly at the bound
314 assert!(history_ok(Some(10), Some(100_000), Some(95_000))); // inside the bound
315 }
316
317 // The all-pipelines minimum is read after the live frontier, so the live
318 // cohort can advance in between and leave it above the live timestamp:
319 // that saturates to zero lag rather than underflowing.
320 #[test]
321 fn history_caught_up_when_above_live() {
322 assert!(history_ok(Some(10), Some(100_000), Some(100_500)));
323 assert!(history_ok(Some(0), Some(100_000), Some(100_500)));
324 }
325
326 // History is backfilling far behind the live frontier (e.g. after a
327 // snapshot restore): unhealthy.
328 #[test]
329 fn history_not_caught_up_beyond_lag() {
330 assert!(!history_ok(Some(10), Some(100_000), Some(89_999))); // one ms past the bound
331 assert!(!history_ok(Some(10), Some(1_700_000_000_000), Some(5_000))); // mid-backfill
332 }
333
334 // A history pipeline with no watermark yet makes the all-pipelines
335 // timestamp `None`, and a live cohort that has not committed makes the
336 // live timestamp `None`: unhealthy, however large the allowed lag.
337 #[test]
338 fn history_not_caught_up_when_uncommitted() {
339 assert!(!history_ok(Some(u32::MAX), Some(100_000), None));
340 assert!(!history_ok(Some(u32::MAX), None, Some(100_000)));
341 assert!(!history_ok(Some(u32::MAX), None, None));
342 }
343
344 // Right after the embedded restore the live cohort's watermarks carry
345 // only the restore checkpoint, so the live and all-pipelines timestamps
346 // both read 0 while history sits at its seed: unhealthy, however large
347 // the allowed lag, until the live indexer commits a real timestamp.
348 #[test]
349 fn history_not_caught_up_without_recorded_timestamps() {
350 assert!(!history_ok(Some(u32::MAX), Some(0), Some(0)));
351 assert!(!history_ok(Some(u32::MAX), Some(100_000), Some(0)));
352 assert!(!history_ok(Some(u32::MAX), Some(0), Some(100_000)));
353 }
354
355 // A failure reading either timestamp propagates rather than being read as
356 // healthy.
357 #[test]
358 fn history_read_error_propagates() {
359 let ok = || -> StorageResult<Option<u64>> { Ok(Some(100_000)) };
360 let fail = || -> StorageResult<Option<u64>> { Err(StorageError::custom("read failed")) };
361 let all_fails = IndexTimestamps {
362 read_live_indexed_ms: &ok,
363 read_all_indexed_ms: &fail,
364 };
365 assert!(history_caught_up(Some(10), Some(all_fails)).is_err());
366 let live_fails = IndexTimestamps {
367 read_live_indexed_ms: &fail,
368 read_all_indexed_ms: &ok,
369 };
370 assert!(history_caught_up(Some(10), Some(live_fails)).is_err());
371 }
372}