Skip to main content

sui_rpc_api/service/
health.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use axum::extract::{Query, State};
5use std::time::Duration;
6use std::time::SystemTime;
7use sui_types::storage::error::Result as StorageResult;
8
9use crate::Result;
10use crate::RpcService;
11
12/// The largest gap, in checkpoints, between the latest executed checkpoint and
13/// the highest checkpoint the live-object index has committed while still
14/// considered healthy.
15///
16/// The embedded indexer follows the tip asynchronously, so the live-object
17/// index always trails the executed tip by a little (roughly the indexer's
18/// snapshot window). A gap larger than this means the live index has fallen
19/// behind -- e.g. the indexer has stalled -- and cannot serve current
20/// live-object reads. The ledger-history cohort backfills separately after a
21/// restore and is excluded from this gap, so by default a node is healthy as
22/// soon as its live-object reads are caught up; callers that also need
23/// history caught up opt in with `max_history_lag_seconds` (see [`Threshold`]).
24const MAX_HEALTHY_INDEX_LAG: u64 = 60;
25
26impl RpcService {
27    /// Perform a simple health check on the service.
28    ///
29    /// `threshold_seconds` is the threshold, or delta, between the server's
30    /// system time and the timestamp in the most recently executed checkpoint
31    /// for which the server is considered to be healthy. If not provided, the
32    /// server's tip is not subject to a staleness check.
33    ///
34    /// Independent of the threshold, when indexing is enabled the server is
35    /// only considered healthy once its live-object indexes have caught up to
36    /// within `MAX_HEALTHY_INDEX_LAG` checkpoints of the latest executed
37    /// checkpoint.
38    ///
39    /// `max_history_lag_seconds`, when provided, additionally requires every
40    /// index pipeline -- including the ledger-history cohort, which backfills
41    /// separately after a restore -- to have caught up to within that many
42    /// seconds of chain time of the live-object index: the chain timestamps
43    /// of the checkpoints their watermarks have reached are compared, so the
44    /// bound means the same on every network and at every checkpoint rate. A
45    /// history index with no watermark yet is never healthy, and neither is a
46    /// node without rpc indexes: its reported tip is its executed checkpoint,
47    /// which can run ahead of what indexed replicas have reached.
48    ///
49    /// When indexing is disabled and `max_history_lag_seconds` is not
50    /// provided, the index checks are skipped.
51    pub fn health_check(
52        &self,
53        threshold_seconds: Option<u32>,
54        max_history_lag_seconds: Option<u32>,
55    ) -> Result<()> {
56        let latest = self.reader.inner().get_latest_checkpoint()?;
57
58        // If we have a provided threshold, check that it's close to the current
59        // time.
60        if let Some(threshold_seconds) = threshold_seconds {
61            let latest_chain_time = latest.timestamp();
62
63            let threshold = SystemTime::now() - Duration::from_secs(threshold_seconds as u64);
64
65            if latest_chain_time < threshold {
66                return Err(anyhow::anyhow!(
67                    "The latest checkpoint timestamp is less than the provided threshold"
68                )
69                .into());
70            }
71        }
72
73        // When indexing is enabled, the node is only healthy once its
74        // live-object indexes (owned objects, types, balances) have kept up
75        // with the executed tip. Those indexes are restored to the tip and
76        // follow it, so a healthy node's live frontier trails execution by at
77        // most the indexer's snapshot window. The ledger-history cohort
78        // backfills independently after a restore and is only checked when the
79        // caller passes `max_history_lag_seconds`: gating on it by default
80        // would report a node unhealthy for the whole backfill even though its
81        // live-object reads are already caught up. The executed tip is read
82        // unbounded (rather than via `get_latest_checkpoint`, which is itself
83        // bounded to the live frontier) so a stalled live indexer, whose
84        // frontier falls behind ongoing execution, is still detected.
85        //
86        // The reported tip is bounded to the live frontier, so a replica
87        // whose history trails it advertises checkpoints its ledger-history
88        // reads cannot serve yet. Measure the history gap against that
89        // frontier rather than the executed tip, so the live index's own
90        // (separately bounded) lag behind execution is not counted twice. A
91        // node without an index surface (indexing disabled) reports its
92        // executed checkpoint as the tip, so it fails the history check
93        // whenever `max_history_lag_seconds` is set and skips the index checks
94        // otherwise.
95        let history_ok = if let Some(indexes) = self.reader.inner().indexes() {
96            let executed = self
97                .reader
98                .inner()
99                .get_highest_executed_checkpoint_seq_number()?;
100            let highest_live_indexed = indexes.get_highest_live_indexed_checkpoint_seq_number()?;
101
102            if !index_caught_up(executed, highest_live_indexed, MAX_HEALTHY_INDEX_LAG) {
103                return Err(anyhow::anyhow!(
104                    "the live-object index is not caught up to within {MAX_HEALTHY_INDEX_LAG} \
105                     checkpoints of the latest executed checkpoint"
106                )
107                .into());
108            }
109
110            history_caught_up(
111                max_history_lag_seconds,
112                Some(IndexTimestamps {
113                    read_live_indexed_ms: &|| {
114                        indexes.get_highest_live_indexed_checkpoint_timestamp_ms()
115                    },
116                    read_all_indexed_ms: &|| indexes.get_highest_indexed_checkpoint_timestamp_ms(),
117                }),
118            )?
119        } else {
120            history_caught_up(max_history_lag_seconds, None)?
121        };
122
123        if !history_ok {
124            return Err(anyhow::anyhow!(
125                "`max_history_lag_seconds` is set and the ledger-history indexes are absent or \
126                 not caught up to within that many seconds of the live-object index"
127            )
128            .into());
129        }
130
131        Ok(())
132    }
133}
134
135/// Whether the highest live-indexed checkpoint is close enough to the executed
136/// tip to be considered healthy.
137///
138/// `highest_live_indexed` is `None` when the live-object index has not committed
139/// any checkpoint yet, which is never healthy. The live frontier never runs
140/// ahead of execution (it indexes executed checkpoints), but an equal frontier
141/// saturates to a zero lag rather than underflowing.
142fn index_caught_up(executed_seq: u64, highest_live_indexed: Option<u64>, max_lag: u64) -> bool {
143    match highest_live_indexed {
144        Some(indexed) => executed_seq.saturating_sub(indexed) <= max_lag,
145        None => false,
146    }
147}
148
149/// The rpc index surface as the history check sees it: the chain timestamps,
150/// in milliseconds, of the checkpoints its watermarks have reached.
151struct IndexTimestamps<'a> {
152    /// Reads the live-object frontier's timestamp; `None` before the live
153    /// cohort's first commit.
154    read_live_indexed_ms: &'a dyn Fn() -> StorageResult<Option<u64>>,
155    /// Reads the timestamp of the lowest watermark across all pipelines (live
156    /// and ledger-history); `None` while any pipeline has no watermark yet.
157    read_all_indexed_ms: &'a dyn Fn() -> StorageResult<Option<u64>>,
158}
159
160/// Whether every index pipeline (live and ledger-history) has caught up to
161/// within `max_lag_seconds` of chain time of the live-object frontier.
162///
163/// With no `max_lag_seconds` the history cohort is not checked: the result is
164/// `true` and nothing is read. Otherwise `indexes` is `None` for a node without
165/// rpc indexes, which is never healthy. The all-pipelines minimum includes the
166/// live pipelines, but it is read after the live frontier, so a live cohort
167/// that advanced in between can leave it slightly above; that counts as zero
168/// lag. A `None` minimum (in practice a history pipeline after a restore,
169/// before its seed) is never healthy, and neither is a missing live frontier.
170///
171/// A timestamp of 0 means the watermark was written without one, not a real
172/// chain time: the embedded restore anchors the live cohort with only its
173/// checkpoint set, so until the live indexer commits its first checkpoint both
174/// timestamps read 0 while history may still sit at its seed far below. A 0 on
175/// either side is therefore never healthy either.
176fn history_caught_up(
177    max_lag_seconds: Option<u32>,
178    indexes: Option<IndexTimestamps<'_>>,
179) -> StorageResult<bool> {
180    let Some(max_lag_seconds) = max_lag_seconds else {
181        return Ok(true);
182    };
183    let Some(indexes) = indexes else {
184        return Ok(false);
185    };
186    let max_lag_ms = u64::from(max_lag_seconds) * 1_000;
187    let live = (indexes.read_live_indexed_ms)()?;
188    let all = (indexes.read_all_indexed_ms)()?;
189    Ok(match (live, all) {
190        (Some(live), Some(all)) if live > 0 && all > 0 => live.saturating_sub(all) <= max_lag_ms,
191        _ => false,
192    })
193}
194
195#[derive(Debug, serde::Serialize, serde::Deserialize)]
196pub struct Threshold {
197    /// The threshold, or delta, between the server's system time and the timestamp in the most
198    /// recently executed checkpoint for which the server is considered to be healthy.
199    ///
200    /// If not provided, the server will be considered healthy if it can simply fetch the latest
201    /// checkpoint from its store and, when indexing is enabled, its live-object indexes have
202    /// caught up to it.
203    pub threshold_seconds: Option<u32>,
204
205    /// The largest gap, in seconds of chain time, between the live-object index and the slowest
206    /// index pipeline (in practice the ledger-history indexes, which backfill separately after a
207    /// restore) for which the server is considered to be healthy. The gap is measured between
208    /// the timestamps of the checkpoints their watermarks have reached. A history index with no
209    /// watermark yet is unhealthy, and so is a node without rpc indexes (indexing disabled),
210    /// whose reported tip is its executed checkpoint.
211    ///
212    /// If not provided, ledger-history progress and the presence of rpc indexes do not affect
213    /// health.
214    pub max_history_lag_seconds: Option<u32>,
215}
216
217pub async fn health(
218    Query(Threshold {
219        threshold_seconds,
220        max_history_lag_seconds,
221    }): Query<Threshold>,
222    State(state): State<RpcService>,
223) -> impl axum::response::IntoResponse {
224    match state.health_check(threshold_seconds, max_history_lag_seconds) {
225        Ok(()) => (axum::http::StatusCode::OK, "up"),
226        Err(_) => (axum::http::StatusCode::SERVICE_UNAVAILABLE, "down"),
227    }
228}
229
230#[cfg(test)]
231mod tests {
232    use super::*;
233    use sui_types::storage::error::Error as StorageError;
234
235    // The live-object index has not committed any checkpoint yet: never
236    // healthy.
237    #[test]
238    fn not_caught_up_when_unindexed() {
239        assert!(!index_caught_up(100, None, MAX_HEALTHY_INDEX_LAG));
240        assert!(!index_caught_up(0, None, MAX_HEALTHY_INDEX_LAG));
241    }
242
243    // The live frontier is within (or at) the allowed lag of the executed tip:
244    // healthy.
245    #[test]
246    fn caught_up_within_lag() {
247        assert!(index_caught_up(100, Some(100), 60)); // no lag
248        assert!(index_caught_up(100, Some(40), 60)); // exactly at the bound
249        assert!(index_caught_up(100, Some(41), 60)); // inside the bound
250    }
251
252    // The live frontier trails the executed tip by more than the allowed lag
253    // (e.g. the live indexer stalled while execution advanced): unhealthy. A
254    // lagging ledger-history backfill does not reach this path -- it is not part
255    // of the live frontier.
256    #[test]
257    fn not_caught_up_beyond_lag() {
258        assert!(!index_caught_up(100, Some(39), 60)); // one past the bound
259        assert!(!index_caught_up(1_000, Some(0), 60)); // live index far behind
260    }
261
262    // A live frontier level with the executed tip saturates to zero lag rather
263    // than underflowing.
264    #[test]
265    fn caught_up_when_index_at_tip() {
266        assert!(index_caught_up(100, Some(100), 60));
267        assert!(index_caught_up(100, Some(200), 60));
268    }
269
270    /// Evaluate the history check on a node with rpc indexes, with fixed
271    /// live-frontier and all-pipelines timestamps in milliseconds.
272    fn history_ok(max_lag_seconds: Option<u32>, live_ms: Option<u64>, all_ms: Option<u64>) -> bool {
273        let read_live = || Ok(live_ms);
274        let read_all = || Ok(all_ms);
275        let indexes = IndexTimestamps {
276            read_live_indexed_ms: &read_live,
277            read_all_indexed_ms: &read_all,
278        };
279        history_caught_up(max_lag_seconds, Some(indexes)).unwrap()
280    }
281
282    // Without `max_history_lag_seconds` the history cohort does not affect
283    // health and no index timestamp is read, so neither a mid-backfill history
284    // cohort nor a failing read can take the node out of rotation. A node
285    // without rpc indexes is likewise unaffected.
286    #[test]
287    fn history_ignored_when_unset() {
288        let read = || -> StorageResult<Option<u64>> {
289            panic!("index timestamp read with max_history_lag_seconds unset")
290        };
291        let indexes = IndexTimestamps {
292            read_live_indexed_ms: &read,
293            read_all_indexed_ms: &read,
294        };
295        assert!(history_caught_up(None, Some(indexes)).unwrap());
296        assert!(history_caught_up(None, None).unwrap());
297    }
298
299    // A node without rpc indexes reports its executed checkpoint as the tip,
300    // which can run ahead of what indexed replicas have reached: unhealthy
301    // whenever `max_history_lag_seconds` is set, however large.
302    #[test]
303    fn history_not_caught_up_without_indexes() {
304        assert!(!history_caught_up(Some(u32::MAX), None).unwrap());
305        assert!(!history_caught_up(Some(0), None).unwrap());
306    }
307
308    // Every pipeline is within (or at) the allowed lag of the live frontier:
309    // healthy. The bound is in seconds and the timestamps in milliseconds.
310    #[test]
311    fn history_caught_up_within_lag() {
312        assert!(history_ok(Some(10), Some(100_000), Some(100_000))); // no lag
313        assert!(history_ok(Some(10), Some(100_000), Some(90_000))); // exactly at the bound
314        assert!(history_ok(Some(10), Some(100_000), Some(95_000))); // inside the bound
315    }
316
317    // The all-pipelines minimum is read after the live frontier, so the live
318    // cohort can advance in between and leave it above the live timestamp:
319    // that saturates to zero lag rather than underflowing.
320    #[test]
321    fn history_caught_up_when_above_live() {
322        assert!(history_ok(Some(10), Some(100_000), Some(100_500)));
323        assert!(history_ok(Some(0), Some(100_000), Some(100_500)));
324    }
325
326    // History is backfilling far behind the live frontier (e.g. after a
327    // snapshot restore): unhealthy.
328    #[test]
329    fn history_not_caught_up_beyond_lag() {
330        assert!(!history_ok(Some(10), Some(100_000), Some(89_999))); // one ms past the bound
331        assert!(!history_ok(Some(10), Some(1_700_000_000_000), Some(5_000))); // mid-backfill
332    }
333
334    // A history pipeline with no watermark yet makes the all-pipelines
335    // timestamp `None`, and a live cohort that has not committed makes the
336    // live timestamp `None`: unhealthy, however large the allowed lag.
337    #[test]
338    fn history_not_caught_up_when_uncommitted() {
339        assert!(!history_ok(Some(u32::MAX), Some(100_000), None));
340        assert!(!history_ok(Some(u32::MAX), None, Some(100_000)));
341        assert!(!history_ok(Some(u32::MAX), None, None));
342    }
343
344    // Right after the embedded restore the live cohort's watermarks carry
345    // only the restore checkpoint, so the live and all-pipelines timestamps
346    // both read 0 while history sits at its seed: unhealthy, however large
347    // the allowed lag, until the live indexer commits a real timestamp.
348    #[test]
349    fn history_not_caught_up_without_recorded_timestamps() {
350        assert!(!history_ok(Some(u32::MAX), Some(0), Some(0)));
351        assert!(!history_ok(Some(u32::MAX), Some(100_000), Some(0)));
352        assert!(!history_ok(Some(u32::MAX), Some(0), Some(100_000)));
353    }
354
355    // A failure reading either timestamp propagates rather than being read as
356    // healthy.
357    #[test]
358    fn history_read_error_propagates() {
359        let ok = || -> StorageResult<Option<u64>> { Ok(Some(100_000)) };
360        let fail = || -> StorageResult<Option<u64>> { Err(StorageError::custom("read failed")) };
361        let all_fails = IndexTimestamps {
362            read_live_indexed_ms: &ok,
363            read_all_indexed_ms: &fail,
364        };
365        assert!(history_caught_up(Some(10), Some(all_fails)).is_err());
366        let live_fails = IndexTimestamps {
367            read_live_indexed_ms: &fail,
368            read_all_indexed_ms: &ok,
369        };
370        assert!(history_caught_up(Some(10), Some(live_fails)).is_err());
371    }
372}