1use mysten_metrics::histogram::Histogram as MystenHistogram;
5use prometheus::{
6 Histogram, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
7 register_histogram_with_registry, register_int_counter_vec_with_registry,
8 register_int_counter_with_registry, register_int_gauge_vec_with_registry,
9 register_int_gauge_with_registry,
10};
11use std::convert::TryFrom;
12use std::sync::Arc;
13
14pub struct CheckpointMetrics {
15 pub last_certified_checkpoint: IntGauge,
16 pub last_constructed_checkpoint: IntGauge,
17 pub checkpoint_errors: IntCounter,
18 pub transactions_included_in_checkpoint: IntCounter,
19 pub checkpoint_roots_count: IntCounter,
20 pub checkpoint_participation: IntCounterVec,
21 pub last_received_checkpoint_signatures: IntGaugeVec,
22 pub last_sent_checkpoint_signature: IntGauge,
23 pub last_skipped_checkpoint_signature_submission: IntGauge,
24 pub last_ignored_checkpoint_signature_received: IntGauge,
25 pub highest_accumulated_epoch: IntGauge,
26 pub checkpoint_creation_latency: Histogram,
27 pub checkpoint_creation_latency_ms: MystenHistogram,
29 pub remote_checkpoint_forks: IntCounter,
30 pub split_brain_checkpoint_forks: IntCounter,
31 pub checkpoint_fork_crash_mode: IntGaugeVec,
32 pub transaction_fork_crash_mode: IntGaugeVec,
33 pub checkpoint_fork_auto_recovered: IntGauge,
34 pub transaction_fork_auto_recovered: IntGauge,
35 pub fork_auto_recovery_awaiting_new_binary: IntGauge,
36 pub fork_auto_recovery_blocked_uncertified: IntGauge,
37 pub last_created_checkpoint_age: Histogram,
38 pub last_created_checkpoint_age_ms: MystenHistogram,
40 pub last_certified_checkpoint_age: Histogram,
41 pub last_certified_checkpoint_age_ms: MystenHistogram,
43 pub accumulator_accounts_created: IntCounter,
44 pub accumulator_accounts_deleted: IntCounter,
45 pub accumulator_accounts_live: IntGauge,
46}
47
48impl CheckpointMetrics {
49 pub fn new(registry: &Registry) -> Arc<Self> {
50 let this = Self {
51 last_certified_checkpoint: register_int_gauge_with_registry!(
52 "last_certified_checkpoint",
53 "Last certified checkpoint",
54 registry
55 )
56 .unwrap(),
57 last_constructed_checkpoint: register_int_gauge_with_registry!(
58 "last_constructed_checkpoint",
59 "Last constructed checkpoint",
60 registry
61 )
62 .unwrap(),
63 last_created_checkpoint_age: register_histogram_with_registry!(
64 "last_created_checkpoint_age",
65 "Age of the last created checkpoint",
66 mysten_metrics::LATENCY_SEC_BUCKETS.to_vec(),
67 registry
68 ).unwrap(),
69 last_created_checkpoint_age_ms: MystenHistogram::new_in_registry(
70 "last_created_checkpoint_age_ms",
71 "Age of the last created checkpoint",
72 registry
73 ),
74 last_certified_checkpoint_age: register_histogram_with_registry!(
75 "last_certified_checkpoint_age",
76 "Age of the last certified checkpoint",
77 mysten_metrics::LATENCY_SEC_BUCKETS.to_vec(),
78 registry
79 ).unwrap(),
80 last_certified_checkpoint_age_ms: MystenHistogram::new_in_registry(
81 "last_certified_checkpoint_age_ms",
82 "Age of the last certified checkpoint",
83 registry
84 ),
85 checkpoint_errors: register_int_counter_with_registry!(
86 "checkpoint_errors",
87 "Checkpoints errors count",
88 registry
89 )
90 .unwrap(),
91 transactions_included_in_checkpoint: register_int_counter_with_registry!(
92 "transactions_included_in_checkpoint",
93 "Transactions included in a checkpoint",
94 registry
95 )
96 .unwrap(),
97 checkpoint_roots_count: register_int_counter_with_registry!(
98 "checkpoint_roots_count",
99 "Number of checkpoint roots received from consensus",
100 registry
101 )
102 .unwrap(),
103 checkpoint_participation: register_int_counter_vec_with_registry!(
104 "checkpoint_participation",
105 "Participation in checkpoint certification by validator",
106 &["signer"],
107 registry
108 )
109 .unwrap(),
110 last_received_checkpoint_signatures: register_int_gauge_vec_with_registry!(
111 "last_received_checkpoint_signatures",
112 "Last received checkpoint signatures by validator",
113 &["signer"],
114 registry
115 )
116 .unwrap(),
117 last_sent_checkpoint_signature: register_int_gauge_with_registry!(
118 "last_sent_checkpoint_signature",
119 "Last checkpoint signature sent by myself",
120 registry
121 )
122 .unwrap(),
123 last_skipped_checkpoint_signature_submission: register_int_gauge_with_registry!(
124 "last_skipped_checkpoint_signature_submission",
125 "Last checkpoint signature that this validator skipped submitting because it was already certfied.",
126 registry
127 )
128 .unwrap(),
129 last_ignored_checkpoint_signature_received: register_int_gauge_with_registry!(
130 "last_ignored_checkpoint_signature_received",
131 "Last received checkpoint signature that this validator ignored because it was already certfied.",
132 registry
133 )
134 .unwrap(),
135 highest_accumulated_epoch: register_int_gauge_with_registry!(
136 "highest_accumulated_epoch",
137 "Highest accumulated epoch",
138 registry
139 )
140 .unwrap(),
141 checkpoint_creation_latency: register_histogram_with_registry!(
142 "checkpoint_creation_latency",
143 "Latency from consensus commit timstamp to local checkpoint creation in milliseconds",
144 mysten_metrics::LATENCY_SEC_BUCKETS.to_vec(),
145 registry,
146 ).unwrap(),
147 checkpoint_creation_latency_ms: MystenHistogram::new_in_registry(
148 "checkpoint_creation_latency_ms",
149 "Latency from consensus commit timstamp to local checkpoint creation in milliseconds",
150 registry,
151 ),
152 remote_checkpoint_forks: register_int_counter_with_registry!(
153 "remote_checkpoint_forks",
154 "Number of remote checkpoints that forked from local checkpoints",
155 registry
156 )
157 .unwrap(),
158 split_brain_checkpoint_forks: register_int_counter_with_registry!(
159 "split_brain_checkpoint_forks",
160 "Number of checkpoints that have resulted in a split brain",
161 registry
162 )
163 .unwrap(),
164 checkpoint_fork_crash_mode: register_int_gauge_vec_with_registry!(
165 "checkpoint_fork_crash_mode",
166 "Indicates node is in crash mode due to checkpoint fork",
167 &["checkpoint_seq", "checkpoint_digest_prefix", "detected_at"],
168 registry
169 )
170 .unwrap(),
171 transaction_fork_crash_mode: register_int_gauge_vec_with_registry!(
172 "transaction_fork_crash_mode",
173 "Indicates node is in crash mode due to transaction fork",
174 &["tx_digest_prefix", "expected_effects_prefix", "actual_effects_prefix", "detected_at"],
175 registry
176 )
177 .unwrap(),
178 checkpoint_fork_auto_recovered: register_int_gauge_with_registry!(
179 "checkpoint_fork_auto_recovered",
180 "Set to 1 when the node automatically recovered from a checkpoint fork on startup",
181 registry
182 )
183 .unwrap(),
184 transaction_fork_auto_recovered: register_int_gauge_with_registry!(
185 "transaction_fork_auto_recovered",
186 "Set to 1 when the node automatically recovered from a transaction fork on startup",
187 registry
188 )
189 .unwrap(),
190 fork_auto_recovery_awaiting_new_binary: register_int_gauge_with_registry!(
191 "fork_auto_recovery_awaiting_new_binary",
192 "Set to 1 when a fork marker was recorded by the currently running binary \
193 version, which would deterministically fork again; the node is halting until a \
194 corrected binary is deployed",
195 registry
196 )
197 .unwrap(),
198 fork_auto_recovery_blocked_uncertified: register_int_gauge_with_registry!(
199 "fork_auto_recovery_blocked_uncertified",
200 "Set to 1 when a fork marker was left in place because the fork was not detected \
201 against a certified checkpoint or the covering certificate could not be \
202 verified locally, so automatic recovery would risk equivocating on an undecided \
203 outcome; the node is halting awaiting operator intervention",
204 registry
205 )
206 .unwrap(),
207 accumulator_accounts_created: register_int_counter_with_registry!(
208 "accumulator_accounts_created",
209 "Total number of accumulator account objects created by settlement transactions",
210 registry
211 )
212 .unwrap(),
213 accumulator_accounts_deleted: register_int_counter_with_registry!(
214 "accumulator_accounts_deleted",
215 "Total number of accumulator account objects deleted by settlement transactions",
216 registry
217 )
218 .unwrap(),
219 accumulator_accounts_live: register_int_gauge_with_registry!(
220 "accumulator_accounts_live",
221 "Current number of live accumulator account objects after settlement processing",
222 registry
223 )
224 .unwrap(),
225 };
226 Arc::new(this)
227 }
228
229 pub fn new_for_tests() -> Arc<Self> {
230 Self::new(&Registry::new())
231 }
232
233 pub fn initialize_accumulator_accounts_live(&self, live_accounts: u64) {
234 self.accumulator_accounts_live
235 .set(i64::try_from(live_accounts).expect("accumulator account count exceeds i64"));
236 }
237
238 pub fn report_accumulator_account_changes(&self, created: u64, deleted: u64) {
239 self.accumulator_accounts_created.inc_by(created);
240 self.accumulator_accounts_deleted.inc_by(deleted);
241
242 let created = i64::try_from(created).expect("created accumulator accounts exceeds i64");
243 let deleted = i64::try_from(deleted).expect("deleted accumulator accounts exceeds i64");
244 self.accumulator_accounts_live.add(created - deleted);
245 }
246}