1use std::sync::Arc;
5
6use prometheus::{
7 Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
8 Registry, exponential_buckets, register_gauge_vec_with_registry, register_gauge_with_registry,
9 register_histogram_vec_with_registry, register_histogram_with_registry,
10 register_int_counter_vec_with_registry, register_int_counter_with_registry,
11 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
12};
13
14use crate::network::metrics::NetworkMetrics;
15
16const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
18 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
19 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
20 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
21];
22
23const NUM_BUCKETS: &[f64] = &[
24 1.0,
25 2.0,
26 4.0,
27 8.0,
28 10.0,
29 20.0,
30 40.0,
31 80.0,
32 100.0,
33 150.0,
34 200.0,
35 400.0,
36 800.0,
37 1000.0,
38 2000.0,
39 3000.0,
40 5000.0,
41 10000.0,
42 20000.0,
43 30000.0,
44 50000.0,
45 100_000.0,
46 200_000.0,
47 300_000.0,
48 500_000.0,
49 1_000_000.0,
50];
51
52const LATENCY_SEC_BUCKETS: &[f64] = &[
53 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
54 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
55 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
56];
57
58const SIZE_BUCKETS: &[f64] = &[
59 100.,
60 400.,
61 800.,
62 1_000.,
63 2_000.,
64 5_000.,
65 10_000.,
66 20_000.,
67 50_000.,
68 100_000.,
69 200_000.0,
70 300_000.0,
71 400_000.0,
72 500_000.0,
73 1_000_000.0,
74 2_000_000.0,
75 3_000_000.0,
76 5_000_000.0,
77 10_000_000.0,
78]; const ROUND_DELAY_BUCKETS: &[f64] = &[0.0, 0.5, 1.0, 2.0, 3.0, 4.0];
82
83pub struct Metrics {
84 pub(crate) node_metrics: NodeMetrics,
85 pub(crate) network_metrics: NetworkMetrics,
86}
87
88pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
89 let node_metrics = NodeMetrics::new(®istry);
90 let network_metrics = NetworkMetrics::new(®istry);
91
92 Arc::new(Metrics {
93 node_metrics,
94 network_metrics,
95 })
96}
97
98pub(crate) fn test_metrics() -> Arc<Metrics> {
99 initialise_metrics(Registry::new())
100}
101
102pub(crate) struct NodeMetrics {
103 pub(crate) authority_index: IntGaugeVec,
104 pub(crate) protocol_version: IntGauge,
105 pub(crate) block_commit_latency: Histogram,
106 pub(crate) proposed_blocks: IntCounterVec,
107 pub(crate) proposed_block_size: Histogram,
108 pub(crate) proposed_block_transactions: Histogram,
109 pub(crate) proposed_block_ancestors: Histogram,
110 pub(crate) proposed_block_ancestors_depth: HistogramVec,
111 pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec,
112 pub(crate) highest_verified_authority_round: IntGaugeVec,
113 pub(crate) lowest_verified_authority_round: IntGaugeVec,
114 pub(crate) block_proposal_interval: Histogram,
115 pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
116 pub(crate) block_proposal_leader_wait_count: IntCounterVec,
117 pub(crate) block_timestamp_drift_ms: IntCounterVec,
118 pub(crate) blocks_per_commit_count: Histogram,
119 pub(crate) blocks_pruned_on_commit: IntCounterVec,
120 pub(crate) commit_observer_last_recovered_commit_index: IntGauge,
121 pub(crate) core_add_blocks_batch_size: Histogram,
122 pub(crate) core_check_block_refs_batch_size: Histogram,
123 pub(crate) core_lock_dequeued: IntCounter,
124 pub(crate) core_lock_enqueued: IntCounter,
125 pub(crate) core_skipped_proposals: IntCounterVec,
126 pub(crate) handler_received_block_missing_ancestors: IntCounterVec,
127 pub(crate) highest_accepted_authority_round: IntGaugeVec,
128 pub(crate) highest_accepted_round: IntGauge,
129 pub(crate) accepted_block_time_drift_ms: IntCounterVec,
130 pub(crate) accepted_blocks: IntCounterVec,
131 pub(crate) dag_state_recent_blocks: IntGauge,
132 pub(crate) dag_state_recent_refs: IntGauge,
133 pub(crate) dag_state_store_read_count: IntCounterVec,
134 pub(crate) dag_state_store_write_count: IntCounter,
135 pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
136 pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
137 pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
138 pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
139 pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
140 pub(crate) synchronizer_fetch_failures: IntCounterVec,
141 pub(crate) synchronizer_skipped_fetch_requests: IntCounterVec,
142 pub(crate) synchronizer_process_fetched_failures: IntCounterVec,
143 pub(crate) synchronizer_periodic_sync_decision: IntCounterVec,
144 pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
145 pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
146 pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
147 pub(crate) invalid_blocks: IntCounterVec,
148 pub(crate) rejected_blocks: IntCounterVec,
149 pub(crate) subscribed_blocks: IntCounterVec,
150 pub(crate) observer_subscribed_blocks_batch_size: Histogram,
151 pub(crate) verified_blocks: IntCounterVec,
152 pub(crate) committed_leaders_total: IntCounterVec,
153 pub(crate) last_committed_authority_round: IntGaugeVec,
154 pub(crate) last_committed_leader_round: IntGauge,
155 pub(crate) last_commit_index: IntGauge,
156 pub(crate) last_commit_time_diff: Histogram,
157 pub(crate) last_known_own_block_round: IntGauge,
158 pub(crate) sync_last_known_own_block_retries: IntCounter,
159 pub(crate) commit_round_advancement_interval: Histogram,
160 pub(crate) last_decided_leader_round: IntGauge,
161 pub(crate) leader_timeout_total: IntCounterVec,
162 pub(crate) smart_selection_wait: IntCounter,
163 pub(crate) ancestor_state_change_by_authority: IntCounterVec,
164 pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
165 pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
166 pub(crate) missing_blocks_total: IntCounter,
167 pub(crate) missing_blocks_after_fetch_total: IntCounter,
168 pub(crate) num_of_bad_nodes: IntGauge,
169 pub(crate) quorum_receive_latency: Histogram,
170 pub(crate) block_receive_delay: IntCounterVec,
171 pub(crate) reputation_scores: IntGaugeVec,
172 pub(crate) leader_schedule_total_scores: IntGaugeVec,
173 pub(crate) leader_schedule_normalized_scores: GaugeVec,
174 pub(crate) leader_schedule_last_num_leaders: IntGauge,
175 pub(crate) leader_schedule_average_num_leaders: Gauge,
176 pub(crate) scope_processing_time: HistogramVec,
177 pub(crate) sub_dags_per_commit_count: Histogram,
178 pub(crate) block_suspensions: IntCounterVec,
179 pub(crate) block_unsuspensions: IntCounterVec,
180 pub(crate) suspended_block_time: HistogramVec,
181 pub(crate) block_manager_suspended_blocks: IntGauge,
182 pub(crate) block_manager_missing_ancestors: IntGauge,
183 pub(crate) block_manager_missing_blocks: IntGauge,
184 pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
185 pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
186 pub(crate) block_manager_gced_blocks: IntCounterVec,
187 pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
188 pub(crate) block_manager_skipped_blocks: IntCounterVec,
189 pub(crate) threshold_clock_round: IntGauge,
190 pub(crate) subscriber_connection_attempts: IntCounterVec,
191 pub(crate) subscribed_to: IntGaugeVec,
192 pub(crate) subscribed_by: IntGaugeVec,
193 pub(crate) commit_sync_inflight_fetches: IntGauge,
194 pub(crate) commit_sync_pending_fetches: IntGauge,
195 pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
196 pub(crate) commit_sync_fetched_commits: IntCounter,
197 pub(crate) commit_sync_fetched_blocks: IntCounter,
198 pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
199 pub(crate) commit_sync_quorum_index: IntGauge,
200 pub(crate) commit_sync_highest_synced_index: IntGauge,
201 pub(crate) commit_sync_highest_fetched_index: IntGauge,
202 pub(crate) commit_sync_local_index: IntGauge,
203 pub(crate) commit_sync_gap_on_processing: IntCounter,
204 pub(crate) commit_sync_fetch_loop_latency: Histogram,
205 pub(crate) commit_sync_fetch_once_latency: Histogram,
206 pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
207 pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
208 pub(crate) round_tracker_received_quorum_round_gaps: IntGaugeVec,
209 pub(crate) round_tracker_accepted_quorum_round_gaps: IntGaugeVec,
210 pub(crate) round_tracker_low_received_quorum_round: IntGaugeVec,
211 pub(crate) round_tracker_low_accepted_quorum_round: IntGaugeVec,
212 pub(crate) round_tracker_current_received_round_gaps: IntGaugeVec,
213 pub(crate) round_tracker_current_accepted_round_gaps: IntGaugeVec,
214 pub(crate) round_tracker_propagation_delays: Histogram,
215 pub(crate) round_tracker_last_propagation_delay: IntGauge,
216 pub(crate) round_prober_request_errors: IntCounterVec,
217 pub(crate) certifier_gc_round: IntGauge,
218 pub(crate) certifier_own_reject_votes: IntCounterVec,
219 pub(crate) finalizer_buffered_commits: IntGauge,
220 pub(crate) finalizer_round_delay: Histogram,
221 pub(crate) finalizer_transaction_status: IntCounterVec,
222 pub(crate) finalizer_reject_votes: IntCounterVec,
223 pub(crate) finalizer_output_commits: IntCounterVec,
224 pub(crate) finalizer_skipped_voting_blocks: IntCounterVec,
225 pub(crate) uptime: Histogram,
226}
227
228impl NodeMetrics {
229 pub(crate) fn new(registry: &Registry) -> Self {
230 Self {
231 authority_index: register_int_gauge_vec_with_registry!(
232 "authority_index",
233 "The index of the authority",
234 &["name"],
235 registry,
236 ).unwrap(),
237 protocol_version: register_int_gauge_with_registry!(
238 "protocol_version",
239 "The protocol version used in this epoch",
240 registry,
241 ).unwrap(),
242 block_commit_latency: register_histogram_with_registry!(
243 "block_commit_latency",
244 "The time taken between block creation and block commit.",
245 LATENCY_SEC_BUCKETS.to_vec(),
246 registry,
247 ).unwrap(),
248 proposed_blocks: register_int_counter_vec_with_registry!(
249 "proposed_blocks",
250 "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
251 &["force"],
252 registry,
253 ).unwrap(),
254 proposed_block_size: register_histogram_with_registry!(
255 "proposed_block_size",
256 "The size (in bytes) of proposed blocks",
257 SIZE_BUCKETS.to_vec(),
258 registry
259 ).unwrap(),
260 proposed_block_transactions: register_histogram_with_registry!(
261 "proposed_block_transactions",
262 "# of transactions contained in proposed blocks",
263 NUM_BUCKETS.to_vec(),
264 registry
265 ).unwrap(),
266 proposed_block_ancestors: register_histogram_with_registry!(
267 "proposed_block_ancestors",
268 "Number of ancestors in proposed blocks",
269 exponential_buckets(1.0, 1.4, 20).unwrap(),
270 registry,
271 ).unwrap(),
272 proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!(
273 "proposed_block_ancestors_timestamp_drift_ms",
274 "The drift in ms of ancestors' timestamps included in newly proposed blocks",
275 &["authority"],
276 registry,
277 ).unwrap(),
278 proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
279 "proposed_block_ancestors_depth",
280 "The depth in rounds of ancestors included in newly proposed blocks",
281 &["authority"],
282 exponential_buckets(1.0, 2.0, 14).unwrap(),
283 registry,
284 ).unwrap(),
285 highest_verified_authority_round: register_int_gauge_vec_with_registry!(
286 "highest_verified_authority_round",
287 "The highest round of verified block for the corresponding authority",
288 &["authority"],
289 registry,
290 ).unwrap(),
291 lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
292 "lowest_verified_authority_round",
293 "The lowest round of verified block for the corresponding authority",
294 &["authority"],
295 registry,
296 ).unwrap(),
297 block_proposal_interval: register_histogram_with_registry!(
298 "block_proposal_interval",
299 "Intervals (in secs) between block proposals.",
300 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
301 registry,
302 ).unwrap(),
303 block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
304 "block_proposal_leader_wait_ms",
305 "Total time in ms spent waiting for a leader when proposing blocks.",
306 &["authority"],
307 registry,
308 ).unwrap(),
309 block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
310 "block_proposal_leader_wait_count",
311 "Total times waiting for a leader when proposing blocks.",
312 &["authority"],
313 registry,
314 ).unwrap(),
315 block_timestamp_drift_ms: register_int_counter_vec_with_registry!(
316 "block_timestamp_drift_ms",
317 "The clock drift time between a received block and the current node's time.",
318 &["authority", "source"],
319 registry,
320 ).unwrap(),
321 blocks_per_commit_count: register_histogram_with_registry!(
322 "blocks_per_commit_count",
323 "The number of blocks per commit.",
324 NUM_BUCKETS.to_vec(),
325 registry,
326 ).unwrap(),
327 blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
328 "blocks_pruned_on_commit",
329 "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
330 &["authority", "commit_status"],
331 registry,
332 ).unwrap(),
333 commit_observer_last_recovered_commit_index: register_int_gauge_with_registry!(
334 "commit_observer_last_recovered_commit_index",
335 "The last commit index recovered by the commit observer",
336 registry,
337 ).unwrap(),
338 core_add_blocks_batch_size: register_histogram_with_registry!(
339 "core_add_blocks_batch_size",
340 "The number of blocks received from Core for processing on a single batch",
341 NUM_BUCKETS.to_vec(),
342 registry,
343 ).unwrap(),
344 core_check_block_refs_batch_size: register_histogram_with_registry!(
345 "core_check_block_refs_batch_size",
346 "The number of excluded blocks received from Core for search on a single batch",
347 NUM_BUCKETS.to_vec(),
348 registry,
349 ).unwrap(),
350 core_lock_dequeued: register_int_counter_with_registry!(
351 "core_lock_dequeued",
352 "Number of dequeued core requests",
353 registry,
354 ).unwrap(),
355 core_lock_enqueued: register_int_counter_with_registry!(
356 "core_lock_enqueued",
357 "Number of enqueued core requests",
358 registry,
359 ).unwrap(),
360 core_skipped_proposals: register_int_counter_vec_with_registry!(
361 "core_skipped_proposals",
362 "Number of proposals skipped in the Core, per reason",
363 &["reason"],
364 registry,
365 ).unwrap(),
366 handler_received_block_missing_ancestors: register_int_counter_vec_with_registry!(
367 "handler_received_block_missing_ancestors",
368 "Number of missing ancestors in blocks received in the handler, separated by peer",
369 &["authority"],
370 registry,
371 ).unwrap(),
372 highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
373 "highest_accepted_authority_round",
374 "The highest round where a block has been accepted per authority. Resets on restart.",
375 &["authority"],
376 registry,
377 ).unwrap(),
378 highest_accepted_round: register_int_gauge_with_registry!(
379 "highest_accepted_round",
380 "The highest round where a block has been accepted. Resets on restart.",
381 registry,
382 ).unwrap(),
383 accepted_block_time_drift_ms: register_int_counter_vec_with_registry!(
384 "accepted_block_time_drift_ms",
385 "The time drift in ms of an accepted block compared to local time",
386 &["authority"],
387 registry,
388 ).unwrap(),
389 accepted_blocks: register_int_counter_vec_with_registry!(
390 "accepted_blocks",
391 "Number of accepted blocks by source (own, others)",
392 &["source"],
393 registry,
394 ).unwrap(),
395 dag_state_recent_blocks: register_int_gauge_with_registry!(
396 "dag_state_recent_blocks",
397 "Number of recent blocks cached in the DagState",
398 registry,
399 ).unwrap(),
400 dag_state_recent_refs: register_int_gauge_with_registry!(
401 "dag_state_recent_refs",
402 "Number of recent refs cached in the DagState",
403 registry,
404 ).unwrap(),
405 dag_state_store_read_count: register_int_counter_vec_with_registry!(
406 "dag_state_store_read_count",
407 "Number of times DagState needs to read from store per operation type",
408 &["type"],
409 registry,
410 ).unwrap(),
411 dag_state_store_write_count: register_int_counter_with_registry!(
412 "dag_state_store_write_count",
413 "Number of times DagState needs to write to store",
414 registry,
415 ).unwrap(),
416 fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
417 "fetch_blocks_scheduler_inflight",
418 "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
419 registry,
420 ).unwrap(),
421 synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
422 "synchronizer_fetched_blocks_by_peer",
423 "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
424 &["peer", "type"],
425 registry,
426 ).unwrap(),
427 synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
428 "synchronizer_missing_blocks_by_authority",
429 "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
430 &["authority"],
431 registry,
432 ).unwrap(),
433 synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
434 "synchronizer_current_missing_blocks_by_authority",
435 "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
436 &["authority"],
437 registry,
438 ).unwrap(),
439 synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
440 "synchronizer_fetched_blocks_by_authority",
441 "Number of fetched blocks per block author via the synchronizer",
442 &["authority", "type"],
443 registry,
444 ).unwrap(),
445 synchronizer_fetch_failures: register_int_counter_vec_with_registry!(
446 "synchronizer_fetch_failures",
447 "Number of fetch failures against each peer",
448 &["peer", "type"],
449 registry,
450 ).unwrap(),
451 synchronizer_skipped_fetch_requests: register_int_counter_vec_with_registry!(
452 "synchronizer_skipped_fetch_requests",
453 "Number of fetch requests skipped against each peer, because the peer is saturated",
454 &["peer"],
455 registry,
456 ).unwrap(),
457 synchronizer_process_fetched_failures: register_int_counter_vec_with_registry!(
458 "synchronizer_process_fetched_failures",
459 "Number of failures for processing fetched blocks against each peer",
460 &["peer", "type"],
461 registry,
462 ).unwrap(),
463 synchronizer_periodic_sync_decision: register_int_counter_vec_with_registry!(
464 "synchronizer_periodic_sync_decision",
465 "Decision to run periodic sync",
466 &["decision", "reason"],
467 registry,
468 ).unwrap(),
469 network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
470 "network_received_excluded_ancestors_from_authority",
471 "Number of excluded ancestors received from each authority.",
472 &["authority"],
473 registry,
474 ).unwrap(),
475 network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
476 "network_excluded_ancestors_count_by_authority",
477 "Total number of excluded ancestors per authority.",
478 &["authority"],
479 registry,
480 ).unwrap(),
481 network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
482 "network_excluded_ancestors_sent_to_fetch",
483 "Number of excluded ancestors sent to fetch.",
484 &["authority"],
485 registry,
486 ).unwrap(),
487 last_known_own_block_round: register_int_gauge_with_registry!(
488 "last_known_own_block_round",
489 "The highest round of our own block as this has been synced from peers during an amnesia recovery",
490 registry,
491 ).unwrap(),
492 sync_last_known_own_block_retries: register_int_counter_with_registry!(
493 "sync_last_known_own_block_retries",
494 "Number of times this node tried to fetch the last own block from peers",
495 registry,
496 ).unwrap(),
497 invalid_blocks: register_int_counter_vec_with_registry!(
498 "invalid_blocks",
499 "Number of invalid blocks per peer authority",
500 &["authority", "source", "error"],
501 registry,
502 ).unwrap(),
503 rejected_blocks: register_int_counter_vec_with_registry!(
504 "rejected_blocks",
505 "Number of blocks rejected before verifications",
506 &["reason"],
507 registry,
508 ).unwrap(),
509 subscribed_blocks: register_int_counter_vec_with_registry!(
510 "subscribed_blocks",
511 "Number of blocks received from each peer before verification",
512 &["authority"],
513 registry,
514 ).unwrap(),
515 observer_subscribed_blocks_batch_size: register_histogram_with_registry!(
516 "observer_subscribed_blocks_batch_size",
517 "The number of blocks received from a peer before verification in a single batch",
518 NUM_BUCKETS.to_vec(),
519 registry,
520 ).unwrap(),
521 verified_blocks: register_int_counter_vec_with_registry!(
522 "verified_blocks",
523 "Number of blocks received from each peer that are verified",
524 &["authority"],
525 registry,
526 ).unwrap(),
527 committed_leaders_total: register_int_counter_vec_with_registry!(
528 "committed_leaders_total",
529 "Total number of (direct or indirect) committed leaders per authority",
530 &["authority", "commit_type"],
531 registry,
532 ).unwrap(),
533 last_committed_authority_round: register_int_gauge_vec_with_registry!(
534 "last_committed_authority_round",
535 "The last round committed by authority.",
536 &["authority"],
537 registry,
538 ).unwrap(),
539 last_committed_leader_round: register_int_gauge_with_registry!(
540 "last_committed_leader_round",
541 "The last round where a leader was committed to store and sent to commit consumer.",
542 registry,
543 ).unwrap(),
544 last_commit_index: register_int_gauge_with_registry!(
545 "last_commit_index",
546 "Index of the last commit.",
547 registry,
548 ).unwrap(),
549 last_commit_time_diff: register_histogram_with_registry!(
550 "last_commit_time_diff",
551 "The time diff between the last commit and previous one.",
552 LATENCY_SEC_BUCKETS.to_vec(),
553 registry,
554 ).unwrap(),
555 commit_round_advancement_interval: register_histogram_with_registry!(
556 "commit_round_advancement_interval",
557 "Intervals (in secs) between commit round advancements.",
558 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
559 registry,
560 ).unwrap(),
561 last_decided_leader_round: register_int_gauge_with_registry!(
562 "last_decided_leader_round",
563 "The last round where a commit decision was made.",
564 registry,
565 ).unwrap(),
566 leader_timeout_total: register_int_counter_vec_with_registry!(
567 "leader_timeout_total",
568 "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
569 &["timeout_type"],
570 registry,
571 ).unwrap(),
572 smart_selection_wait: register_int_counter_with_registry!(
573 "smart_selection_wait",
574 "Number of times we waited for smart ancestor selection.",
575 registry,
576 ).unwrap(),
577 ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
578 "ancestor_state_change_by_authority",
579 "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
580 &["authority", "state"],
581 registry,
582 ).unwrap(),
583 excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
584 "excluded_proposal_ancestors_count_by_authority",
585 "Total number of excluded ancestors per authority during proposal.",
586 &["authority"],
587 registry,
588 ).unwrap(),
589 included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
590 "included_excluded_proposal_ancestors_count_by_authority",
591 "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
592 &["authority", "type"],
593 registry,
594 ).unwrap(),
595 missing_blocks_total: register_int_counter_with_registry!(
596 "missing_blocks_total",
597 "Total cumulative number of missing blocks",
598 registry,
599 ).unwrap(),
600 missing_blocks_after_fetch_total: register_int_counter_with_registry!(
601 "missing_blocks_after_fetch_total",
602 "Total number of missing blocks after fetching blocks from peer",
603 registry,
604 ).unwrap(),
605 num_of_bad_nodes: register_int_gauge_with_registry!(
606 "num_of_bad_nodes",
607 "The number of bad nodes in the new leader schedule",
608 registry
609 ).unwrap(),
610 quorum_receive_latency: register_histogram_with_registry!(
611 "quorum_receive_latency",
612 "The time it took to receive a new round quorum of blocks",
613 registry
614 ).unwrap(),
615 block_receive_delay: register_int_counter_vec_with_registry!(
616 "block_receive_delay",
617 "Total delay from the start of the round to receiving the block, in milliseconds per authority",
618 &["authority"],
619 registry,
620 ).unwrap(),
621 reputation_scores: register_int_gauge_vec_with_registry!(
622 "reputation_scores",
623 "Reputation scores for each authority",
624 &["authority"],
625 registry,
626 ).unwrap(),
627 leader_schedule_total_scores: register_int_gauge_vec_with_registry!(
628 "leader_schedule_total_scores",
629 "LeaderScheduleV3 running per-authority score over the scoring window. Each commit contributes voted_for_stake * certified_by_stake (stake^2 units)",
630 &["authority"],
631 registry,
632 ).unwrap(),
633 leader_schedule_normalized_scores: register_gauge_vec_with_registry!(
634 "leader_schedule_normalized_scores",
635 "LeaderScheduleV3 per-authority total score divided by (running sum of per-commit leader stake) * committee total stake. Stake^2 over stake^2; a fraction with no stake dimension",
636 &["authority"],
637 registry,
638 ).unwrap(),
639 leader_schedule_last_num_leaders: register_int_gauge_with_registry!(
640 "leader_schedule_last_num_leaders",
641 "LeaderScheduleV3 number of leader-round blocks in the most recently scored commit",
642 registry,
643 ).unwrap(),
644 leader_schedule_average_num_leaders: register_gauge_with_registry!(
645 "leader_schedule_average_num_leaders",
646 "LeaderScheduleV3 moving average of the number of leaders per commit across the scoring window",
647 registry,
648 ).unwrap(),
649 scope_processing_time: register_histogram_vec_with_registry!(
650 "scope_processing_time",
651 "The processing time of a specific code scope",
652 &["scope"],
653 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
654 registry
655 ).unwrap(),
656 sub_dags_per_commit_count: register_histogram_with_registry!(
657 "sub_dags_per_commit_count",
658 "The number of subdags per commit.",
659 registry,
660 ).unwrap(),
661 block_suspensions: register_int_counter_vec_with_registry!(
662 "block_suspensions",
663 "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
664 &["authority"],
665 registry,
666 ).unwrap(),
667 block_unsuspensions: register_int_counter_vec_with_registry!(
668 "block_unsuspensions",
669 "The number of block unsuspensions.",
670 &["authority"],
671 registry,
672 ).unwrap(),
673 suspended_block_time: register_histogram_vec_with_registry!(
674 "suspended_block_time",
675 "The time for which a block remains suspended",
676 &["authority"],
677 registry,
678 ).unwrap(),
679 block_manager_suspended_blocks: register_int_gauge_with_registry!(
680 "block_manager_suspended_blocks",
681 "The number of blocks currently suspended in the block manager",
682 registry,
683 ).unwrap(),
684 block_manager_missing_ancestors: register_int_gauge_with_registry!(
685 "block_manager_missing_ancestors",
686 "The number of missing ancestors tracked in the block manager",
687 registry,
688 ).unwrap(),
689 block_manager_missing_blocks: register_int_gauge_with_registry!(
690 "block_manager_missing_blocks",
691 "The number of blocks missing content tracked in the block manager",
692 registry,
693 ).unwrap(),
694 block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
695 "block_manager_missing_blocks_by_authority",
696 "The number of new missing blocks by block authority",
697 &["authority"],
698 registry,
699 ).unwrap(),
700 block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
701 "block_manager_missing_ancestors_by_authority",
702 "The number of missing ancestors by ancestor authority across received blocks",
703 &["authority"],
704 registry,
705 ).unwrap(),
706 block_manager_gced_blocks: register_int_counter_vec_with_registry!(
707 "block_manager_gced_blocks",
708 "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
709 &["authority"],
710 registry,
711 ).unwrap(),
712 block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
713 "block_manager_gc_unsuspended_blocks",
714 "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
715 &["authority"],
716 registry,
717 ).unwrap(),
718 block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
719 "block_manager_skipped_blocks",
720 "The number of blocks skipped by the block manager due to block round being <= gc_round",
721 &["authority"],
722 registry,
723 ).unwrap(),
724 threshold_clock_round: register_int_gauge_with_registry!(
725 "threshold_clock_round",
726 "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
727 registry,
728 ).unwrap(),
729 subscriber_connection_attempts: register_int_counter_vec_with_registry!(
730 "subscriber_connection_attempts",
731 "The number of connection attempts per peer",
732 &["authority", "status"],
733 registry,
734 ).unwrap(),
735 subscribed_to: register_int_gauge_vec_with_registry!(
736 "subscribed_to",
737 "Peers that this authority subscribed to for block streams.",
738 &["authority"],
739 registry,
740 ).unwrap(),
741 subscribed_by: register_int_gauge_vec_with_registry!(
742 "subscribed_by",
743 "Peers subscribing for block streams from this authority.",
744 &["authority"],
745 registry,
746 ).unwrap(),
747 commit_sync_inflight_fetches: register_int_gauge_with_registry!(
748 "commit_sync_inflight_fetches",
749 "The number of inflight fetches in commit syncer",
750 registry,
751 ).unwrap(),
752 commit_sync_pending_fetches: register_int_gauge_with_registry!(
753 "commit_sync_pending_fetches",
754 "The number of pending fetches in commit syncer",
755 registry,
756 ).unwrap(),
757 commit_sync_fetched_commits: register_int_counter_with_registry!(
758 "commit_sync_fetched_commits",
759 "The number of commits fetched via commit syncer",
760 registry,
761 ).unwrap(),
762 commit_sync_fetched_blocks: register_int_counter_with_registry!(
763 "commit_sync_fetched_blocks",
764 "The number of blocks fetched via commit syncer",
765 registry,
766 ).unwrap(),
767 commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
768 "commit_sync_total_fetched_blocks_size",
769 "The total size in bytes of blocks fetched via commit syncer",
770 registry,
771 ).unwrap(),
772 commit_sync_quorum_index: register_int_gauge_with_registry!(
773 "commit_sync_quorum_index",
774 "The maximum commit index voted by a quorum of authorities",
775 registry,
776 ).unwrap(),
777 commit_sync_highest_synced_index: register_int_gauge_with_registry!(
778 "commit_sync_fetched_index",
779 "The max commit index among local and fetched commits",
780 registry,
781 ).unwrap(),
782 commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
783 "commit_sync_highest_fetched_index",
784 "The max commit index that has been fetched via network",
785 registry,
786 ).unwrap(),
787 commit_sync_local_index: register_int_gauge_with_registry!(
788 "commit_sync_local_index",
789 "The local commit index",
790 registry,
791 ).unwrap(),
792 commit_sync_gap_on_processing: register_int_counter_with_registry!(
793 "commit_sync_gap_on_processing",
794 "Number of instances where a gap was found in fetched commit processing",
795 registry,
796 ).unwrap(),
797 commit_sync_fetch_loop_latency: register_histogram_with_registry!(
798 "commit_sync_fetch_loop_latency",
799 "The time taken to finish fetching commits and blocks from a given range",
800 LATENCY_SEC_BUCKETS.to_vec(),
801 registry,
802 ).unwrap(),
803 commit_sync_fetch_once_latency: register_histogram_with_registry!(
804 "commit_sync_fetch_once_latency",
805 "The time taken to fetch commits and blocks once",
806 LATENCY_SEC_BUCKETS.to_vec(),
807 registry,
808 ).unwrap(),
809 commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
810 "commit_sync_fetch_once_errors",
811 "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
812 &["authority", "error"],
813 registry
814 ).unwrap(),
815 commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
816 "commit_sync_fetch_missing_blocks",
817 "Number of ancestor blocks that are missing when processing blocks via commit sync.",
818 &["authority"],
819 registry,
820 ).unwrap(),
821 commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
822 "commit_sync_fetch_commits_handler_uncertified_skipped",
823 "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
824 registry,
825 ).unwrap(),
826 round_tracker_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
827 "round_tracker_received_quorum_round_gaps",
828 "Received round gaps among peers for blocks proposed from each authority",
829 &["authority"],
830 registry
831 ).unwrap(),
832 round_tracker_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
833 "round_tracker_accepted_quorum_round_gaps",
834 "Accepted round gaps among peers for blocks proposed & accepted from each authority",
835 &["authority"],
836 registry
837 ).unwrap(),
838 round_tracker_low_received_quorum_round: register_int_gauge_vec_with_registry!(
839 "round_tracker_low_received_quorum_round",
840 "Low quorum round among peers for blocks proposed from each authority",
841 &["authority"],
842 registry
843 ).unwrap(),
844 round_tracker_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
845 "round_tracker_low_accepted_quorum_round",
846 "Low quorum round among peers for blocks proposed & accepted from each authority",
847 &["authority"],
848 registry
849 ).unwrap(),
850 round_tracker_current_received_round_gaps: register_int_gauge_vec_with_registry!(
851 "round_tracker_current_received_round_gaps",
852 "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
853 &["authority"],
854 registry
855 ).unwrap(),
856 round_tracker_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
857 "round_tracker_current_accepted_round_gaps",
858 "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
859 &["authority"],
860 registry
861 ).unwrap(),
862 round_tracker_propagation_delays: register_histogram_with_registry!(
863 "round_tracker_propagation_delays",
864 "Round gaps between the last proposed block round and the lower bound of own quorum round",
865 NUM_BUCKETS.to_vec(),
866 registry
867 ).unwrap(),
868 round_tracker_last_propagation_delay: register_int_gauge_with_registry!(
869 "round_tracker_last_propagation_delay",
870 "Most recent propagation delay observed by RoundTracker",
871 registry
872 ).unwrap(),
873 round_prober_request_errors: register_int_counter_vec_with_registry!(
874 "round_prober_request_errors",
875 "Number of errors when probing against peers per error type",
876 &["error_type"],
877 registry
878 ).unwrap(),
879 certifier_gc_round: register_int_gauge_with_registry!(
880 "certifier_gc_round",
881 "The current GC round of the certifier",
882 registry
883 ).unwrap(),
884 certifier_own_reject_votes: register_int_counter_vec_with_registry!(
885 "certifier_own_reject_votes",
886 "Number of own reject votes against each peer authority",
887 &["authority"],
888 registry
889 ).unwrap(),
890 finalizer_buffered_commits: register_int_gauge_with_registry!(
891 "finalizer_buffered_commits",
892 "The number of commits buffered in the finalizer",
893 registry,
894 ).unwrap(),
895 finalizer_round_delay: register_histogram_with_registry!(
896 "finalizer_round_delay",
897 "The delay between the round of the last committed block and the round of the finalized commit.",
898 ROUND_DELAY_BUCKETS.to_vec(),
899 registry,
900 ).unwrap(),
901 finalizer_transaction_status: register_int_counter_vec_with_registry!(
902 "finalizer_transaction_status",
903 "Number of transactions finalized by the finalizer, grouped by status.",
904 &["status"],
905 registry
906 ).unwrap(),
907 finalizer_reject_votes: register_int_counter_vec_with_registry!(
908 "finalizer_reject_votes",
909 "Number of reject votes casted by each authority observed by the finalizer.",
910 &["authority"],
911 registry
912 ).unwrap(),
913 finalizer_output_commits: register_int_counter_vec_with_registry!(
914 "finalizer_output_commits",
915 "Number of output commits finalized by the finalizer, grouped by type.",
916 &["type"],
917 registry
918 ).unwrap(),
919 finalizer_skipped_voting_blocks: register_int_counter_vec_with_registry!(
920 "finalizer_skipped_voting_blocks",
921 "Number of times where another block skipped voting due to potentially out of GC bound. Authority is from the voted block.",
922 &["authority", "type"],
923 registry
924 ).unwrap(),
925 uptime: register_histogram_with_registry!(
926 "uptime",
927 "Total node uptime",
928 LATENCY_SEC_BUCKETS.to_vec(),
929 registry,
930 ).unwrap(),
931 }
932 }
933}