consensus_core/
metrics.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::sync::Arc;
5
6use prometheus::{
7    Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
8    exponential_buckets, register_histogram_vec_with_registry, register_histogram_with_registry,
9    register_int_counter_vec_with_registry, register_int_counter_with_registry,
10    register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
11};
12
13use crate::network::metrics::NetworkMetrics;
14
15// starts from 1μs, 50μs, 100μs...
16const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
17    0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
18    0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
19    4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
20];
21
22const NUM_BUCKETS: &[f64] = &[
23    1.0,
24    2.0,
25    4.0,
26    8.0,
27    10.0,
28    20.0,
29    40.0,
30    80.0,
31    100.0,
32    150.0,
33    200.0,
34    400.0,
35    800.0,
36    1000.0,
37    2000.0,
38    3000.0,
39    5000.0,
40    10000.0,
41    20000.0,
42    30000.0,
43    50000.0,
44    100_000.0,
45    200_000.0,
46    300_000.0,
47    500_000.0,
48    1_000_000.0,
49];
50
51const LATENCY_SEC_BUCKETS: &[f64] = &[
52    0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
53    1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
54    9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
55];
56
57const SIZE_BUCKETS: &[f64] = &[
58    100.,
59    400.,
60    800.,
61    1_000.,
62    2_000.,
63    5_000.,
64    10_000.,
65    20_000.,
66    50_000.,
67    100_000.,
68    200_000.0,
69    300_000.0,
70    400_000.0,
71    500_000.0,
72    1_000_000.0,
73    2_000_000.0,
74    3_000_000.0,
75    5_000_000.0,
76    10_000_000.0,
77]; // size in bytes
78
79// Because of indirect finalization, the round delay should be at most 3 rounds.
80const ROUND_DELAY_BUCKETS: &[f64] = &[0.0, 0.5, 1.0, 2.0, 3.0, 4.0];
81
82pub struct Metrics {
83    pub(crate) node_metrics: NodeMetrics,
84    pub(crate) network_metrics: NetworkMetrics,
85}
86
87pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
88    let node_metrics = NodeMetrics::new(&registry);
89    let network_metrics = NetworkMetrics::new(&registry);
90
91    Arc::new(Metrics {
92        node_metrics,
93        network_metrics,
94    })
95}
96
97pub(crate) fn test_metrics() -> Arc<Metrics> {
98    initialise_metrics(Registry::new())
99}
100
101pub(crate) struct NodeMetrics {
102    pub(crate) authority_index: IntGaugeVec,
103    pub(crate) protocol_version: IntGauge,
104    pub(crate) block_commit_latency: Histogram,
105    pub(crate) proposed_blocks: IntCounterVec,
106    pub(crate) proposed_block_size: Histogram,
107    pub(crate) proposed_block_transactions: Histogram,
108    pub(crate) proposed_block_ancestors: Histogram,
109    pub(crate) proposed_block_ancestors_depth: HistogramVec,
110    pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec,
111    pub(crate) highest_verified_authority_round: IntGaugeVec,
112    pub(crate) lowest_verified_authority_round: IntGaugeVec,
113    pub(crate) block_proposal_interval: Histogram,
114    pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
115    pub(crate) block_proposal_leader_wait_count: IntCounterVec,
116    pub(crate) block_timestamp_drift_ms: IntCounterVec,
117    pub(crate) blocks_per_commit_count: Histogram,
118    pub(crate) blocks_pruned_on_commit: IntCounterVec,
119    pub(crate) commit_observer_last_recovered_commit_index: IntGauge,
120    pub(crate) core_add_blocks_batch_size: Histogram,
121    pub(crate) core_check_block_refs_batch_size: Histogram,
122    pub(crate) core_lock_dequeued: IntCounter,
123    pub(crate) core_lock_enqueued: IntCounter,
124    pub(crate) core_skipped_proposals: IntCounterVec,
125    pub(crate) handler_received_block_missing_ancestors: IntCounterVec,
126    pub(crate) highest_accepted_authority_round: IntGaugeVec,
127    pub(crate) highest_accepted_round: IntGauge,
128    pub(crate) accepted_block_time_drift_ms: IntCounterVec,
129    pub(crate) accepted_blocks: IntCounterVec,
130    pub(crate) dag_state_recent_blocks: IntGauge,
131    pub(crate) dag_state_recent_refs: IntGauge,
132    pub(crate) dag_state_store_read_count: IntCounterVec,
133    pub(crate) dag_state_store_write_count: IntCounter,
134    pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
135    pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
136    pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
137    pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
138    pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
139    pub(crate) synchronizer_fetch_failures: IntCounterVec,
140    pub(crate) synchronizer_skipped_fetch_requests: IntCounterVec,
141    pub(crate) synchronizer_process_fetched_failures: IntCounterVec,
142    pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
143    pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
144    pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
145    pub(crate) invalid_blocks: IntCounterVec,
146    pub(crate) rejected_blocks: IntCounterVec,
147    pub(crate) subscribed_blocks: IntCounterVec,
148    pub(crate) verified_blocks: IntCounterVec,
149    pub(crate) committed_leaders_total: IntCounterVec,
150    pub(crate) last_committed_authority_round: IntGaugeVec,
151    pub(crate) last_committed_leader_round: IntGauge,
152    pub(crate) last_commit_index: IntGauge,
153    pub(crate) last_commit_time_diff: Histogram,
154    pub(crate) last_known_own_block_round: IntGauge,
155    pub(crate) sync_last_known_own_block_retries: IntCounter,
156    pub(crate) commit_round_advancement_interval: Histogram,
157    pub(crate) last_decided_leader_round: IntGauge,
158    pub(crate) leader_timeout_total: IntCounterVec,
159    pub(crate) smart_selection_wait: IntCounter,
160    pub(crate) ancestor_state_change_by_authority: IntCounterVec,
161    pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
162    pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
163    pub(crate) missing_blocks_total: IntCounter,
164    pub(crate) missing_blocks_after_fetch_total: IntCounter,
165    pub(crate) num_of_bad_nodes: IntGauge,
166    pub(crate) quorum_receive_latency: Histogram,
167    pub(crate) block_receive_delay: IntCounterVec,
168    pub(crate) reputation_scores: IntGaugeVec,
169    pub(crate) scope_processing_time: HistogramVec,
170    pub(crate) sub_dags_per_commit_count: Histogram,
171    pub(crate) block_suspensions: IntCounterVec,
172    pub(crate) block_unsuspensions: IntCounterVec,
173    pub(crate) suspended_block_time: HistogramVec,
174    pub(crate) block_manager_suspended_blocks: IntGauge,
175    pub(crate) block_manager_missing_ancestors: IntGauge,
176    pub(crate) block_manager_missing_blocks: IntGauge,
177    pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
178    pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
179    pub(crate) block_manager_gced_blocks: IntCounterVec,
180    pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
181    pub(crate) block_manager_skipped_blocks: IntCounterVec,
182    pub(crate) threshold_clock_round: IntGauge,
183    pub(crate) subscriber_connection_attempts: IntCounterVec,
184    pub(crate) subscribed_to: IntGaugeVec,
185    pub(crate) subscribed_by: IntGaugeVec,
186    pub(crate) commit_sync_inflight_fetches: IntGauge,
187    pub(crate) commit_sync_pending_fetches: IntGauge,
188    pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
189    pub(crate) commit_sync_fetched_commits: IntCounter,
190    pub(crate) commit_sync_fetched_blocks: IntCounter,
191    pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
192    pub(crate) commit_sync_quorum_index: IntGauge,
193    pub(crate) commit_sync_highest_synced_index: IntGauge,
194    pub(crate) commit_sync_highest_fetched_index: IntGauge,
195    pub(crate) commit_sync_local_index: IntGauge,
196    pub(crate) commit_sync_gap_on_processing: IntCounter,
197    pub(crate) commit_sync_fetch_loop_latency: Histogram,
198    pub(crate) commit_sync_fetch_once_latency: Histogram,
199    pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
200    pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
201    pub(crate) round_tracker_received_quorum_round_gaps: IntGaugeVec,
202    pub(crate) round_tracker_accepted_quorum_round_gaps: IntGaugeVec,
203    pub(crate) round_tracker_low_received_quorum_round: IntGaugeVec,
204    pub(crate) round_tracker_low_accepted_quorum_round: IntGaugeVec,
205    pub(crate) round_tracker_current_received_round_gaps: IntGaugeVec,
206    pub(crate) round_tracker_current_accepted_round_gaps: IntGaugeVec,
207    pub(crate) round_tracker_propagation_delays: Histogram,
208    pub(crate) round_tracker_last_propagation_delay: IntGauge,
209    pub(crate) round_prober_request_errors: IntCounterVec,
210    pub(crate) certifier_gc_round: IntGauge,
211    pub(crate) certifier_block_latency: HistogramVec,
212    pub(crate) certifier_own_reject_votes: IntCounterVec,
213    pub(crate) certifier_output_blocks: IntCounterVec,
214    pub(crate) certifier_rejected_transactions: IntCounterVec,
215    pub(crate) certifier_accepted_transactions: IntCounterVec,
216    pub(crate) certifier_missing_ancestor_during_certification: IntCounterVec,
217    pub(crate) finalizer_buffered_commits: IntGauge,
218    pub(crate) finalizer_round_delay: Histogram,
219    pub(crate) finalizer_transaction_status: IntCounterVec,
220    pub(crate) finalizer_reject_votes: IntCounterVec,
221    pub(crate) finalizer_output_commits: IntCounterVec,
222    pub(crate) finalizer_skipped_voting_blocks: IntCounterVec,
223    pub(crate) uptime: Histogram,
224}
225
226impl NodeMetrics {
227    pub(crate) fn new(registry: &Registry) -> Self {
228        Self {
229            authority_index: register_int_gauge_vec_with_registry!(
230                "authority_index",
231                "The index of the authority",
232                &["name"],
233                registry,
234            ).unwrap(),
235            protocol_version: register_int_gauge_with_registry!(
236                "protocol_version",
237                "The protocol version used in this epoch",
238                registry,
239            ).unwrap(),
240            block_commit_latency: register_histogram_with_registry!(
241                "block_commit_latency",
242                "The time taken between block creation and block commit.",
243                LATENCY_SEC_BUCKETS.to_vec(),
244                registry,
245            ).unwrap(),
246            proposed_blocks: register_int_counter_vec_with_registry!(
247                "proposed_blocks",
248                "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
249                &["force"],
250                registry,
251            ).unwrap(),
252            proposed_block_size: register_histogram_with_registry!(
253                "proposed_block_size",
254                "The size (in bytes) of proposed blocks",
255                SIZE_BUCKETS.to_vec(),
256                registry
257            ).unwrap(),
258            proposed_block_transactions: register_histogram_with_registry!(
259                "proposed_block_transactions",
260                "# of transactions contained in proposed blocks",
261                NUM_BUCKETS.to_vec(),
262                registry
263            ).unwrap(),
264            proposed_block_ancestors: register_histogram_with_registry!(
265                "proposed_block_ancestors",
266                "Number of ancestors in proposed blocks",
267                exponential_buckets(1.0, 1.4, 20).unwrap(),
268                registry,
269            ).unwrap(),
270            proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!(
271                "proposed_block_ancestors_timestamp_drift_ms",
272                "The drift in ms of ancestors' timestamps included in newly proposed blocks",
273                &["authority"],
274                registry,
275            ).unwrap(),
276            proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
277                "proposed_block_ancestors_depth",
278                "The depth in rounds of ancestors included in newly proposed blocks",
279                &["authority"],
280                exponential_buckets(1.0, 2.0, 14).unwrap(),
281                registry,
282            ).unwrap(),
283            highest_verified_authority_round: register_int_gauge_vec_with_registry!(
284                "highest_verified_authority_round",
285                "The highest round of verified block for the corresponding authority",
286                &["authority"],
287                registry,
288            ).unwrap(),
289            lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
290                "lowest_verified_authority_round",
291                "The lowest round of verified block for the corresponding authority",
292                &["authority"],
293                registry,
294            ).unwrap(),
295            block_proposal_interval: register_histogram_with_registry!(
296                "block_proposal_interval",
297                "Intervals (in secs) between block proposals.",
298                FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
299                registry,
300            ).unwrap(),
301            block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
302                "block_proposal_leader_wait_ms",
303                "Total time in ms spent waiting for a leader when proposing blocks.",
304                &["authority"],
305                registry,
306            ).unwrap(),
307            block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
308                "block_proposal_leader_wait_count",
309                "Total times waiting for a leader when proposing blocks.",
310                &["authority"],
311                registry,
312            ).unwrap(),
313            block_timestamp_drift_ms: register_int_counter_vec_with_registry!(
314                "block_timestamp_drift_ms",
315                "The clock drift time between a received block and the current node's time.",
316                &["authority", "source"],
317                registry,
318            ).unwrap(),
319            blocks_per_commit_count: register_histogram_with_registry!(
320                "blocks_per_commit_count",
321                "The number of blocks per commit.",
322                NUM_BUCKETS.to_vec(),
323                registry,
324            ).unwrap(),
325            blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
326                "blocks_pruned_on_commit",
327                "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
328                &["authority", "commit_status"],
329                registry,
330            ).unwrap(),
331            commit_observer_last_recovered_commit_index: register_int_gauge_with_registry!(
332                "commit_observer_last_recovered_commit_index",
333                "The last commit index recovered by the commit observer",
334                registry,
335            ).unwrap(),
336            core_add_blocks_batch_size: register_histogram_with_registry!(
337                "core_add_blocks_batch_size",
338                "The number of blocks received from Core for processing on a single batch",
339                NUM_BUCKETS.to_vec(),
340                registry,
341            ).unwrap(),
342            core_check_block_refs_batch_size: register_histogram_with_registry!(
343                "core_check_block_refs_batch_size",
344                "The number of excluded blocks received from Core for search on a single batch",
345                NUM_BUCKETS.to_vec(),
346                registry,
347            ).unwrap(),
348            core_lock_dequeued: register_int_counter_with_registry!(
349                "core_lock_dequeued",
350                "Number of dequeued core requests",
351                registry,
352            ).unwrap(),
353            core_lock_enqueued: register_int_counter_with_registry!(
354                "core_lock_enqueued",
355                "Number of enqueued core requests",
356                registry,
357            ).unwrap(),
358            core_skipped_proposals: register_int_counter_vec_with_registry!(
359                "core_skipped_proposals",
360                "Number of proposals skipped in the Core, per reason",
361                &["reason"],
362                registry,
363            ).unwrap(),
364            handler_received_block_missing_ancestors: register_int_counter_vec_with_registry!(
365                "handler_received_block_missing_ancestors",
366                "Number of missing ancestors in blocks received in the handler, separated by peer",
367                &["authority"],
368                registry,
369            ).unwrap(),
370            highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
371                "highest_accepted_authority_round",
372                "The highest round where a block has been accepted per authority. Resets on restart.",
373                &["authority"],
374                registry,
375            ).unwrap(),
376            highest_accepted_round: register_int_gauge_with_registry!(
377                "highest_accepted_round",
378                "The highest round where a block has been accepted. Resets on restart.",
379                registry,
380            ).unwrap(),
381            accepted_block_time_drift_ms: register_int_counter_vec_with_registry!(
382                "accepted_block_time_drift_ms",
383                "The time drift in ms of an accepted block compared to local time",
384                &["authority"],
385                registry,
386            ).unwrap(),
387            accepted_blocks: register_int_counter_vec_with_registry!(
388                "accepted_blocks",
389                "Number of accepted blocks by source (own, others)",
390                &["source"],
391                registry,
392            ).unwrap(),
393            dag_state_recent_blocks: register_int_gauge_with_registry!(
394                "dag_state_recent_blocks",
395                "Number of recent blocks cached in the DagState",
396                registry,
397            ).unwrap(),
398            dag_state_recent_refs: register_int_gauge_with_registry!(
399                "dag_state_recent_refs",
400                "Number of recent refs cached in the DagState",
401                registry,
402            ).unwrap(),
403            dag_state_store_read_count: register_int_counter_vec_with_registry!(
404                "dag_state_store_read_count",
405                "Number of times DagState needs to read from store per operation type",
406                &["type"],
407                registry,
408            ).unwrap(),
409            dag_state_store_write_count: register_int_counter_with_registry!(
410                "dag_state_store_write_count",
411                "Number of times DagState needs to write to store",
412                registry,
413            ).unwrap(),
414            fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
415                "fetch_blocks_scheduler_inflight",
416                "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
417                registry,
418            ).unwrap(),
419            synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
420                "synchronizer_fetched_blocks_by_peer",
421                "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
422                &["peer", "type"],
423                registry,
424            ).unwrap(),
425            synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
426                "synchronizer_missing_blocks_by_authority",
427                "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
428                &["authority"],
429                registry,
430            ).unwrap(),
431            synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
432                "synchronizer_current_missing_blocks_by_authority",
433                "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
434                &["authority"],
435                registry,
436            ).unwrap(),
437            synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
438                "synchronizer_fetched_blocks_by_authority",
439                "Number of fetched blocks per block author via the synchronizer",
440                &["authority", "type"],
441                registry,
442            ).unwrap(),
443            synchronizer_fetch_failures: register_int_counter_vec_with_registry!(
444                "synchronizer_fetch_failures",
445                "Number of fetch failures against each peer",
446                &["peer", "type"],
447                registry,
448            ).unwrap(),
449            synchronizer_skipped_fetch_requests: register_int_counter_vec_with_registry!(
450                "synchronizer_skipped_fetch_requests",
451                "Number of fetch requests skipped against each peer, because the peer is saturated",
452                &["peer"],
453                registry,
454            ).unwrap(),
455            synchronizer_process_fetched_failures: register_int_counter_vec_with_registry!(
456                "synchronizer_process_fetched_failures",
457                "Number of failures for processing fetched blocks against each peer",
458                &["peer", "type"],
459                registry,
460            ).unwrap(),
461            network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
462                "network_received_excluded_ancestors_from_authority",
463                "Number of excluded ancestors received from each authority.",
464                &["authority"],
465                registry,
466            ).unwrap(),
467            network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
468                "network_excluded_ancestors_count_by_authority",
469                "Total number of excluded ancestors per authority.",
470                &["authority"],
471                registry,
472            ).unwrap(),
473            network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
474                "network_excluded_ancestors_sent_to_fetch",
475                "Number of excluded ancestors sent to fetch.",
476                &["authority"],
477                registry,
478            ).unwrap(),
479            last_known_own_block_round: register_int_gauge_with_registry!(
480                "last_known_own_block_round",
481                "The highest round of our own block as this has been synced from peers during an amnesia recovery",
482                registry,
483            ).unwrap(),
484            sync_last_known_own_block_retries: register_int_counter_with_registry!(
485                "sync_last_known_own_block_retries",
486                "Number of times this node tried to fetch the last own block from peers",
487                registry,
488            ).unwrap(),
489            invalid_blocks: register_int_counter_vec_with_registry!(
490                "invalid_blocks",
491                "Number of invalid blocks per peer authority",
492                &["authority", "source", "error"],
493                registry,
494            ).unwrap(),
495            certifier_block_latency: register_histogram_vec_with_registry!(
496                "certifier_block_latency",
497                "The latency of a block being certified by the transaction certifier. The block's authority is the label",
498                &["authority"],
499                FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
500                registry,
501            ).unwrap(),
502            certifier_rejected_transactions: register_int_counter_vec_with_registry!(
503                "certifier_rejected_transactions",
504                "Number of transactions rejected by authority in transaction certifier",
505                &["authority"],
506                registry,
507            ).unwrap(),
508            certifier_accepted_transactions: register_int_counter_vec_with_registry!(
509                "certifier_accepted_transactions",
510                "Number of transactions accepted by authority in transaction certifier",
511                &["authority"],
512                registry,
513            ).unwrap(),
514            certifier_missing_ancestor_during_certification: register_int_counter_vec_with_registry!(
515                "certifier_missing_ancestor_during_certification",
516                "Number of missing ancestors during certification",
517                &["reason"],
518                registry,
519            ).unwrap(),
520            rejected_blocks: register_int_counter_vec_with_registry!(
521                "rejected_blocks",
522                "Number of blocks rejected before verifications",
523                &["reason"],
524                registry,
525            ).unwrap(),
526            subscribed_blocks: register_int_counter_vec_with_registry!(
527                "subscribed_blocks",
528                "Number of blocks received from each peer before verification",
529                &["authority"],
530                registry,
531            ).unwrap(),
532            verified_blocks: register_int_counter_vec_with_registry!(
533                "verified_blocks",
534                "Number of blocks received from each peer that are verified",
535                &["authority"],
536                registry,
537            ).unwrap(),
538            committed_leaders_total: register_int_counter_vec_with_registry!(
539                "committed_leaders_total",
540                "Total number of (direct or indirect) committed leaders per authority",
541                &["authority", "commit_type"],
542                registry,
543            ).unwrap(),
544            last_committed_authority_round: register_int_gauge_vec_with_registry!(
545                "last_committed_authority_round",
546                "The last round committed by authority.",
547                &["authority"],
548                registry,
549            ).unwrap(),
550            last_committed_leader_round: register_int_gauge_with_registry!(
551                "last_committed_leader_round",
552                "The last round where a leader was committed to store and sent to commit consumer.",
553                registry,
554            ).unwrap(),
555            last_commit_index: register_int_gauge_with_registry!(
556                "last_commit_index",
557                "Index of the last commit.",
558                registry,
559            ).unwrap(),
560            last_commit_time_diff: register_histogram_with_registry!(
561                "last_commit_time_diff",
562                "The time diff between the last commit and previous one.",
563                LATENCY_SEC_BUCKETS.to_vec(),
564                registry,
565            ).unwrap(),
566            commit_round_advancement_interval: register_histogram_with_registry!(
567                "commit_round_advancement_interval",
568                "Intervals (in secs) between commit round advancements.",
569                FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
570                registry,
571            ).unwrap(),
572            last_decided_leader_round: register_int_gauge_with_registry!(
573                "last_decided_leader_round",
574                "The last round where a commit decision was made.",
575                registry,
576            ).unwrap(),
577            leader_timeout_total: register_int_counter_vec_with_registry!(
578                "leader_timeout_total",
579                "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
580                &["timeout_type"],
581                registry,
582            ).unwrap(),
583            smart_selection_wait: register_int_counter_with_registry!(
584                "smart_selection_wait",
585                "Number of times we waited for smart ancestor selection.",
586                registry,
587            ).unwrap(),
588            ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
589                "ancestor_state_change_by_authority",
590                "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
591                &["authority", "state"],
592                registry,
593            ).unwrap(),
594            excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
595                "excluded_proposal_ancestors_count_by_authority",
596                "Total number of excluded ancestors per authority during proposal.",
597                &["authority"],
598                registry,
599            ).unwrap(),
600            included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
601                "included_excluded_proposal_ancestors_count_by_authority",
602                "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
603                &["authority", "type"],
604                registry,
605            ).unwrap(),
606            missing_blocks_total: register_int_counter_with_registry!(
607                "missing_blocks_total",
608                "Total cumulative number of missing blocks",
609                registry,
610            ).unwrap(),
611            missing_blocks_after_fetch_total: register_int_counter_with_registry!(
612                "missing_blocks_after_fetch_total",
613                "Total number of missing blocks after fetching blocks from peer",
614                registry,
615            ).unwrap(),
616            num_of_bad_nodes: register_int_gauge_with_registry!(
617                "num_of_bad_nodes",
618                "The number of bad nodes in the new leader schedule",
619                registry
620            ).unwrap(),
621            quorum_receive_latency: register_histogram_with_registry!(
622                "quorum_receive_latency",
623                "The time it took to receive a new round quorum of blocks",
624                registry
625            ).unwrap(),
626            block_receive_delay: register_int_counter_vec_with_registry!(
627                "block_receive_delay",
628                "Total delay from the start of the round to receiving the block, in milliseconds per authority",
629                &["authority"],
630                registry,
631            ).unwrap(),
632            reputation_scores: register_int_gauge_vec_with_registry!(
633                "reputation_scores",
634                "Reputation scores for each authority",
635                &["authority"],
636                registry,
637            ).unwrap(),
638            scope_processing_time: register_histogram_vec_with_registry!(
639                "scope_processing_time",
640                "The processing time of a specific code scope",
641                &["scope"],
642                FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
643                registry
644            ).unwrap(),
645            sub_dags_per_commit_count: register_histogram_with_registry!(
646                "sub_dags_per_commit_count",
647                "The number of subdags per commit.",
648                registry,
649            ).unwrap(),
650            block_suspensions: register_int_counter_vec_with_registry!(
651                "block_suspensions",
652                "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
653                &["authority"],
654                registry,
655            ).unwrap(),
656            block_unsuspensions: register_int_counter_vec_with_registry!(
657                "block_unsuspensions",
658                "The number of block unsuspensions.",
659                &["authority"],
660                registry,
661            ).unwrap(),
662            suspended_block_time: register_histogram_vec_with_registry!(
663                "suspended_block_time",
664                "The time for which a block remains suspended",
665                &["authority"],
666                registry,
667            ).unwrap(),
668            block_manager_suspended_blocks: register_int_gauge_with_registry!(
669                "block_manager_suspended_blocks",
670                "The number of blocks currently suspended in the block manager",
671                registry,
672            ).unwrap(),
673            block_manager_missing_ancestors: register_int_gauge_with_registry!(
674                "block_manager_missing_ancestors",
675                "The number of missing ancestors tracked in the block manager",
676                registry,
677            ).unwrap(),
678            block_manager_missing_blocks: register_int_gauge_with_registry!(
679                "block_manager_missing_blocks",
680                "The number of blocks missing content tracked in the block manager",
681                registry,
682            ).unwrap(),
683            block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
684                "block_manager_missing_blocks_by_authority",
685                "The number of new missing blocks by block authority",
686                &["authority"],
687                registry,
688            ).unwrap(),
689            block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
690                "block_manager_missing_ancestors_by_authority",
691                "The number of missing ancestors by ancestor authority across received blocks",
692                &["authority"],
693                registry,
694            ).unwrap(),
695            block_manager_gced_blocks: register_int_counter_vec_with_registry!(
696                "block_manager_gced_blocks",
697                "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
698                &["authority"],
699                registry,
700            ).unwrap(),
701            block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
702                "block_manager_gc_unsuspended_blocks",
703                "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
704                &["authority"],
705                registry,
706            ).unwrap(),
707            block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
708                "block_manager_skipped_blocks",
709                "The number of blocks skipped by the block manager due to block round being <= gc_round",
710                &["authority"],
711                registry,
712            ).unwrap(),
713            threshold_clock_round: register_int_gauge_with_registry!(
714                "threshold_clock_round",
715                "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
716                registry,
717            ).unwrap(),
718            subscriber_connection_attempts: register_int_counter_vec_with_registry!(
719                "subscriber_connection_attempts",
720                "The number of connection attempts per peer",
721                &["authority", "status"],
722                registry,
723            ).unwrap(),
724            subscribed_to: register_int_gauge_vec_with_registry!(
725                "subscribed_to",
726                "Peers that this authority subscribed to for block streams.",
727                &["authority"],
728                registry,
729            ).unwrap(),
730            subscribed_by: register_int_gauge_vec_with_registry!(
731                "subscribed_by",
732                "Peers subscribing for block streams from this authority.",
733                &["authority"],
734                registry,
735            ).unwrap(),
736            commit_sync_inflight_fetches: register_int_gauge_with_registry!(
737                "commit_sync_inflight_fetches",
738                "The number of inflight fetches in commit syncer",
739                registry,
740            ).unwrap(),
741            commit_sync_pending_fetches: register_int_gauge_with_registry!(
742                "commit_sync_pending_fetches",
743                "The number of pending fetches in commit syncer",
744                registry,
745            ).unwrap(),
746            commit_sync_fetched_commits: register_int_counter_with_registry!(
747                "commit_sync_fetched_commits",
748                "The number of commits fetched via commit syncer",
749                registry,
750            ).unwrap(),
751            commit_sync_fetched_blocks: register_int_counter_with_registry!(
752                "commit_sync_fetched_blocks",
753                "The number of blocks fetched via commit syncer",
754                registry,
755            ).unwrap(),
756            commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
757                "commit_sync_total_fetched_blocks_size",
758                "The total size in bytes of blocks fetched via commit syncer",
759                registry,
760            ).unwrap(),
761            commit_sync_quorum_index: register_int_gauge_with_registry!(
762                "commit_sync_quorum_index",
763                "The maximum commit index voted by a quorum of authorities",
764                registry,
765            ).unwrap(),
766            commit_sync_highest_synced_index: register_int_gauge_with_registry!(
767                "commit_sync_fetched_index",
768                "The max commit index among local and fetched commits",
769                registry,
770            ).unwrap(),
771            commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
772                "commit_sync_highest_fetched_index",
773                "The max commit index that has been fetched via network",
774                registry,
775            ).unwrap(),
776            commit_sync_local_index: register_int_gauge_with_registry!(
777                "commit_sync_local_index",
778                "The local commit index",
779                registry,
780            ).unwrap(),
781            commit_sync_gap_on_processing: register_int_counter_with_registry!(
782                "commit_sync_gap_on_processing",
783                "Number of instances where a gap was found in fetched commit processing",
784                registry,
785            ).unwrap(),
786            commit_sync_fetch_loop_latency: register_histogram_with_registry!(
787                "commit_sync_fetch_loop_latency",
788                "The time taken to finish fetching commits and blocks from a given range",
789                LATENCY_SEC_BUCKETS.to_vec(),
790                registry,
791            ).unwrap(),
792            commit_sync_fetch_once_latency: register_histogram_with_registry!(
793                "commit_sync_fetch_once_latency",
794                "The time taken to fetch commits and blocks once",
795                LATENCY_SEC_BUCKETS.to_vec(),
796                registry,
797            ).unwrap(),
798            commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
799                "commit_sync_fetch_once_errors",
800                "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
801                &["authority", "error"],
802                registry
803            ).unwrap(),
804            commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
805                "commit_sync_fetch_missing_blocks",
806                "Number of ancestor blocks that are missing when processing blocks via commit sync.",
807                &["authority"],
808                registry,
809            ).unwrap(),
810            commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
811                "commit_sync_fetch_commits_handler_uncertified_skipped",
812                "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
813                registry,
814            ).unwrap(),
815            round_tracker_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
816                "round_tracker_received_quorum_round_gaps",
817                "Received round gaps among peers for blocks proposed from each authority",
818                &["authority"],
819                registry
820            ).unwrap(),
821            round_tracker_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
822                "round_tracker_accepted_quorum_round_gaps",
823                "Accepted round gaps among peers for blocks proposed & accepted from each authority",
824                &["authority"],
825                registry
826            ).unwrap(),
827            round_tracker_low_received_quorum_round: register_int_gauge_vec_with_registry!(
828                "round_tracker_low_received_quorum_round",
829                "Low quorum round among peers for blocks proposed from each authority",
830                &["authority"],
831                registry
832            ).unwrap(),
833            round_tracker_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
834                "round_tracker_low_accepted_quorum_round",
835                "Low quorum round among peers for blocks proposed & accepted from each authority",
836                &["authority"],
837                registry
838            ).unwrap(),
839            round_tracker_current_received_round_gaps: register_int_gauge_vec_with_registry!(
840                "round_tracker_current_received_round_gaps",
841                "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
842                &["authority"],
843                registry
844            ).unwrap(),
845            round_tracker_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
846                "round_tracker_current_accepted_round_gaps",
847                "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
848                &["authority"],
849                registry
850            ).unwrap(),
851            round_tracker_propagation_delays: register_histogram_with_registry!(
852                "round_tracker_propagation_delays",
853                "Round gaps between the last proposed block round and the lower bound of own quorum round",
854                NUM_BUCKETS.to_vec(),
855                registry
856            ).unwrap(),
857            round_tracker_last_propagation_delay: register_int_gauge_with_registry!(
858                "round_tracker_last_propagation_delay",
859                "Most recent propagation delay observed by RoundTracker",
860                registry
861            ).unwrap(),
862            round_prober_request_errors: register_int_counter_vec_with_registry!(
863                "round_prober_request_errors",
864                "Number of errors when probing against peers per error type",
865                &["error_type"],
866                registry
867            ).unwrap(),
868            certifier_gc_round: register_int_gauge_with_registry!(
869                "certifier_gc_round",
870                "The current GC round of the certifier",
871                registry
872            ).unwrap(),
873            certifier_own_reject_votes: register_int_counter_vec_with_registry!(
874                "certifier_own_reject_votes",
875                "Number of own reject votes against each peer authority",
876                &["authority"],
877                registry
878            ).unwrap(),
879            certifier_output_blocks: register_int_counter_vec_with_registry!(
880                "certifier_output_blocks",
881                "Number of output blocks certified by the certifier, grouped by type.",
882                &["type"],
883                registry
884            ).unwrap(),
885            finalizer_buffered_commits: register_int_gauge_with_registry!(
886                "finalizer_buffered_commits",
887                "The number of commits buffered in the finalizer",
888                registry,
889            ).unwrap(),
890            finalizer_round_delay: register_histogram_with_registry!(
891                "finalizer_round_delay",
892                "The delay between the round of the last committed block and the round of the finalized commit.",
893                ROUND_DELAY_BUCKETS.to_vec(),
894                registry,
895            ).unwrap(),
896            finalizer_transaction_status: register_int_counter_vec_with_registry!(
897                "finalizer_transaction_status",
898                "Number of transactions finalized by the finalizer, grouped by status.",
899                &["status"],
900                registry
901            ).unwrap(),
902            finalizer_reject_votes: register_int_counter_vec_with_registry!(
903                "finalizer_reject_votes",
904                "Number of reject votes casted by each authority observed by the finalizer.",
905                &["authority"],
906                registry
907            ).unwrap(),
908            finalizer_output_commits: register_int_counter_vec_with_registry!(
909                "finalizer_output_commits",
910                "Number of output commits finalized by the finalizer, grouped by type.",
911                &["type"],
912                registry
913            ).unwrap(),
914            finalizer_skipped_voting_blocks: register_int_counter_vec_with_registry!(
915                "finalizer_skipped_voting_blocks",
916                "Number of blocks skipped from voting due to potentially not being an immediate descendant.",
917                &["authority"],
918                registry
919            ).unwrap(),
920            uptime: register_histogram_with_registry!(
921                "uptime",
922                "Total node uptime",
923                LATENCY_SEC_BUCKETS.to_vec(),
924                registry,
925            ).unwrap(),
926        }
927    }
928}