1use std::sync::Arc;
5
6use prometheus::{
7 Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
8 exponential_buckets, register_histogram_vec_with_registry, register_histogram_with_registry,
9 register_int_counter_vec_with_registry, register_int_counter_with_registry,
10 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
11};
12
13use crate::network::metrics::NetworkMetrics;
14
15const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
17 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
18 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
19 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
20];
21
22const NUM_BUCKETS: &[f64] = &[
23 1.0,
24 2.0,
25 4.0,
26 8.0,
27 10.0,
28 20.0,
29 40.0,
30 80.0,
31 100.0,
32 150.0,
33 200.0,
34 400.0,
35 800.0,
36 1000.0,
37 2000.0,
38 3000.0,
39 5000.0,
40 10000.0,
41 20000.0,
42 30000.0,
43 50000.0,
44 100_000.0,
45 200_000.0,
46 300_000.0,
47 500_000.0,
48 1_000_000.0,
49];
50
51const LATENCY_SEC_BUCKETS: &[f64] = &[
52 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
53 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
54 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
55];
56
57const SIZE_BUCKETS: &[f64] = &[
58 100.,
59 400.,
60 800.,
61 1_000.,
62 2_000.,
63 5_000.,
64 10_000.,
65 20_000.,
66 50_000.,
67 100_000.,
68 200_000.0,
69 300_000.0,
70 400_000.0,
71 500_000.0,
72 1_000_000.0,
73 2_000_000.0,
74 3_000_000.0,
75 5_000_000.0,
76 10_000_000.0,
77]; const ROUND_DELAY_BUCKETS: &[f64] = &[0.0, 0.5, 1.0, 2.0, 3.0, 4.0];
81
82pub struct Metrics {
83 pub(crate) node_metrics: NodeMetrics,
84 pub(crate) network_metrics: NetworkMetrics,
85}
86
87pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
88 let node_metrics = NodeMetrics::new(®istry);
89 let network_metrics = NetworkMetrics::new(®istry);
90
91 Arc::new(Metrics {
92 node_metrics,
93 network_metrics,
94 })
95}
96
97pub(crate) fn test_metrics() -> Arc<Metrics> {
98 initialise_metrics(Registry::new())
99}
100
101pub(crate) struct NodeMetrics {
102 pub(crate) authority_index: IntGaugeVec,
103 pub(crate) protocol_version: IntGauge,
104 pub(crate) block_commit_latency: Histogram,
105 pub(crate) proposed_blocks: IntCounterVec,
106 pub(crate) proposed_block_size: Histogram,
107 pub(crate) proposed_block_transactions: Histogram,
108 pub(crate) proposed_block_ancestors: Histogram,
109 pub(crate) proposed_block_ancestors_depth: HistogramVec,
110 pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec,
111 pub(crate) highest_verified_authority_round: IntGaugeVec,
112 pub(crate) lowest_verified_authority_round: IntGaugeVec,
113 pub(crate) block_proposal_interval: Histogram,
114 pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
115 pub(crate) block_proposal_leader_wait_count: IntCounterVec,
116 pub(crate) block_timestamp_drift_ms: IntCounterVec,
117 pub(crate) blocks_per_commit_count: Histogram,
118 pub(crate) blocks_pruned_on_commit: IntCounterVec,
119 pub(crate) broadcaster_rtt_estimate_ms: IntGaugeVec,
120 pub(crate) commit_observer_last_recovered_commit_index: IntGauge,
121 pub(crate) core_add_blocks_batch_size: Histogram,
122 pub(crate) core_check_block_refs_batch_size: Histogram,
123 pub(crate) core_lock_dequeued: IntCounter,
124 pub(crate) core_lock_enqueued: IntCounter,
125 pub(crate) core_skipped_proposals: IntCounterVec,
126 pub(crate) handler_received_block_missing_ancestors: IntCounterVec,
127 pub(crate) highest_accepted_authority_round: IntGaugeVec,
128 pub(crate) highest_accepted_round: IntGauge,
129 pub(crate) accepted_block_time_drift_ms: IntCounterVec,
130 pub(crate) accepted_blocks: IntCounterVec,
131 pub(crate) dag_state_recent_blocks: IntGauge,
132 pub(crate) dag_state_recent_refs: IntGauge,
133 pub(crate) dag_state_store_read_count: IntCounterVec,
134 pub(crate) dag_state_store_write_count: IntCounter,
135 pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
136 pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
137 pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
138 pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
139 pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
140 pub(crate) synchronizer_fetch_failures: IntCounterVec,
141 pub(crate) synchronizer_skipped_fetch_requests: IntCounterVec,
142 pub(crate) synchronizer_process_fetched_failures: IntCounterVec,
143 pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
144 pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
145 pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
146 pub(crate) invalid_blocks: IntCounterVec,
147 pub(crate) rejected_blocks: IntCounterVec,
148 pub(crate) subscribed_blocks: IntCounterVec,
149 pub(crate) verified_blocks: IntCounterVec,
150 pub(crate) committed_leaders_total: IntCounterVec,
151 pub(crate) last_committed_authority_round: IntGaugeVec,
152 pub(crate) last_committed_leader_round: IntGauge,
153 pub(crate) last_commit_index: IntGauge,
154 pub(crate) last_commit_time_diff: Histogram,
155 pub(crate) last_known_own_block_round: IntGauge,
156 pub(crate) sync_last_known_own_block_retries: IntCounter,
157 pub(crate) commit_round_advancement_interval: Histogram,
158 pub(crate) last_decided_leader_round: IntGauge,
159 pub(crate) leader_timeout_total: IntCounterVec,
160 pub(crate) smart_selection_wait: IntCounter,
161 pub(crate) ancestor_state_change_by_authority: IntCounterVec,
162 pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
163 pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
164 pub(crate) missing_blocks_total: IntCounter,
165 pub(crate) missing_blocks_after_fetch_total: IntCounter,
166 pub(crate) num_of_bad_nodes: IntGauge,
167 pub(crate) quorum_receive_latency: Histogram,
168 pub(crate) block_receive_delay: IntCounterVec,
169 pub(crate) reputation_scores: IntGaugeVec,
170 pub(crate) scope_processing_time: HistogramVec,
171 pub(crate) sub_dags_per_commit_count: Histogram,
172 pub(crate) block_suspensions: IntCounterVec,
173 pub(crate) block_unsuspensions: IntCounterVec,
174 pub(crate) suspended_block_time: HistogramVec,
175 pub(crate) block_manager_suspended_blocks: IntGauge,
176 pub(crate) block_manager_missing_ancestors: IntGauge,
177 pub(crate) block_manager_missing_blocks: IntGauge,
178 pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
179 pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
180 pub(crate) block_manager_gced_blocks: IntCounterVec,
181 pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
182 pub(crate) block_manager_skipped_blocks: IntCounterVec,
183 pub(crate) threshold_clock_round: IntGauge,
184 pub(crate) subscriber_connection_attempts: IntCounterVec,
185 pub(crate) subscribed_to: IntGaugeVec,
186 pub(crate) subscribed_by: IntGaugeVec,
187 pub(crate) commit_sync_inflight_fetches: IntGauge,
188 pub(crate) commit_sync_pending_fetches: IntGauge,
189 pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
190 pub(crate) commit_sync_fetched_commits: IntCounter,
191 pub(crate) commit_sync_fetched_blocks: IntCounter,
192 pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
193 pub(crate) commit_sync_quorum_index: IntGauge,
194 pub(crate) commit_sync_highest_synced_index: IntGauge,
195 pub(crate) commit_sync_highest_fetched_index: IntGauge,
196 pub(crate) commit_sync_local_index: IntGauge,
197 pub(crate) commit_sync_gap_on_processing: IntCounter,
198 pub(crate) commit_sync_fetch_loop_latency: Histogram,
199 pub(crate) commit_sync_fetch_once_latency: Histogram,
200 pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
201 pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
202 pub(crate) round_tracker_received_quorum_round_gaps: IntGaugeVec,
203 pub(crate) round_tracker_accepted_quorum_round_gaps: IntGaugeVec,
204 pub(crate) round_tracker_low_received_quorum_round: IntGaugeVec,
205 pub(crate) round_tracker_low_accepted_quorum_round: IntGaugeVec,
206 pub(crate) round_tracker_current_received_round_gaps: IntGaugeVec,
207 pub(crate) round_tracker_current_accepted_round_gaps: IntGaugeVec,
208 pub(crate) round_tracker_propagation_delays: Histogram,
209 pub(crate) round_tracker_last_propagation_delay: IntGauge,
210 pub(crate) round_prober_request_errors: IntCounterVec,
211 pub(crate) certifier_gc_round: IntGauge,
212 pub(crate) certifier_block_latency: HistogramVec,
213 pub(crate) certifier_own_reject_votes: IntCounterVec,
214 pub(crate) certifier_output_blocks: IntCounterVec,
215 pub(crate) certifier_rejected_transactions: IntCounterVec,
216 pub(crate) certifier_accepted_transactions: IntCounterVec,
217 pub(crate) certifier_missing_ancestor_during_certification: IntCounterVec,
218 pub(crate) finalizer_buffered_commits: IntGauge,
219 pub(crate) finalizer_round_delay: Histogram,
220 pub(crate) finalizer_transaction_status: IntCounterVec,
221 pub(crate) finalizer_reject_votes: IntCounterVec,
222 pub(crate) finalizer_output_commits: IntCounterVec,
223 pub(crate) uptime: Histogram,
224}
225
226impl NodeMetrics {
227 pub(crate) fn new(registry: &Registry) -> Self {
228 Self {
229 authority_index: register_int_gauge_vec_with_registry!(
230 "authority_index",
231 "The index of the authority",
232 &["name"],
233 registry,
234 ).unwrap(),
235 protocol_version: register_int_gauge_with_registry!(
236 "protocol_version",
237 "The protocol version used in this epoch",
238 registry,
239 ).unwrap(),
240 block_commit_latency: register_histogram_with_registry!(
241 "block_commit_latency",
242 "The time taken between block creation and block commit.",
243 LATENCY_SEC_BUCKETS.to_vec(),
244 registry,
245 ).unwrap(),
246 proposed_blocks: register_int_counter_vec_with_registry!(
247 "proposed_blocks",
248 "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
249 &["force"],
250 registry,
251 ).unwrap(),
252 proposed_block_size: register_histogram_with_registry!(
253 "proposed_block_size",
254 "The size (in bytes) of proposed blocks",
255 SIZE_BUCKETS.to_vec(),
256 registry
257 ).unwrap(),
258 proposed_block_transactions: register_histogram_with_registry!(
259 "proposed_block_transactions",
260 "# of transactions contained in proposed blocks",
261 NUM_BUCKETS.to_vec(),
262 registry
263 ).unwrap(),
264 proposed_block_ancestors: register_histogram_with_registry!(
265 "proposed_block_ancestors",
266 "Number of ancestors in proposed blocks",
267 exponential_buckets(1.0, 1.4, 20).unwrap(),
268 registry,
269 ).unwrap(),
270 proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!(
271 "proposed_block_ancestors_timestamp_drift_ms",
272 "The drift in ms of ancestors' timestamps included in newly proposed blocks",
273 &["authority"],
274 registry,
275 ).unwrap(),
276 proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
277 "proposed_block_ancestors_depth",
278 "The depth in rounds of ancestors included in newly proposed blocks",
279 &["authority"],
280 exponential_buckets(1.0, 2.0, 14).unwrap(),
281 registry,
282 ).unwrap(),
283 highest_verified_authority_round: register_int_gauge_vec_with_registry!(
284 "highest_verified_authority_round",
285 "The highest round of verified block for the corresponding authority",
286 &["authority"],
287 registry,
288 ).unwrap(),
289 lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
290 "lowest_verified_authority_round",
291 "The lowest round of verified block for the corresponding authority",
292 &["authority"],
293 registry,
294 ).unwrap(),
295 block_proposal_interval: register_histogram_with_registry!(
296 "block_proposal_interval",
297 "Intervals (in secs) between block proposals.",
298 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
299 registry,
300 ).unwrap(),
301 block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
302 "block_proposal_leader_wait_ms",
303 "Total time in ms spent waiting for a leader when proposing blocks.",
304 &["authority"],
305 registry,
306 ).unwrap(),
307 block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
308 "block_proposal_leader_wait_count",
309 "Total times waiting for a leader when proposing blocks.",
310 &["authority"],
311 registry,
312 ).unwrap(),
313 block_timestamp_drift_ms: register_int_counter_vec_with_registry!(
314 "block_timestamp_drift_ms",
315 "The clock drift time between a received block and the current node's time.",
316 &["authority", "source"],
317 registry,
318 ).unwrap(),
319 blocks_per_commit_count: register_histogram_with_registry!(
320 "blocks_per_commit_count",
321 "The number of blocks per commit.",
322 NUM_BUCKETS.to_vec(),
323 registry,
324 ).unwrap(),
325 blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
326 "blocks_pruned_on_commit",
327 "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
328 &["authority", "commit_status"],
329 registry,
330 ).unwrap(),
331 broadcaster_rtt_estimate_ms: register_int_gauge_vec_with_registry!(
332 "broadcaster_rtt_estimate_ms",
333 "Estimated RTT latency per peer authority, for block sending in Broadcaster",
334 &["peer"],
335 registry,
336 ).unwrap(),
337 commit_observer_last_recovered_commit_index: register_int_gauge_with_registry!(
338 "commit_observer_last_recovered_commit_index",
339 "The last commit index recovered by the commit observer",
340 registry,
341 ).unwrap(),
342 core_add_blocks_batch_size: register_histogram_with_registry!(
343 "core_add_blocks_batch_size",
344 "The number of blocks received from Core for processing on a single batch",
345 NUM_BUCKETS.to_vec(),
346 registry,
347 ).unwrap(),
348 core_check_block_refs_batch_size: register_histogram_with_registry!(
349 "core_check_block_refs_batch_size",
350 "The number of excluded blocks received from Core for search on a single batch",
351 NUM_BUCKETS.to_vec(),
352 registry,
353 ).unwrap(),
354 core_lock_dequeued: register_int_counter_with_registry!(
355 "core_lock_dequeued",
356 "Number of dequeued core requests",
357 registry,
358 ).unwrap(),
359 core_lock_enqueued: register_int_counter_with_registry!(
360 "core_lock_enqueued",
361 "Number of enqueued core requests",
362 registry,
363 ).unwrap(),
364 core_skipped_proposals: register_int_counter_vec_with_registry!(
365 "core_skipped_proposals",
366 "Number of proposals skipped in the Core, per reason",
367 &["reason"],
368 registry,
369 ).unwrap(),
370 handler_received_block_missing_ancestors: register_int_counter_vec_with_registry!(
371 "handler_received_block_missing_ancestors",
372 "Number of missing ancestors in blocks received in the handler, separated by peer",
373 &["authority"],
374 registry,
375 ).unwrap(),
376 highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
377 "highest_accepted_authority_round",
378 "The highest round where a block has been accepted per authority. Resets on restart.",
379 &["authority"],
380 registry,
381 ).unwrap(),
382 highest_accepted_round: register_int_gauge_with_registry!(
383 "highest_accepted_round",
384 "The highest round where a block has been accepted. Resets on restart.",
385 registry,
386 ).unwrap(),
387 accepted_block_time_drift_ms: register_int_counter_vec_with_registry!(
388 "accepted_block_time_drift_ms",
389 "The time drift in ms of an accepted block compared to local time",
390 &["authority"],
391 registry,
392 ).unwrap(),
393 accepted_blocks: register_int_counter_vec_with_registry!(
394 "accepted_blocks",
395 "Number of accepted blocks by source (own, others)",
396 &["source"],
397 registry,
398 ).unwrap(),
399 dag_state_recent_blocks: register_int_gauge_with_registry!(
400 "dag_state_recent_blocks",
401 "Number of recent blocks cached in the DagState",
402 registry,
403 ).unwrap(),
404 dag_state_recent_refs: register_int_gauge_with_registry!(
405 "dag_state_recent_refs",
406 "Number of recent refs cached in the DagState",
407 registry,
408 ).unwrap(),
409 dag_state_store_read_count: register_int_counter_vec_with_registry!(
410 "dag_state_store_read_count",
411 "Number of times DagState needs to read from store per operation type",
412 &["type"],
413 registry,
414 ).unwrap(),
415 dag_state_store_write_count: register_int_counter_with_registry!(
416 "dag_state_store_write_count",
417 "Number of times DagState needs to write to store",
418 registry,
419 ).unwrap(),
420 fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
421 "fetch_blocks_scheduler_inflight",
422 "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
423 registry,
424 ).unwrap(),
425 synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
426 "synchronizer_fetched_blocks_by_peer",
427 "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
428 &["peer", "type"],
429 registry,
430 ).unwrap(),
431 synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
432 "synchronizer_missing_blocks_by_authority",
433 "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
434 &["authority"],
435 registry,
436 ).unwrap(),
437 synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
438 "synchronizer_current_missing_blocks_by_authority",
439 "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
440 &["authority"],
441 registry,
442 ).unwrap(),
443 synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
444 "synchronizer_fetched_blocks_by_authority",
445 "Number of fetched blocks per block author via the synchronizer",
446 &["authority", "type"],
447 registry,
448 ).unwrap(),
449 synchronizer_fetch_failures: register_int_counter_vec_with_registry!(
450 "synchronizer_fetch_failures",
451 "Number of fetch failures against each peer",
452 &["peer", "type"],
453 registry,
454 ).unwrap(),
455 synchronizer_skipped_fetch_requests: register_int_counter_vec_with_registry!(
456 "synchronizer_skipped_fetch_requests",
457 "Number of fetch requests skipped against each peer, because the peer is saturated",
458 &["peer"],
459 registry,
460 ).unwrap(),
461 synchronizer_process_fetched_failures: register_int_counter_vec_with_registry!(
462 "synchronizer_process_fetched_failures",
463 "Number of failures for processing fetched blocks against each peer",
464 &["peer", "type"],
465 registry,
466 ).unwrap(),
467 network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
468 "network_received_excluded_ancestors_from_authority",
469 "Number of excluded ancestors received from each authority.",
470 &["authority"],
471 registry,
472 ).unwrap(),
473 network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
474 "network_excluded_ancestors_count_by_authority",
475 "Total number of excluded ancestors per authority.",
476 &["authority"],
477 registry,
478 ).unwrap(),
479 network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
480 "network_excluded_ancestors_sent_to_fetch",
481 "Number of excluded ancestors sent to fetch.",
482 &["authority"],
483 registry,
484 ).unwrap(),
485 last_known_own_block_round: register_int_gauge_with_registry!(
486 "last_known_own_block_round",
487 "The highest round of our own block as this has been synced from peers during an amnesia recovery",
488 registry,
489 ).unwrap(),
490 sync_last_known_own_block_retries: register_int_counter_with_registry!(
491 "sync_last_known_own_block_retries",
492 "Number of times this node tried to fetch the last own block from peers",
493 registry,
494 ).unwrap(),
495 invalid_blocks: register_int_counter_vec_with_registry!(
496 "invalid_blocks",
497 "Number of invalid blocks per peer authority",
498 &["authority", "source", "error"],
499 registry,
500 ).unwrap(),
501 certifier_block_latency: register_histogram_vec_with_registry!(
502 "certifier_block_latency",
503 "The latency of a block being certified by the transaction certifier. The block's authority is the label",
504 &["authority"],
505 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
506 registry,
507 ).unwrap(),
508 certifier_rejected_transactions: register_int_counter_vec_with_registry!(
509 "certifier_rejected_transactions",
510 "Number of transactions rejected by authority in transaction certifier",
511 &["authority"],
512 registry,
513 ).unwrap(),
514 certifier_accepted_transactions: register_int_counter_vec_with_registry!(
515 "certifier_accepted_transactions",
516 "Number of transactions accepted by authority in transaction certifier",
517 &["authority"],
518 registry,
519 ).unwrap(),
520 certifier_missing_ancestor_during_certification: register_int_counter_vec_with_registry!(
521 "certifier_missing_ancestor_during_certification",
522 "Number of missing ancestors during certification",
523 &["reason"],
524 registry,
525 ).unwrap(),
526 rejected_blocks: register_int_counter_vec_with_registry!(
527 "rejected_blocks",
528 "Number of blocks rejected before verifications",
529 &["reason"],
530 registry,
531 ).unwrap(),
532 subscribed_blocks: register_int_counter_vec_with_registry!(
533 "subscribed_blocks",
534 "Number of blocks received from each peer before verification",
535 &["authority"],
536 registry,
537 ).unwrap(),
538 verified_blocks: register_int_counter_vec_with_registry!(
539 "verified_blocks",
540 "Number of blocks received from each peer that are verified",
541 &["authority"],
542 registry,
543 ).unwrap(),
544 committed_leaders_total: register_int_counter_vec_with_registry!(
545 "committed_leaders_total",
546 "Total number of (direct or indirect) committed leaders per authority",
547 &["authority", "commit_type"],
548 registry,
549 ).unwrap(),
550 last_committed_authority_round: register_int_gauge_vec_with_registry!(
551 "last_committed_authority_round",
552 "The last round committed by authority.",
553 &["authority"],
554 registry,
555 ).unwrap(),
556 last_committed_leader_round: register_int_gauge_with_registry!(
557 "last_committed_leader_round",
558 "The last round where a leader was committed to store and sent to commit consumer.",
559 registry,
560 ).unwrap(),
561 last_commit_index: register_int_gauge_with_registry!(
562 "last_commit_index",
563 "Index of the last commit.",
564 registry,
565 ).unwrap(),
566 last_commit_time_diff: register_histogram_with_registry!(
567 "last_commit_time_diff",
568 "The time diff between the last commit and previous one.",
569 LATENCY_SEC_BUCKETS.to_vec(),
570 registry,
571 ).unwrap(),
572 commit_round_advancement_interval: register_histogram_with_registry!(
573 "commit_round_advancement_interval",
574 "Intervals (in secs) between commit round advancements.",
575 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
576 registry,
577 ).unwrap(),
578 last_decided_leader_round: register_int_gauge_with_registry!(
579 "last_decided_leader_round",
580 "The last round where a commit decision was made.",
581 registry,
582 ).unwrap(),
583 leader_timeout_total: register_int_counter_vec_with_registry!(
584 "leader_timeout_total",
585 "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
586 &["timeout_type"],
587 registry,
588 ).unwrap(),
589 smart_selection_wait: register_int_counter_with_registry!(
590 "smart_selection_wait",
591 "Number of times we waited for smart ancestor selection.",
592 registry,
593 ).unwrap(),
594 ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
595 "ancestor_state_change_by_authority",
596 "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
597 &["authority", "state"],
598 registry,
599 ).unwrap(),
600 excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
601 "excluded_proposal_ancestors_count_by_authority",
602 "Total number of excluded ancestors per authority during proposal.",
603 &["authority"],
604 registry,
605 ).unwrap(),
606 included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
607 "included_excluded_proposal_ancestors_count_by_authority",
608 "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
609 &["authority", "type"],
610 registry,
611 ).unwrap(),
612 missing_blocks_total: register_int_counter_with_registry!(
613 "missing_blocks_total",
614 "Total cumulative number of missing blocks",
615 registry,
616 ).unwrap(),
617 missing_blocks_after_fetch_total: register_int_counter_with_registry!(
618 "missing_blocks_after_fetch_total",
619 "Total number of missing blocks after fetching blocks from peer",
620 registry,
621 ).unwrap(),
622 num_of_bad_nodes: register_int_gauge_with_registry!(
623 "num_of_bad_nodes",
624 "The number of bad nodes in the new leader schedule",
625 registry
626 ).unwrap(),
627 quorum_receive_latency: register_histogram_with_registry!(
628 "quorum_receive_latency",
629 "The time it took to receive a new round quorum of blocks",
630 registry
631 ).unwrap(),
632 block_receive_delay: register_int_counter_vec_with_registry!(
633 "block_receive_delay",
634 "Total delay from the start of the round to receiving the block, in milliseconds per authority",
635 &["authority"],
636 registry,
637 ).unwrap(),
638 reputation_scores: register_int_gauge_vec_with_registry!(
639 "reputation_scores",
640 "Reputation scores for each authority",
641 &["authority"],
642 registry,
643 ).unwrap(),
644 scope_processing_time: register_histogram_vec_with_registry!(
645 "scope_processing_time",
646 "The processing time of a specific code scope",
647 &["scope"],
648 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
649 registry
650 ).unwrap(),
651 sub_dags_per_commit_count: register_histogram_with_registry!(
652 "sub_dags_per_commit_count",
653 "The number of subdags per commit.",
654 registry,
655 ).unwrap(),
656 block_suspensions: register_int_counter_vec_with_registry!(
657 "block_suspensions",
658 "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
659 &["authority"],
660 registry,
661 ).unwrap(),
662 block_unsuspensions: register_int_counter_vec_with_registry!(
663 "block_unsuspensions",
664 "The number of block unsuspensions.",
665 &["authority"],
666 registry,
667 ).unwrap(),
668 suspended_block_time: register_histogram_vec_with_registry!(
669 "suspended_block_time",
670 "The time for which a block remains suspended",
671 &["authority"],
672 registry,
673 ).unwrap(),
674 block_manager_suspended_blocks: register_int_gauge_with_registry!(
675 "block_manager_suspended_blocks",
676 "The number of blocks currently suspended in the block manager",
677 registry,
678 ).unwrap(),
679 block_manager_missing_ancestors: register_int_gauge_with_registry!(
680 "block_manager_missing_ancestors",
681 "The number of missing ancestors tracked in the block manager",
682 registry,
683 ).unwrap(),
684 block_manager_missing_blocks: register_int_gauge_with_registry!(
685 "block_manager_missing_blocks",
686 "The number of blocks missing content tracked in the block manager",
687 registry,
688 ).unwrap(),
689 block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
690 "block_manager_missing_blocks_by_authority",
691 "The number of new missing blocks by block authority",
692 &["authority"],
693 registry,
694 ).unwrap(),
695 block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
696 "block_manager_missing_ancestors_by_authority",
697 "The number of missing ancestors by ancestor authority across received blocks",
698 &["authority"],
699 registry,
700 ).unwrap(),
701 block_manager_gced_blocks: register_int_counter_vec_with_registry!(
702 "block_manager_gced_blocks",
703 "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
704 &["authority"],
705 registry,
706 ).unwrap(),
707 block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
708 "block_manager_gc_unsuspended_blocks",
709 "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
710 &["authority"],
711 registry,
712 ).unwrap(),
713 block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
714 "block_manager_skipped_blocks",
715 "The number of blocks skipped by the block manager due to block round being <= gc_round",
716 &["authority"],
717 registry,
718 ).unwrap(),
719 threshold_clock_round: register_int_gauge_with_registry!(
720 "threshold_clock_round",
721 "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
722 registry,
723 ).unwrap(),
724 subscriber_connection_attempts: register_int_counter_vec_with_registry!(
725 "subscriber_connection_attempts",
726 "The number of connection attempts per peer",
727 &["authority", "status"],
728 registry,
729 ).unwrap(),
730 subscribed_to: register_int_gauge_vec_with_registry!(
731 "subscribed_to",
732 "Peers that this authority subscribed to for block streams.",
733 &["authority"],
734 registry,
735 ).unwrap(),
736 subscribed_by: register_int_gauge_vec_with_registry!(
737 "subscribed_by",
738 "Peers subscribing for block streams from this authority.",
739 &["authority"],
740 registry,
741 ).unwrap(),
742 commit_sync_inflight_fetches: register_int_gauge_with_registry!(
743 "commit_sync_inflight_fetches",
744 "The number of inflight fetches in commit syncer",
745 registry,
746 ).unwrap(),
747 commit_sync_pending_fetches: register_int_gauge_with_registry!(
748 "commit_sync_pending_fetches",
749 "The number of pending fetches in commit syncer",
750 registry,
751 ).unwrap(),
752 commit_sync_fetched_commits: register_int_counter_with_registry!(
753 "commit_sync_fetched_commits",
754 "The number of commits fetched via commit syncer",
755 registry,
756 ).unwrap(),
757 commit_sync_fetched_blocks: register_int_counter_with_registry!(
758 "commit_sync_fetched_blocks",
759 "The number of blocks fetched via commit syncer",
760 registry,
761 ).unwrap(),
762 commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
763 "commit_sync_total_fetched_blocks_size",
764 "The total size in bytes of blocks fetched via commit syncer",
765 registry,
766 ).unwrap(),
767 commit_sync_quorum_index: register_int_gauge_with_registry!(
768 "commit_sync_quorum_index",
769 "The maximum commit index voted by a quorum of authorities",
770 registry,
771 ).unwrap(),
772 commit_sync_highest_synced_index: register_int_gauge_with_registry!(
773 "commit_sync_fetched_index",
774 "The max commit index among local and fetched commits",
775 registry,
776 ).unwrap(),
777 commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
778 "commit_sync_highest_fetched_index",
779 "The max commit index that has been fetched via network",
780 registry,
781 ).unwrap(),
782 commit_sync_local_index: register_int_gauge_with_registry!(
783 "commit_sync_local_index",
784 "The local commit index",
785 registry,
786 ).unwrap(),
787 commit_sync_gap_on_processing: register_int_counter_with_registry!(
788 "commit_sync_gap_on_processing",
789 "Number of instances where a gap was found in fetched commit processing",
790 registry,
791 ).unwrap(),
792 commit_sync_fetch_loop_latency: register_histogram_with_registry!(
793 "commit_sync_fetch_loop_latency",
794 "The time taken to finish fetching commits and blocks from a given range",
795 LATENCY_SEC_BUCKETS.to_vec(),
796 registry,
797 ).unwrap(),
798 commit_sync_fetch_once_latency: register_histogram_with_registry!(
799 "commit_sync_fetch_once_latency",
800 "The time taken to fetch commits and blocks once",
801 LATENCY_SEC_BUCKETS.to_vec(),
802 registry,
803 ).unwrap(),
804 commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
805 "commit_sync_fetch_once_errors",
806 "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
807 &["authority", "error"],
808 registry
809 ).unwrap(),
810 commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
811 "commit_sync_fetch_missing_blocks",
812 "Number of ancestor blocks that are missing when processing blocks via commit sync.",
813 &["authority"],
814 registry,
815 ).unwrap(),
816 commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
817 "commit_sync_fetch_commits_handler_uncertified_skipped",
818 "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
819 registry,
820 ).unwrap(),
821 round_tracker_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
822 "round_tracker_received_quorum_round_gaps",
823 "Received round gaps among peers for blocks proposed from each authority",
824 &["authority"],
825 registry
826 ).unwrap(),
827 round_tracker_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
828 "round_tracker_accepted_quorum_round_gaps",
829 "Accepted round gaps among peers for blocks proposed & accepted from each authority",
830 &["authority"],
831 registry
832 ).unwrap(),
833 round_tracker_low_received_quorum_round: register_int_gauge_vec_with_registry!(
834 "round_tracker_low_received_quorum_round",
835 "Low quorum round among peers for blocks proposed from each authority",
836 &["authority"],
837 registry
838 ).unwrap(),
839 round_tracker_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
840 "round_tracker_low_accepted_quorum_round",
841 "Low quorum round among peers for blocks proposed & accepted from each authority",
842 &["authority"],
843 registry
844 ).unwrap(),
845 round_tracker_current_received_round_gaps: register_int_gauge_vec_with_registry!(
846 "round_tracker_current_received_round_gaps",
847 "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
848 &["authority"],
849 registry
850 ).unwrap(),
851 round_tracker_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
852 "round_tracker_current_accepted_round_gaps",
853 "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
854 &["authority"],
855 registry
856 ).unwrap(),
857 round_tracker_propagation_delays: register_histogram_with_registry!(
858 "round_tracker_propagation_delays",
859 "Round gaps between the last proposed block round and the lower bound of own quorum round",
860 NUM_BUCKETS.to_vec(),
861 registry
862 ).unwrap(),
863 round_tracker_last_propagation_delay: register_int_gauge_with_registry!(
864 "round_tracker_last_propagation_delay",
865 "Most recent propagation delay observed by RoundTracker",
866 registry
867 ).unwrap(),
868 round_prober_request_errors: register_int_counter_vec_with_registry!(
869 "round_prober_request_errors",
870 "Number of errors when probing against peers per error type",
871 &["error_type"],
872 registry
873 ).unwrap(),
874 certifier_gc_round: register_int_gauge_with_registry!(
875 "certifier_gc_round",
876 "The current GC round of the certifier",
877 registry
878 ).unwrap(),
879 certifier_own_reject_votes: register_int_counter_vec_with_registry!(
880 "certifier_own_reject_votes",
881 "Number of own reject votes against each peer authority",
882 &["authority"],
883 registry
884 ).unwrap(),
885 certifier_output_blocks: register_int_counter_vec_with_registry!(
886 "certifier_output_blocks",
887 "Number of output blocks certified by the certifier, grouped by type.",
888 &["type"],
889 registry
890 ).unwrap(),
891 finalizer_buffered_commits: register_int_gauge_with_registry!(
892 "finalizer_buffered_commits",
893 "The number of commits buffered in the finalizer",
894 registry,
895 ).unwrap(),
896 finalizer_round_delay: register_histogram_with_registry!(
897 "finalizer_round_delay",
898 "The delay between the round of the last committed block and the round of the finalized commit.",
899 ROUND_DELAY_BUCKETS.to_vec(),
900 registry,
901 ).unwrap(),
902 finalizer_transaction_status: register_int_counter_vec_with_registry!(
903 "finalizer_transaction_status",
904 "Number of transactions finalized by the finalizer, grouped by status.",
905 &["status"],
906 registry
907 ).unwrap(),
908 finalizer_reject_votes: register_int_counter_vec_with_registry!(
909 "finalizer_reject_votes",
910 "Number of reject votes casted by each authority observed by the finalizer.",
911 &["authority"],
912 registry
913 ).unwrap(),
914 finalizer_output_commits: register_int_counter_vec_with_registry!(
915 "finalizer_output_commits",
916 "Number of output commits finalized by the finalizer, grouped by type.",
917 &["type"],
918 registry
919 ).unwrap(),
920 uptime: register_histogram_with_registry!(
921 "uptime",
922 "Total node uptime",
923 LATENCY_SEC_BUCKETS.to_vec(),
924 registry,
925 ).unwrap(),
926 }
927 }
928}