1use std::sync::Arc;
5
6use prometheus::{
7 Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
8 exponential_buckets, register_histogram_vec_with_registry, register_histogram_with_registry,
9 register_int_counter_vec_with_registry, register_int_counter_with_registry,
10 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
11};
12
13use crate::network::metrics::NetworkMetrics;
14
15const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
17 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
18 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
19 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
20];
21
22const NUM_BUCKETS: &[f64] = &[
23 1.0,
24 2.0,
25 4.0,
26 8.0,
27 10.0,
28 20.0,
29 40.0,
30 80.0,
31 100.0,
32 150.0,
33 200.0,
34 400.0,
35 800.0,
36 1000.0,
37 2000.0,
38 3000.0,
39 5000.0,
40 10000.0,
41 20000.0,
42 30000.0,
43 50000.0,
44 100_000.0,
45 200_000.0,
46 300_000.0,
47 500_000.0,
48 1_000_000.0,
49];
50
51const LATENCY_SEC_BUCKETS: &[f64] = &[
52 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
53 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
54 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
55];
56
57const SIZE_BUCKETS: &[f64] = &[
58 100.,
59 400.,
60 800.,
61 1_000.,
62 2_000.,
63 5_000.,
64 10_000.,
65 20_000.,
66 50_000.,
67 100_000.,
68 200_000.0,
69 300_000.0,
70 400_000.0,
71 500_000.0,
72 1_000_000.0,
73 2_000_000.0,
74 3_000_000.0,
75 5_000_000.0,
76 10_000_000.0,
77]; const ROUND_DELAY_BUCKETS: &[f64] = &[0.0, 0.5, 1.0, 2.0, 3.0, 4.0];
81
82pub struct Metrics {
83 pub(crate) node_metrics: NodeMetrics,
84 pub(crate) network_metrics: NetworkMetrics,
85}
86
87pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
88 let node_metrics = NodeMetrics::new(®istry);
89 let network_metrics = NetworkMetrics::new(®istry);
90
91 Arc::new(Metrics {
92 node_metrics,
93 network_metrics,
94 })
95}
96
97pub(crate) fn test_metrics() -> Arc<Metrics> {
98 initialise_metrics(Registry::new())
99}
100
101pub(crate) struct NodeMetrics {
102 pub(crate) authority_index: IntGaugeVec,
103 pub(crate) protocol_version: IntGauge,
104 pub(crate) block_commit_latency: Histogram,
105 pub(crate) proposed_blocks: IntCounterVec,
106 pub(crate) proposed_block_size: Histogram,
107 pub(crate) proposed_block_transactions: Histogram,
108 pub(crate) proposed_block_ancestors: Histogram,
109 pub(crate) proposed_block_ancestors_depth: HistogramVec,
110 pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec,
111 pub(crate) highest_verified_authority_round: IntGaugeVec,
112 pub(crate) lowest_verified_authority_round: IntGaugeVec,
113 pub(crate) block_proposal_interval: Histogram,
114 pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
115 pub(crate) block_proposal_leader_wait_count: IntCounterVec,
116 pub(crate) block_timestamp_drift_ms: IntCounterVec,
117 pub(crate) blocks_per_commit_count: Histogram,
118 pub(crate) blocks_pruned_on_commit: IntCounterVec,
119 pub(crate) commit_observer_last_recovered_commit_index: IntGauge,
120 pub(crate) core_add_blocks_batch_size: Histogram,
121 pub(crate) core_check_block_refs_batch_size: Histogram,
122 pub(crate) core_lock_dequeued: IntCounter,
123 pub(crate) core_lock_enqueued: IntCounter,
124 pub(crate) core_skipped_proposals: IntCounterVec,
125 pub(crate) handler_received_block_missing_ancestors: IntCounterVec,
126 pub(crate) highest_accepted_authority_round: IntGaugeVec,
127 pub(crate) highest_accepted_round: IntGauge,
128 pub(crate) accepted_block_time_drift_ms: IntCounterVec,
129 pub(crate) accepted_blocks: IntCounterVec,
130 pub(crate) dag_state_recent_blocks: IntGauge,
131 pub(crate) dag_state_recent_refs: IntGauge,
132 pub(crate) dag_state_store_read_count: IntCounterVec,
133 pub(crate) dag_state_store_write_count: IntCounter,
134 pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
135 pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
136 pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
137 pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
138 pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
139 pub(crate) synchronizer_fetch_failures: IntCounterVec,
140 pub(crate) synchronizer_skipped_fetch_requests: IntCounterVec,
141 pub(crate) synchronizer_process_fetched_failures: IntCounterVec,
142 pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
143 pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
144 pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
145 pub(crate) invalid_blocks: IntCounterVec,
146 pub(crate) rejected_blocks: IntCounterVec,
147 pub(crate) subscribed_blocks: IntCounterVec,
148 pub(crate) verified_blocks: IntCounterVec,
149 pub(crate) committed_leaders_total: IntCounterVec,
150 pub(crate) last_committed_authority_round: IntGaugeVec,
151 pub(crate) last_committed_leader_round: IntGauge,
152 pub(crate) last_commit_index: IntGauge,
153 pub(crate) last_commit_time_diff: Histogram,
154 pub(crate) last_known_own_block_round: IntGauge,
155 pub(crate) sync_last_known_own_block_retries: IntCounter,
156 pub(crate) commit_round_advancement_interval: Histogram,
157 pub(crate) last_decided_leader_round: IntGauge,
158 pub(crate) leader_timeout_total: IntCounterVec,
159 pub(crate) smart_selection_wait: IntCounter,
160 pub(crate) ancestor_state_change_by_authority: IntCounterVec,
161 pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
162 pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
163 pub(crate) missing_blocks_total: IntCounter,
164 pub(crate) missing_blocks_after_fetch_total: IntCounter,
165 pub(crate) num_of_bad_nodes: IntGauge,
166 pub(crate) quorum_receive_latency: Histogram,
167 pub(crate) block_receive_delay: IntCounterVec,
168 pub(crate) reputation_scores: IntGaugeVec,
169 pub(crate) scope_processing_time: HistogramVec,
170 pub(crate) sub_dags_per_commit_count: Histogram,
171 pub(crate) block_suspensions: IntCounterVec,
172 pub(crate) block_unsuspensions: IntCounterVec,
173 pub(crate) suspended_block_time: HistogramVec,
174 pub(crate) block_manager_suspended_blocks: IntGauge,
175 pub(crate) block_manager_missing_ancestors: IntGauge,
176 pub(crate) block_manager_missing_blocks: IntGauge,
177 pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
178 pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
179 pub(crate) block_manager_gced_blocks: IntCounterVec,
180 pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
181 pub(crate) block_manager_skipped_blocks: IntCounterVec,
182 pub(crate) threshold_clock_round: IntGauge,
183 pub(crate) subscriber_connection_attempts: IntCounterVec,
184 pub(crate) subscribed_to: IntGaugeVec,
185 pub(crate) subscribed_by: IntGaugeVec,
186 pub(crate) commit_sync_inflight_fetches: IntGauge,
187 pub(crate) commit_sync_pending_fetches: IntGauge,
188 pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
189 pub(crate) commit_sync_fetched_commits: IntCounter,
190 pub(crate) commit_sync_fetched_blocks: IntCounter,
191 pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
192 pub(crate) commit_sync_quorum_index: IntGauge,
193 pub(crate) commit_sync_highest_synced_index: IntGauge,
194 pub(crate) commit_sync_highest_fetched_index: IntGauge,
195 pub(crate) commit_sync_local_index: IntGauge,
196 pub(crate) commit_sync_gap_on_processing: IntCounter,
197 pub(crate) commit_sync_fetch_loop_latency: Histogram,
198 pub(crate) commit_sync_fetch_once_latency: Histogram,
199 pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
200 pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
201 pub(crate) round_tracker_received_quorum_round_gaps: IntGaugeVec,
202 pub(crate) round_tracker_accepted_quorum_round_gaps: IntGaugeVec,
203 pub(crate) round_tracker_low_received_quorum_round: IntGaugeVec,
204 pub(crate) round_tracker_low_accepted_quorum_round: IntGaugeVec,
205 pub(crate) round_tracker_current_received_round_gaps: IntGaugeVec,
206 pub(crate) round_tracker_current_accepted_round_gaps: IntGaugeVec,
207 pub(crate) round_tracker_propagation_delays: Histogram,
208 pub(crate) round_tracker_last_propagation_delay: IntGauge,
209 pub(crate) round_prober_request_errors: IntCounterVec,
210 pub(crate) certifier_gc_round: IntGauge,
211 pub(crate) certifier_block_latency: HistogramVec,
212 pub(crate) certifier_own_reject_votes: IntCounterVec,
213 pub(crate) certifier_output_blocks: IntCounterVec,
214 pub(crate) certifier_rejected_transactions: IntCounterVec,
215 pub(crate) certifier_accepted_transactions: IntCounterVec,
216 pub(crate) certifier_missing_ancestor_during_certification: IntCounterVec,
217 pub(crate) finalizer_buffered_commits: IntGauge,
218 pub(crate) finalizer_round_delay: Histogram,
219 pub(crate) finalizer_transaction_status: IntCounterVec,
220 pub(crate) finalizer_reject_votes: IntCounterVec,
221 pub(crate) finalizer_output_commits: IntCounterVec,
222 pub(crate) finalizer_skipped_voting_blocks: IntCounterVec,
223 pub(crate) uptime: Histogram,
224}
225
226impl NodeMetrics {
227 pub(crate) fn new(registry: &Registry) -> Self {
228 Self {
229 authority_index: register_int_gauge_vec_with_registry!(
230 "authority_index",
231 "The index of the authority",
232 &["name"],
233 registry,
234 ).unwrap(),
235 protocol_version: register_int_gauge_with_registry!(
236 "protocol_version",
237 "The protocol version used in this epoch",
238 registry,
239 ).unwrap(),
240 block_commit_latency: register_histogram_with_registry!(
241 "block_commit_latency",
242 "The time taken between block creation and block commit.",
243 LATENCY_SEC_BUCKETS.to_vec(),
244 registry,
245 ).unwrap(),
246 proposed_blocks: register_int_counter_vec_with_registry!(
247 "proposed_blocks",
248 "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
249 &["force"],
250 registry,
251 ).unwrap(),
252 proposed_block_size: register_histogram_with_registry!(
253 "proposed_block_size",
254 "The size (in bytes) of proposed blocks",
255 SIZE_BUCKETS.to_vec(),
256 registry
257 ).unwrap(),
258 proposed_block_transactions: register_histogram_with_registry!(
259 "proposed_block_transactions",
260 "# of transactions contained in proposed blocks",
261 NUM_BUCKETS.to_vec(),
262 registry
263 ).unwrap(),
264 proposed_block_ancestors: register_histogram_with_registry!(
265 "proposed_block_ancestors",
266 "Number of ancestors in proposed blocks",
267 exponential_buckets(1.0, 1.4, 20).unwrap(),
268 registry,
269 ).unwrap(),
270 proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!(
271 "proposed_block_ancestors_timestamp_drift_ms",
272 "The drift in ms of ancestors' timestamps included in newly proposed blocks",
273 &["authority"],
274 registry,
275 ).unwrap(),
276 proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
277 "proposed_block_ancestors_depth",
278 "The depth in rounds of ancestors included in newly proposed blocks",
279 &["authority"],
280 exponential_buckets(1.0, 2.0, 14).unwrap(),
281 registry,
282 ).unwrap(),
283 highest_verified_authority_round: register_int_gauge_vec_with_registry!(
284 "highest_verified_authority_round",
285 "The highest round of verified block for the corresponding authority",
286 &["authority"],
287 registry,
288 ).unwrap(),
289 lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
290 "lowest_verified_authority_round",
291 "The lowest round of verified block for the corresponding authority",
292 &["authority"],
293 registry,
294 ).unwrap(),
295 block_proposal_interval: register_histogram_with_registry!(
296 "block_proposal_interval",
297 "Intervals (in secs) between block proposals.",
298 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
299 registry,
300 ).unwrap(),
301 block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
302 "block_proposal_leader_wait_ms",
303 "Total time in ms spent waiting for a leader when proposing blocks.",
304 &["authority"],
305 registry,
306 ).unwrap(),
307 block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
308 "block_proposal_leader_wait_count",
309 "Total times waiting for a leader when proposing blocks.",
310 &["authority"],
311 registry,
312 ).unwrap(),
313 block_timestamp_drift_ms: register_int_counter_vec_with_registry!(
314 "block_timestamp_drift_ms",
315 "The clock drift time between a received block and the current node's time.",
316 &["authority", "source"],
317 registry,
318 ).unwrap(),
319 blocks_per_commit_count: register_histogram_with_registry!(
320 "blocks_per_commit_count",
321 "The number of blocks per commit.",
322 NUM_BUCKETS.to_vec(),
323 registry,
324 ).unwrap(),
325 blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
326 "blocks_pruned_on_commit",
327 "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
328 &["authority", "commit_status"],
329 registry,
330 ).unwrap(),
331 commit_observer_last_recovered_commit_index: register_int_gauge_with_registry!(
332 "commit_observer_last_recovered_commit_index",
333 "The last commit index recovered by the commit observer",
334 registry,
335 ).unwrap(),
336 core_add_blocks_batch_size: register_histogram_with_registry!(
337 "core_add_blocks_batch_size",
338 "The number of blocks received from Core for processing on a single batch",
339 NUM_BUCKETS.to_vec(),
340 registry,
341 ).unwrap(),
342 core_check_block_refs_batch_size: register_histogram_with_registry!(
343 "core_check_block_refs_batch_size",
344 "The number of excluded blocks received from Core for search on a single batch",
345 NUM_BUCKETS.to_vec(),
346 registry,
347 ).unwrap(),
348 core_lock_dequeued: register_int_counter_with_registry!(
349 "core_lock_dequeued",
350 "Number of dequeued core requests",
351 registry,
352 ).unwrap(),
353 core_lock_enqueued: register_int_counter_with_registry!(
354 "core_lock_enqueued",
355 "Number of enqueued core requests",
356 registry,
357 ).unwrap(),
358 core_skipped_proposals: register_int_counter_vec_with_registry!(
359 "core_skipped_proposals",
360 "Number of proposals skipped in the Core, per reason",
361 &["reason"],
362 registry,
363 ).unwrap(),
364 handler_received_block_missing_ancestors: register_int_counter_vec_with_registry!(
365 "handler_received_block_missing_ancestors",
366 "Number of missing ancestors in blocks received in the handler, separated by peer",
367 &["authority"],
368 registry,
369 ).unwrap(),
370 highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
371 "highest_accepted_authority_round",
372 "The highest round where a block has been accepted per authority. Resets on restart.",
373 &["authority"],
374 registry,
375 ).unwrap(),
376 highest_accepted_round: register_int_gauge_with_registry!(
377 "highest_accepted_round",
378 "The highest round where a block has been accepted. Resets on restart.",
379 registry,
380 ).unwrap(),
381 accepted_block_time_drift_ms: register_int_counter_vec_with_registry!(
382 "accepted_block_time_drift_ms",
383 "The time drift in ms of an accepted block compared to local time",
384 &["authority"],
385 registry,
386 ).unwrap(),
387 accepted_blocks: register_int_counter_vec_with_registry!(
388 "accepted_blocks",
389 "Number of accepted blocks by source (own, others)",
390 &["source"],
391 registry,
392 ).unwrap(),
393 dag_state_recent_blocks: register_int_gauge_with_registry!(
394 "dag_state_recent_blocks",
395 "Number of recent blocks cached in the DagState",
396 registry,
397 ).unwrap(),
398 dag_state_recent_refs: register_int_gauge_with_registry!(
399 "dag_state_recent_refs",
400 "Number of recent refs cached in the DagState",
401 registry,
402 ).unwrap(),
403 dag_state_store_read_count: register_int_counter_vec_with_registry!(
404 "dag_state_store_read_count",
405 "Number of times DagState needs to read from store per operation type",
406 &["type"],
407 registry,
408 ).unwrap(),
409 dag_state_store_write_count: register_int_counter_with_registry!(
410 "dag_state_store_write_count",
411 "Number of times DagState needs to write to store",
412 registry,
413 ).unwrap(),
414 fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
415 "fetch_blocks_scheduler_inflight",
416 "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
417 registry,
418 ).unwrap(),
419 synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
420 "synchronizer_fetched_blocks_by_peer",
421 "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
422 &["peer", "type"],
423 registry,
424 ).unwrap(),
425 synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
426 "synchronizer_missing_blocks_by_authority",
427 "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
428 &["authority"],
429 registry,
430 ).unwrap(),
431 synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
432 "synchronizer_current_missing_blocks_by_authority",
433 "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
434 &["authority"],
435 registry,
436 ).unwrap(),
437 synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
438 "synchronizer_fetched_blocks_by_authority",
439 "Number of fetched blocks per block author via the synchronizer",
440 &["authority", "type"],
441 registry,
442 ).unwrap(),
443 synchronizer_fetch_failures: register_int_counter_vec_with_registry!(
444 "synchronizer_fetch_failures",
445 "Number of fetch failures against each peer",
446 &["peer", "type"],
447 registry,
448 ).unwrap(),
449 synchronizer_skipped_fetch_requests: register_int_counter_vec_with_registry!(
450 "synchronizer_skipped_fetch_requests",
451 "Number of fetch requests skipped against each peer, because the peer is saturated",
452 &["peer"],
453 registry,
454 ).unwrap(),
455 synchronizer_process_fetched_failures: register_int_counter_vec_with_registry!(
456 "synchronizer_process_fetched_failures",
457 "Number of failures for processing fetched blocks against each peer",
458 &["peer", "type"],
459 registry,
460 ).unwrap(),
461 network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
462 "network_received_excluded_ancestors_from_authority",
463 "Number of excluded ancestors received from each authority.",
464 &["authority"],
465 registry,
466 ).unwrap(),
467 network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
468 "network_excluded_ancestors_count_by_authority",
469 "Total number of excluded ancestors per authority.",
470 &["authority"],
471 registry,
472 ).unwrap(),
473 network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
474 "network_excluded_ancestors_sent_to_fetch",
475 "Number of excluded ancestors sent to fetch.",
476 &["authority"],
477 registry,
478 ).unwrap(),
479 last_known_own_block_round: register_int_gauge_with_registry!(
480 "last_known_own_block_round",
481 "The highest round of our own block as this has been synced from peers during an amnesia recovery",
482 registry,
483 ).unwrap(),
484 sync_last_known_own_block_retries: register_int_counter_with_registry!(
485 "sync_last_known_own_block_retries",
486 "Number of times this node tried to fetch the last own block from peers",
487 registry,
488 ).unwrap(),
489 invalid_blocks: register_int_counter_vec_with_registry!(
490 "invalid_blocks",
491 "Number of invalid blocks per peer authority",
492 &["authority", "source", "error"],
493 registry,
494 ).unwrap(),
495 certifier_block_latency: register_histogram_vec_with_registry!(
496 "certifier_block_latency",
497 "The latency of a block being certified by the transaction certifier. The block's authority is the label",
498 &["authority"],
499 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
500 registry,
501 ).unwrap(),
502 certifier_rejected_transactions: register_int_counter_vec_with_registry!(
503 "certifier_rejected_transactions",
504 "Number of transactions rejected by authority in transaction certifier",
505 &["authority"],
506 registry,
507 ).unwrap(),
508 certifier_accepted_transactions: register_int_counter_vec_with_registry!(
509 "certifier_accepted_transactions",
510 "Number of transactions accepted by authority in transaction certifier",
511 &["authority"],
512 registry,
513 ).unwrap(),
514 certifier_missing_ancestor_during_certification: register_int_counter_vec_with_registry!(
515 "certifier_missing_ancestor_during_certification",
516 "Number of missing ancestors during certification",
517 &["reason"],
518 registry,
519 ).unwrap(),
520 rejected_blocks: register_int_counter_vec_with_registry!(
521 "rejected_blocks",
522 "Number of blocks rejected before verifications",
523 &["reason"],
524 registry,
525 ).unwrap(),
526 subscribed_blocks: register_int_counter_vec_with_registry!(
527 "subscribed_blocks",
528 "Number of blocks received from each peer before verification",
529 &["authority"],
530 registry,
531 ).unwrap(),
532 verified_blocks: register_int_counter_vec_with_registry!(
533 "verified_blocks",
534 "Number of blocks received from each peer that are verified",
535 &["authority"],
536 registry,
537 ).unwrap(),
538 committed_leaders_total: register_int_counter_vec_with_registry!(
539 "committed_leaders_total",
540 "Total number of (direct or indirect) committed leaders per authority",
541 &["authority", "commit_type"],
542 registry,
543 ).unwrap(),
544 last_committed_authority_round: register_int_gauge_vec_with_registry!(
545 "last_committed_authority_round",
546 "The last round committed by authority.",
547 &["authority"],
548 registry,
549 ).unwrap(),
550 last_committed_leader_round: register_int_gauge_with_registry!(
551 "last_committed_leader_round",
552 "The last round where a leader was committed to store and sent to commit consumer.",
553 registry,
554 ).unwrap(),
555 last_commit_index: register_int_gauge_with_registry!(
556 "last_commit_index",
557 "Index of the last commit.",
558 registry,
559 ).unwrap(),
560 last_commit_time_diff: register_histogram_with_registry!(
561 "last_commit_time_diff",
562 "The time diff between the last commit and previous one.",
563 LATENCY_SEC_BUCKETS.to_vec(),
564 registry,
565 ).unwrap(),
566 commit_round_advancement_interval: register_histogram_with_registry!(
567 "commit_round_advancement_interval",
568 "Intervals (in secs) between commit round advancements.",
569 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
570 registry,
571 ).unwrap(),
572 last_decided_leader_round: register_int_gauge_with_registry!(
573 "last_decided_leader_round",
574 "The last round where a commit decision was made.",
575 registry,
576 ).unwrap(),
577 leader_timeout_total: register_int_counter_vec_with_registry!(
578 "leader_timeout_total",
579 "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
580 &["timeout_type"],
581 registry,
582 ).unwrap(),
583 smart_selection_wait: register_int_counter_with_registry!(
584 "smart_selection_wait",
585 "Number of times we waited for smart ancestor selection.",
586 registry,
587 ).unwrap(),
588 ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
589 "ancestor_state_change_by_authority",
590 "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
591 &["authority", "state"],
592 registry,
593 ).unwrap(),
594 excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
595 "excluded_proposal_ancestors_count_by_authority",
596 "Total number of excluded ancestors per authority during proposal.",
597 &["authority"],
598 registry,
599 ).unwrap(),
600 included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
601 "included_excluded_proposal_ancestors_count_by_authority",
602 "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
603 &["authority", "type"],
604 registry,
605 ).unwrap(),
606 missing_blocks_total: register_int_counter_with_registry!(
607 "missing_blocks_total",
608 "Total cumulative number of missing blocks",
609 registry,
610 ).unwrap(),
611 missing_blocks_after_fetch_total: register_int_counter_with_registry!(
612 "missing_blocks_after_fetch_total",
613 "Total number of missing blocks after fetching blocks from peer",
614 registry,
615 ).unwrap(),
616 num_of_bad_nodes: register_int_gauge_with_registry!(
617 "num_of_bad_nodes",
618 "The number of bad nodes in the new leader schedule",
619 registry
620 ).unwrap(),
621 quorum_receive_latency: register_histogram_with_registry!(
622 "quorum_receive_latency",
623 "The time it took to receive a new round quorum of blocks",
624 registry
625 ).unwrap(),
626 block_receive_delay: register_int_counter_vec_with_registry!(
627 "block_receive_delay",
628 "Total delay from the start of the round to receiving the block, in milliseconds per authority",
629 &["authority"],
630 registry,
631 ).unwrap(),
632 reputation_scores: register_int_gauge_vec_with_registry!(
633 "reputation_scores",
634 "Reputation scores for each authority",
635 &["authority"],
636 registry,
637 ).unwrap(),
638 scope_processing_time: register_histogram_vec_with_registry!(
639 "scope_processing_time",
640 "The processing time of a specific code scope",
641 &["scope"],
642 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
643 registry
644 ).unwrap(),
645 sub_dags_per_commit_count: register_histogram_with_registry!(
646 "sub_dags_per_commit_count",
647 "The number of subdags per commit.",
648 registry,
649 ).unwrap(),
650 block_suspensions: register_int_counter_vec_with_registry!(
651 "block_suspensions",
652 "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
653 &["authority"],
654 registry,
655 ).unwrap(),
656 block_unsuspensions: register_int_counter_vec_with_registry!(
657 "block_unsuspensions",
658 "The number of block unsuspensions.",
659 &["authority"],
660 registry,
661 ).unwrap(),
662 suspended_block_time: register_histogram_vec_with_registry!(
663 "suspended_block_time",
664 "The time for which a block remains suspended",
665 &["authority"],
666 registry,
667 ).unwrap(),
668 block_manager_suspended_blocks: register_int_gauge_with_registry!(
669 "block_manager_suspended_blocks",
670 "The number of blocks currently suspended in the block manager",
671 registry,
672 ).unwrap(),
673 block_manager_missing_ancestors: register_int_gauge_with_registry!(
674 "block_manager_missing_ancestors",
675 "The number of missing ancestors tracked in the block manager",
676 registry,
677 ).unwrap(),
678 block_manager_missing_blocks: register_int_gauge_with_registry!(
679 "block_manager_missing_blocks",
680 "The number of blocks missing content tracked in the block manager",
681 registry,
682 ).unwrap(),
683 block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
684 "block_manager_missing_blocks_by_authority",
685 "The number of new missing blocks by block authority",
686 &["authority"],
687 registry,
688 ).unwrap(),
689 block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
690 "block_manager_missing_ancestors_by_authority",
691 "The number of missing ancestors by ancestor authority across received blocks",
692 &["authority"],
693 registry,
694 ).unwrap(),
695 block_manager_gced_blocks: register_int_counter_vec_with_registry!(
696 "block_manager_gced_blocks",
697 "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
698 &["authority"],
699 registry,
700 ).unwrap(),
701 block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
702 "block_manager_gc_unsuspended_blocks",
703 "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
704 &["authority"],
705 registry,
706 ).unwrap(),
707 block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
708 "block_manager_skipped_blocks",
709 "The number of blocks skipped by the block manager due to block round being <= gc_round",
710 &["authority"],
711 registry,
712 ).unwrap(),
713 threshold_clock_round: register_int_gauge_with_registry!(
714 "threshold_clock_round",
715 "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
716 registry,
717 ).unwrap(),
718 subscriber_connection_attempts: register_int_counter_vec_with_registry!(
719 "subscriber_connection_attempts",
720 "The number of connection attempts per peer",
721 &["authority", "status"],
722 registry,
723 ).unwrap(),
724 subscribed_to: register_int_gauge_vec_with_registry!(
725 "subscribed_to",
726 "Peers that this authority subscribed to for block streams.",
727 &["authority"],
728 registry,
729 ).unwrap(),
730 subscribed_by: register_int_gauge_vec_with_registry!(
731 "subscribed_by",
732 "Peers subscribing for block streams from this authority.",
733 &["authority"],
734 registry,
735 ).unwrap(),
736 commit_sync_inflight_fetches: register_int_gauge_with_registry!(
737 "commit_sync_inflight_fetches",
738 "The number of inflight fetches in commit syncer",
739 registry,
740 ).unwrap(),
741 commit_sync_pending_fetches: register_int_gauge_with_registry!(
742 "commit_sync_pending_fetches",
743 "The number of pending fetches in commit syncer",
744 registry,
745 ).unwrap(),
746 commit_sync_fetched_commits: register_int_counter_with_registry!(
747 "commit_sync_fetched_commits",
748 "The number of commits fetched via commit syncer",
749 registry,
750 ).unwrap(),
751 commit_sync_fetched_blocks: register_int_counter_with_registry!(
752 "commit_sync_fetched_blocks",
753 "The number of blocks fetched via commit syncer",
754 registry,
755 ).unwrap(),
756 commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
757 "commit_sync_total_fetched_blocks_size",
758 "The total size in bytes of blocks fetched via commit syncer",
759 registry,
760 ).unwrap(),
761 commit_sync_quorum_index: register_int_gauge_with_registry!(
762 "commit_sync_quorum_index",
763 "The maximum commit index voted by a quorum of authorities",
764 registry,
765 ).unwrap(),
766 commit_sync_highest_synced_index: register_int_gauge_with_registry!(
767 "commit_sync_fetched_index",
768 "The max commit index among local and fetched commits",
769 registry,
770 ).unwrap(),
771 commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
772 "commit_sync_highest_fetched_index",
773 "The max commit index that has been fetched via network",
774 registry,
775 ).unwrap(),
776 commit_sync_local_index: register_int_gauge_with_registry!(
777 "commit_sync_local_index",
778 "The local commit index",
779 registry,
780 ).unwrap(),
781 commit_sync_gap_on_processing: register_int_counter_with_registry!(
782 "commit_sync_gap_on_processing",
783 "Number of instances where a gap was found in fetched commit processing",
784 registry,
785 ).unwrap(),
786 commit_sync_fetch_loop_latency: register_histogram_with_registry!(
787 "commit_sync_fetch_loop_latency",
788 "The time taken to finish fetching commits and blocks from a given range",
789 LATENCY_SEC_BUCKETS.to_vec(),
790 registry,
791 ).unwrap(),
792 commit_sync_fetch_once_latency: register_histogram_with_registry!(
793 "commit_sync_fetch_once_latency",
794 "The time taken to fetch commits and blocks once",
795 LATENCY_SEC_BUCKETS.to_vec(),
796 registry,
797 ).unwrap(),
798 commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
799 "commit_sync_fetch_once_errors",
800 "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
801 &["authority", "error"],
802 registry
803 ).unwrap(),
804 commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
805 "commit_sync_fetch_missing_blocks",
806 "Number of ancestor blocks that are missing when processing blocks via commit sync.",
807 &["authority"],
808 registry,
809 ).unwrap(),
810 commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
811 "commit_sync_fetch_commits_handler_uncertified_skipped",
812 "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
813 registry,
814 ).unwrap(),
815 round_tracker_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
816 "round_tracker_received_quorum_round_gaps",
817 "Received round gaps among peers for blocks proposed from each authority",
818 &["authority"],
819 registry
820 ).unwrap(),
821 round_tracker_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
822 "round_tracker_accepted_quorum_round_gaps",
823 "Accepted round gaps among peers for blocks proposed & accepted from each authority",
824 &["authority"],
825 registry
826 ).unwrap(),
827 round_tracker_low_received_quorum_round: register_int_gauge_vec_with_registry!(
828 "round_tracker_low_received_quorum_round",
829 "Low quorum round among peers for blocks proposed from each authority",
830 &["authority"],
831 registry
832 ).unwrap(),
833 round_tracker_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
834 "round_tracker_low_accepted_quorum_round",
835 "Low quorum round among peers for blocks proposed & accepted from each authority",
836 &["authority"],
837 registry
838 ).unwrap(),
839 round_tracker_current_received_round_gaps: register_int_gauge_vec_with_registry!(
840 "round_tracker_current_received_round_gaps",
841 "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
842 &["authority"],
843 registry
844 ).unwrap(),
845 round_tracker_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
846 "round_tracker_current_accepted_round_gaps",
847 "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
848 &["authority"],
849 registry
850 ).unwrap(),
851 round_tracker_propagation_delays: register_histogram_with_registry!(
852 "round_tracker_propagation_delays",
853 "Round gaps between the last proposed block round and the lower bound of own quorum round",
854 NUM_BUCKETS.to_vec(),
855 registry
856 ).unwrap(),
857 round_tracker_last_propagation_delay: register_int_gauge_with_registry!(
858 "round_tracker_last_propagation_delay",
859 "Most recent propagation delay observed by RoundTracker",
860 registry
861 ).unwrap(),
862 round_prober_request_errors: register_int_counter_vec_with_registry!(
863 "round_prober_request_errors",
864 "Number of errors when probing against peers per error type",
865 &["error_type"],
866 registry
867 ).unwrap(),
868 certifier_gc_round: register_int_gauge_with_registry!(
869 "certifier_gc_round",
870 "The current GC round of the certifier",
871 registry
872 ).unwrap(),
873 certifier_own_reject_votes: register_int_counter_vec_with_registry!(
874 "certifier_own_reject_votes",
875 "Number of own reject votes against each peer authority",
876 &["authority"],
877 registry
878 ).unwrap(),
879 certifier_output_blocks: register_int_counter_vec_with_registry!(
880 "certifier_output_blocks",
881 "Number of output blocks certified by the certifier, grouped by type.",
882 &["type"],
883 registry
884 ).unwrap(),
885 finalizer_buffered_commits: register_int_gauge_with_registry!(
886 "finalizer_buffered_commits",
887 "The number of commits buffered in the finalizer",
888 registry,
889 ).unwrap(),
890 finalizer_round_delay: register_histogram_with_registry!(
891 "finalizer_round_delay",
892 "The delay between the round of the last committed block and the round of the finalized commit.",
893 ROUND_DELAY_BUCKETS.to_vec(),
894 registry,
895 ).unwrap(),
896 finalizer_transaction_status: register_int_counter_vec_with_registry!(
897 "finalizer_transaction_status",
898 "Number of transactions finalized by the finalizer, grouped by status.",
899 &["status"],
900 registry
901 ).unwrap(),
902 finalizer_reject_votes: register_int_counter_vec_with_registry!(
903 "finalizer_reject_votes",
904 "Number of reject votes casted by each authority observed by the finalizer.",
905 &["authority"],
906 registry
907 ).unwrap(),
908 finalizer_output_commits: register_int_counter_vec_with_registry!(
909 "finalizer_output_commits",
910 "Number of output commits finalized by the finalizer, grouped by type.",
911 &["type"],
912 registry
913 ).unwrap(),
914 finalizer_skipped_voting_blocks: register_int_counter_vec_with_registry!(
915 "finalizer_skipped_voting_blocks",
916 "Number of blocks skipped from voting due to potentially not being an immediate descendant.",
917 &["authority"],
918 registry
919 ).unwrap(),
920 uptime: register_histogram_with_registry!(
921 "uptime",
922 "Total node uptime",
923 LATENCY_SEC_BUCKETS.to_vec(),
924 registry,
925 ).unwrap(),
926 }
927 }
928}