1use std::sync::Arc;
5
6use prometheus::{
7 Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry,
8 exponential_buckets, register_histogram_vec_with_registry, register_histogram_with_registry,
9 register_int_counter_vec_with_registry, register_int_counter_with_registry,
10 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
11};
12
13use crate::network::metrics::NetworkMetrics;
14
15const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
17 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
18 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
19 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
20];
21
22const NUM_BUCKETS: &[f64] = &[
23 1.0,
24 2.0,
25 4.0,
26 8.0,
27 10.0,
28 20.0,
29 40.0,
30 80.0,
31 100.0,
32 150.0,
33 200.0,
34 400.0,
35 800.0,
36 1000.0,
37 2000.0,
38 3000.0,
39 5000.0,
40 10000.0,
41 20000.0,
42 30000.0,
43 50000.0,
44 100_000.0,
45 200_000.0,
46 300_000.0,
47 500_000.0,
48 1_000_000.0,
49];
50
51const LATENCY_SEC_BUCKETS: &[f64] = &[
52 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
53 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
54 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
55];
56
57const SIZE_BUCKETS: &[f64] = &[
58 100.,
59 400.,
60 800.,
61 1_000.,
62 2_000.,
63 5_000.,
64 10_000.,
65 20_000.,
66 50_000.,
67 100_000.,
68 200_000.0,
69 300_000.0,
70 400_000.0,
71 500_000.0,
72 1_000_000.0,
73 2_000_000.0,
74 3_000_000.0,
75 5_000_000.0,
76 10_000_000.0,
77]; const ROUND_DELAY_BUCKETS: &[f64] = &[0.0, 0.5, 1.0, 2.0, 3.0, 4.0];
81
82pub struct Metrics {
83 pub(crate) node_metrics: NodeMetrics,
84 pub(crate) network_metrics: NetworkMetrics,
85}
86
87pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
88 let node_metrics = NodeMetrics::new(®istry);
89 let network_metrics = NetworkMetrics::new(®istry);
90
91 Arc::new(Metrics {
92 node_metrics,
93 network_metrics,
94 })
95}
96
97pub(crate) fn test_metrics() -> Arc<Metrics> {
98 initialise_metrics(Registry::new())
99}
100
101pub(crate) struct NodeMetrics {
102 pub(crate) authority_index: IntGaugeVec,
103 pub(crate) protocol_version: IntGauge,
104 pub(crate) block_commit_latency: Histogram,
105 pub(crate) proposed_blocks: IntCounterVec,
106 pub(crate) proposed_block_size: Histogram,
107 pub(crate) proposed_block_transactions: Histogram,
108 pub(crate) proposed_block_ancestors: Histogram,
109 pub(crate) proposed_block_ancestors_depth: HistogramVec,
110 pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec,
111 pub(crate) highest_verified_authority_round: IntGaugeVec,
112 pub(crate) lowest_verified_authority_round: IntGaugeVec,
113 pub(crate) block_proposal_interval: Histogram,
114 pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
115 pub(crate) block_proposal_leader_wait_count: IntCounterVec,
116 pub(crate) block_timestamp_drift_ms: IntCounterVec,
117 pub(crate) blocks_per_commit_count: Histogram,
118 pub(crate) blocks_pruned_on_commit: IntCounterVec,
119 pub(crate) commit_observer_last_recovered_commit_index: IntGauge,
120 pub(crate) core_add_blocks_batch_size: Histogram,
121 pub(crate) core_check_block_refs_batch_size: Histogram,
122 pub(crate) core_lock_dequeued: IntCounter,
123 pub(crate) core_lock_enqueued: IntCounter,
124 pub(crate) core_skipped_proposals: IntCounterVec,
125 pub(crate) handler_received_block_missing_ancestors: IntCounterVec,
126 pub(crate) highest_accepted_authority_round: IntGaugeVec,
127 pub(crate) highest_accepted_round: IntGauge,
128 pub(crate) accepted_block_time_drift_ms: IntCounterVec,
129 pub(crate) accepted_blocks: IntCounterVec,
130 pub(crate) dag_state_recent_blocks: IntGauge,
131 pub(crate) dag_state_recent_refs: IntGauge,
132 pub(crate) dag_state_store_read_count: IntCounterVec,
133 pub(crate) dag_state_store_write_count: IntCounter,
134 pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
135 pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
136 pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
137 pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
138 pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
139 pub(crate) synchronizer_fetch_failures: IntCounterVec,
140 pub(crate) synchronizer_skipped_fetch_requests: IntCounterVec,
141 pub(crate) synchronizer_process_fetched_failures: IntCounterVec,
142 pub(crate) synchronizer_periodic_sync_decision: IntCounterVec,
143 pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
144 pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
145 pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
146 pub(crate) invalid_blocks: IntCounterVec,
147 pub(crate) rejected_blocks: IntCounterVec,
148 pub(crate) subscribed_blocks: IntCounterVec,
149 pub(crate) verified_blocks: IntCounterVec,
150 pub(crate) committed_leaders_total: IntCounterVec,
151 pub(crate) last_committed_authority_round: IntGaugeVec,
152 pub(crate) last_committed_leader_round: IntGauge,
153 pub(crate) last_commit_index: IntGauge,
154 pub(crate) last_commit_time_diff: Histogram,
155 pub(crate) last_known_own_block_round: IntGauge,
156 pub(crate) sync_last_known_own_block_retries: IntCounter,
157 pub(crate) commit_round_advancement_interval: Histogram,
158 pub(crate) last_decided_leader_round: IntGauge,
159 pub(crate) leader_timeout_total: IntCounterVec,
160 pub(crate) smart_selection_wait: IntCounter,
161 pub(crate) ancestor_state_change_by_authority: IntCounterVec,
162 pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
163 pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
164 pub(crate) missing_blocks_total: IntCounter,
165 pub(crate) missing_blocks_after_fetch_total: IntCounter,
166 pub(crate) num_of_bad_nodes: IntGauge,
167 pub(crate) quorum_receive_latency: Histogram,
168 pub(crate) block_receive_delay: IntCounterVec,
169 pub(crate) reputation_scores: IntGaugeVec,
170 pub(crate) scope_processing_time: HistogramVec,
171 pub(crate) sub_dags_per_commit_count: Histogram,
172 pub(crate) block_suspensions: IntCounterVec,
173 pub(crate) block_unsuspensions: IntCounterVec,
174 pub(crate) suspended_block_time: HistogramVec,
175 pub(crate) block_manager_suspended_blocks: IntGauge,
176 pub(crate) block_manager_missing_ancestors: IntGauge,
177 pub(crate) block_manager_missing_blocks: IntGauge,
178 pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
179 pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
180 pub(crate) block_manager_gced_blocks: IntCounterVec,
181 pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
182 pub(crate) block_manager_skipped_blocks: IntCounterVec,
183 pub(crate) threshold_clock_round: IntGauge,
184 pub(crate) subscriber_connection_attempts: IntCounterVec,
185 pub(crate) subscribed_to: IntGaugeVec,
186 pub(crate) subscribed_by: IntGaugeVec,
187 pub(crate) commit_sync_inflight_fetches: IntGauge,
188 pub(crate) commit_sync_pending_fetches: IntGauge,
189 pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
190 pub(crate) commit_sync_fetched_commits: IntCounter,
191 pub(crate) commit_sync_fetched_blocks: IntCounter,
192 pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
193 pub(crate) commit_sync_quorum_index: IntGauge,
194 pub(crate) commit_sync_highest_synced_index: IntGauge,
195 pub(crate) commit_sync_highest_fetched_index: IntGauge,
196 pub(crate) commit_sync_local_index: IntGauge,
197 pub(crate) commit_sync_gap_on_processing: IntCounter,
198 pub(crate) commit_sync_fetch_loop_latency: Histogram,
199 pub(crate) commit_sync_fetch_once_latency: Histogram,
200 pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
201 pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
202 pub(crate) round_tracker_received_quorum_round_gaps: IntGaugeVec,
203 pub(crate) round_tracker_accepted_quorum_round_gaps: IntGaugeVec,
204 pub(crate) round_tracker_low_received_quorum_round: IntGaugeVec,
205 pub(crate) round_tracker_low_accepted_quorum_round: IntGaugeVec,
206 pub(crate) round_tracker_current_received_round_gaps: IntGaugeVec,
207 pub(crate) round_tracker_current_accepted_round_gaps: IntGaugeVec,
208 pub(crate) round_tracker_propagation_delays: Histogram,
209 pub(crate) round_tracker_last_propagation_delay: IntGauge,
210 pub(crate) round_prober_request_errors: IntCounterVec,
211 pub(crate) certifier_gc_round: IntGauge,
212 pub(crate) certifier_block_latency: HistogramVec,
213 pub(crate) certifier_own_reject_votes: IntCounterVec,
214 pub(crate) certifier_output_blocks: IntCounterVec,
215 pub(crate) certifier_rejected_transactions: IntCounterVec,
216 pub(crate) certifier_accepted_transactions: IntCounterVec,
217 pub(crate) certifier_missing_ancestor_during_certification: IntCounterVec,
218 pub(crate) finalizer_buffered_commits: IntGauge,
219 pub(crate) finalizer_round_delay: Histogram,
220 pub(crate) finalizer_transaction_status: IntCounterVec,
221 pub(crate) finalizer_reject_votes: IntCounterVec,
222 pub(crate) finalizer_output_commits: IntCounterVec,
223 pub(crate) finalizer_skipped_voting_blocks: IntCounterVec,
224 pub(crate) uptime: Histogram,
225}
226
227impl NodeMetrics {
228 pub(crate) fn new(registry: &Registry) -> Self {
229 Self {
230 authority_index: register_int_gauge_vec_with_registry!(
231 "authority_index",
232 "The index of the authority",
233 &["name"],
234 registry,
235 ).unwrap(),
236 protocol_version: register_int_gauge_with_registry!(
237 "protocol_version",
238 "The protocol version used in this epoch",
239 registry,
240 ).unwrap(),
241 block_commit_latency: register_histogram_with_registry!(
242 "block_commit_latency",
243 "The time taken between block creation and block commit.",
244 LATENCY_SEC_BUCKETS.to_vec(),
245 registry,
246 ).unwrap(),
247 proposed_blocks: register_int_counter_vec_with_registry!(
248 "proposed_blocks",
249 "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
250 &["force"],
251 registry,
252 ).unwrap(),
253 proposed_block_size: register_histogram_with_registry!(
254 "proposed_block_size",
255 "The size (in bytes) of proposed blocks",
256 SIZE_BUCKETS.to_vec(),
257 registry
258 ).unwrap(),
259 proposed_block_transactions: register_histogram_with_registry!(
260 "proposed_block_transactions",
261 "# of transactions contained in proposed blocks",
262 NUM_BUCKETS.to_vec(),
263 registry
264 ).unwrap(),
265 proposed_block_ancestors: register_histogram_with_registry!(
266 "proposed_block_ancestors",
267 "Number of ancestors in proposed blocks",
268 exponential_buckets(1.0, 1.4, 20).unwrap(),
269 registry,
270 ).unwrap(),
271 proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!(
272 "proposed_block_ancestors_timestamp_drift_ms",
273 "The drift in ms of ancestors' timestamps included in newly proposed blocks",
274 &["authority"],
275 registry,
276 ).unwrap(),
277 proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
278 "proposed_block_ancestors_depth",
279 "The depth in rounds of ancestors included in newly proposed blocks",
280 &["authority"],
281 exponential_buckets(1.0, 2.0, 14).unwrap(),
282 registry,
283 ).unwrap(),
284 highest_verified_authority_round: register_int_gauge_vec_with_registry!(
285 "highest_verified_authority_round",
286 "The highest round of verified block for the corresponding authority",
287 &["authority"],
288 registry,
289 ).unwrap(),
290 lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
291 "lowest_verified_authority_round",
292 "The lowest round of verified block for the corresponding authority",
293 &["authority"],
294 registry,
295 ).unwrap(),
296 block_proposal_interval: register_histogram_with_registry!(
297 "block_proposal_interval",
298 "Intervals (in secs) between block proposals.",
299 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
300 registry,
301 ).unwrap(),
302 block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
303 "block_proposal_leader_wait_ms",
304 "Total time in ms spent waiting for a leader when proposing blocks.",
305 &["authority"],
306 registry,
307 ).unwrap(),
308 block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
309 "block_proposal_leader_wait_count",
310 "Total times waiting for a leader when proposing blocks.",
311 &["authority"],
312 registry,
313 ).unwrap(),
314 block_timestamp_drift_ms: register_int_counter_vec_with_registry!(
315 "block_timestamp_drift_ms",
316 "The clock drift time between a received block and the current node's time.",
317 &["authority", "source"],
318 registry,
319 ).unwrap(),
320 blocks_per_commit_count: register_histogram_with_registry!(
321 "blocks_per_commit_count",
322 "The number of blocks per commit.",
323 NUM_BUCKETS.to_vec(),
324 registry,
325 ).unwrap(),
326 blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
327 "blocks_pruned_on_commit",
328 "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
329 &["authority", "commit_status"],
330 registry,
331 ).unwrap(),
332 commit_observer_last_recovered_commit_index: register_int_gauge_with_registry!(
333 "commit_observer_last_recovered_commit_index",
334 "The last commit index recovered by the commit observer",
335 registry,
336 ).unwrap(),
337 core_add_blocks_batch_size: register_histogram_with_registry!(
338 "core_add_blocks_batch_size",
339 "The number of blocks received from Core for processing on a single batch",
340 NUM_BUCKETS.to_vec(),
341 registry,
342 ).unwrap(),
343 core_check_block_refs_batch_size: register_histogram_with_registry!(
344 "core_check_block_refs_batch_size",
345 "The number of excluded blocks received from Core for search on a single batch",
346 NUM_BUCKETS.to_vec(),
347 registry,
348 ).unwrap(),
349 core_lock_dequeued: register_int_counter_with_registry!(
350 "core_lock_dequeued",
351 "Number of dequeued core requests",
352 registry,
353 ).unwrap(),
354 core_lock_enqueued: register_int_counter_with_registry!(
355 "core_lock_enqueued",
356 "Number of enqueued core requests",
357 registry,
358 ).unwrap(),
359 core_skipped_proposals: register_int_counter_vec_with_registry!(
360 "core_skipped_proposals",
361 "Number of proposals skipped in the Core, per reason",
362 &["reason"],
363 registry,
364 ).unwrap(),
365 handler_received_block_missing_ancestors: register_int_counter_vec_with_registry!(
366 "handler_received_block_missing_ancestors",
367 "Number of missing ancestors in blocks received in the handler, separated by peer",
368 &["authority"],
369 registry,
370 ).unwrap(),
371 highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
372 "highest_accepted_authority_round",
373 "The highest round where a block has been accepted per authority. Resets on restart.",
374 &["authority"],
375 registry,
376 ).unwrap(),
377 highest_accepted_round: register_int_gauge_with_registry!(
378 "highest_accepted_round",
379 "The highest round where a block has been accepted. Resets on restart.",
380 registry,
381 ).unwrap(),
382 accepted_block_time_drift_ms: register_int_counter_vec_with_registry!(
383 "accepted_block_time_drift_ms",
384 "The time drift in ms of an accepted block compared to local time",
385 &["authority"],
386 registry,
387 ).unwrap(),
388 accepted_blocks: register_int_counter_vec_with_registry!(
389 "accepted_blocks",
390 "Number of accepted blocks by source (own, others)",
391 &["source"],
392 registry,
393 ).unwrap(),
394 dag_state_recent_blocks: register_int_gauge_with_registry!(
395 "dag_state_recent_blocks",
396 "Number of recent blocks cached in the DagState",
397 registry,
398 ).unwrap(),
399 dag_state_recent_refs: register_int_gauge_with_registry!(
400 "dag_state_recent_refs",
401 "Number of recent refs cached in the DagState",
402 registry,
403 ).unwrap(),
404 dag_state_store_read_count: register_int_counter_vec_with_registry!(
405 "dag_state_store_read_count",
406 "Number of times DagState needs to read from store per operation type",
407 &["type"],
408 registry,
409 ).unwrap(),
410 dag_state_store_write_count: register_int_counter_with_registry!(
411 "dag_state_store_write_count",
412 "Number of times DagState needs to write to store",
413 registry,
414 ).unwrap(),
415 fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
416 "fetch_blocks_scheduler_inflight",
417 "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
418 registry,
419 ).unwrap(),
420 synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
421 "synchronizer_fetched_blocks_by_peer",
422 "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
423 &["peer", "type"],
424 registry,
425 ).unwrap(),
426 synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
427 "synchronizer_missing_blocks_by_authority",
428 "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
429 &["authority"],
430 registry,
431 ).unwrap(),
432 synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
433 "synchronizer_current_missing_blocks_by_authority",
434 "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
435 &["authority"],
436 registry,
437 ).unwrap(),
438 synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
439 "synchronizer_fetched_blocks_by_authority",
440 "Number of fetched blocks per block author via the synchronizer",
441 &["authority", "type"],
442 registry,
443 ).unwrap(),
444 synchronizer_fetch_failures: register_int_counter_vec_with_registry!(
445 "synchronizer_fetch_failures",
446 "Number of fetch failures against each peer",
447 &["peer", "type"],
448 registry,
449 ).unwrap(),
450 synchronizer_skipped_fetch_requests: register_int_counter_vec_with_registry!(
451 "synchronizer_skipped_fetch_requests",
452 "Number of fetch requests skipped against each peer, because the peer is saturated",
453 &["peer"],
454 registry,
455 ).unwrap(),
456 synchronizer_process_fetched_failures: register_int_counter_vec_with_registry!(
457 "synchronizer_process_fetched_failures",
458 "Number of failures for processing fetched blocks against each peer",
459 &["peer", "type"],
460 registry,
461 ).unwrap(),
462 synchronizer_periodic_sync_decision: register_int_counter_vec_with_registry!(
463 "synchronizer_periodic_sync_decision",
464 "Decision to run periodic sync",
465 &["decision", "reason"],
466 registry,
467 ).unwrap(),
468 network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
469 "network_received_excluded_ancestors_from_authority",
470 "Number of excluded ancestors received from each authority.",
471 &["authority"],
472 registry,
473 ).unwrap(),
474 network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
475 "network_excluded_ancestors_count_by_authority",
476 "Total number of excluded ancestors per authority.",
477 &["authority"],
478 registry,
479 ).unwrap(),
480 network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
481 "network_excluded_ancestors_sent_to_fetch",
482 "Number of excluded ancestors sent to fetch.",
483 &["authority"],
484 registry,
485 ).unwrap(),
486 last_known_own_block_round: register_int_gauge_with_registry!(
487 "last_known_own_block_round",
488 "The highest round of our own block as this has been synced from peers during an amnesia recovery",
489 registry,
490 ).unwrap(),
491 sync_last_known_own_block_retries: register_int_counter_with_registry!(
492 "sync_last_known_own_block_retries",
493 "Number of times this node tried to fetch the last own block from peers",
494 registry,
495 ).unwrap(),
496 invalid_blocks: register_int_counter_vec_with_registry!(
497 "invalid_blocks",
498 "Number of invalid blocks per peer authority",
499 &["authority", "source", "error"],
500 registry,
501 ).unwrap(),
502 certifier_block_latency: register_histogram_vec_with_registry!(
503 "certifier_block_latency",
504 "The latency of a block being certified by the transaction certifier. The block's authority is the label",
505 &["authority"],
506 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
507 registry,
508 ).unwrap(),
509 certifier_rejected_transactions: register_int_counter_vec_with_registry!(
510 "certifier_rejected_transactions",
511 "Number of transactions rejected by authority in transaction certifier",
512 &["authority"],
513 registry,
514 ).unwrap(),
515 certifier_accepted_transactions: register_int_counter_vec_with_registry!(
516 "certifier_accepted_transactions",
517 "Number of transactions accepted by authority in transaction certifier",
518 &["authority"],
519 registry,
520 ).unwrap(),
521 certifier_missing_ancestor_during_certification: register_int_counter_vec_with_registry!(
522 "certifier_missing_ancestor_during_certification",
523 "Number of missing ancestors during certification",
524 &["reason"],
525 registry,
526 ).unwrap(),
527 rejected_blocks: register_int_counter_vec_with_registry!(
528 "rejected_blocks",
529 "Number of blocks rejected before verifications",
530 &["reason"],
531 registry,
532 ).unwrap(),
533 subscribed_blocks: register_int_counter_vec_with_registry!(
534 "subscribed_blocks",
535 "Number of blocks received from each peer before verification",
536 &["authority"],
537 registry,
538 ).unwrap(),
539 verified_blocks: register_int_counter_vec_with_registry!(
540 "verified_blocks",
541 "Number of blocks received from each peer that are verified",
542 &["authority"],
543 registry,
544 ).unwrap(),
545 committed_leaders_total: register_int_counter_vec_with_registry!(
546 "committed_leaders_total",
547 "Total number of (direct or indirect) committed leaders per authority",
548 &["authority", "commit_type"],
549 registry,
550 ).unwrap(),
551 last_committed_authority_round: register_int_gauge_vec_with_registry!(
552 "last_committed_authority_round",
553 "The last round committed by authority.",
554 &["authority"],
555 registry,
556 ).unwrap(),
557 last_committed_leader_round: register_int_gauge_with_registry!(
558 "last_committed_leader_round",
559 "The last round where a leader was committed to store and sent to commit consumer.",
560 registry,
561 ).unwrap(),
562 last_commit_index: register_int_gauge_with_registry!(
563 "last_commit_index",
564 "Index of the last commit.",
565 registry,
566 ).unwrap(),
567 last_commit_time_diff: register_histogram_with_registry!(
568 "last_commit_time_diff",
569 "The time diff between the last commit and previous one.",
570 LATENCY_SEC_BUCKETS.to_vec(),
571 registry,
572 ).unwrap(),
573 commit_round_advancement_interval: register_histogram_with_registry!(
574 "commit_round_advancement_interval",
575 "Intervals (in secs) between commit round advancements.",
576 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
577 registry,
578 ).unwrap(),
579 last_decided_leader_round: register_int_gauge_with_registry!(
580 "last_decided_leader_round",
581 "The last round where a commit decision was made.",
582 registry,
583 ).unwrap(),
584 leader_timeout_total: register_int_counter_vec_with_registry!(
585 "leader_timeout_total",
586 "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
587 &["timeout_type"],
588 registry,
589 ).unwrap(),
590 smart_selection_wait: register_int_counter_with_registry!(
591 "smart_selection_wait",
592 "Number of times we waited for smart ancestor selection.",
593 registry,
594 ).unwrap(),
595 ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
596 "ancestor_state_change_by_authority",
597 "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
598 &["authority", "state"],
599 registry,
600 ).unwrap(),
601 excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
602 "excluded_proposal_ancestors_count_by_authority",
603 "Total number of excluded ancestors per authority during proposal.",
604 &["authority"],
605 registry,
606 ).unwrap(),
607 included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
608 "included_excluded_proposal_ancestors_count_by_authority",
609 "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
610 &["authority", "type"],
611 registry,
612 ).unwrap(),
613 missing_blocks_total: register_int_counter_with_registry!(
614 "missing_blocks_total",
615 "Total cumulative number of missing blocks",
616 registry,
617 ).unwrap(),
618 missing_blocks_after_fetch_total: register_int_counter_with_registry!(
619 "missing_blocks_after_fetch_total",
620 "Total number of missing blocks after fetching blocks from peer",
621 registry,
622 ).unwrap(),
623 num_of_bad_nodes: register_int_gauge_with_registry!(
624 "num_of_bad_nodes",
625 "The number of bad nodes in the new leader schedule",
626 registry
627 ).unwrap(),
628 quorum_receive_latency: register_histogram_with_registry!(
629 "quorum_receive_latency",
630 "The time it took to receive a new round quorum of blocks",
631 registry
632 ).unwrap(),
633 block_receive_delay: register_int_counter_vec_with_registry!(
634 "block_receive_delay",
635 "Total delay from the start of the round to receiving the block, in milliseconds per authority",
636 &["authority"],
637 registry,
638 ).unwrap(),
639 reputation_scores: register_int_gauge_vec_with_registry!(
640 "reputation_scores",
641 "Reputation scores for each authority",
642 &["authority"],
643 registry,
644 ).unwrap(),
645 scope_processing_time: register_histogram_vec_with_registry!(
646 "scope_processing_time",
647 "The processing time of a specific code scope",
648 &["scope"],
649 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
650 registry
651 ).unwrap(),
652 sub_dags_per_commit_count: register_histogram_with_registry!(
653 "sub_dags_per_commit_count",
654 "The number of subdags per commit.",
655 registry,
656 ).unwrap(),
657 block_suspensions: register_int_counter_vec_with_registry!(
658 "block_suspensions",
659 "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
660 &["authority"],
661 registry,
662 ).unwrap(),
663 block_unsuspensions: register_int_counter_vec_with_registry!(
664 "block_unsuspensions",
665 "The number of block unsuspensions.",
666 &["authority"],
667 registry,
668 ).unwrap(),
669 suspended_block_time: register_histogram_vec_with_registry!(
670 "suspended_block_time",
671 "The time for which a block remains suspended",
672 &["authority"],
673 registry,
674 ).unwrap(),
675 block_manager_suspended_blocks: register_int_gauge_with_registry!(
676 "block_manager_suspended_blocks",
677 "The number of blocks currently suspended in the block manager",
678 registry,
679 ).unwrap(),
680 block_manager_missing_ancestors: register_int_gauge_with_registry!(
681 "block_manager_missing_ancestors",
682 "The number of missing ancestors tracked in the block manager",
683 registry,
684 ).unwrap(),
685 block_manager_missing_blocks: register_int_gauge_with_registry!(
686 "block_manager_missing_blocks",
687 "The number of blocks missing content tracked in the block manager",
688 registry,
689 ).unwrap(),
690 block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
691 "block_manager_missing_blocks_by_authority",
692 "The number of new missing blocks by block authority",
693 &["authority"],
694 registry,
695 ).unwrap(),
696 block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
697 "block_manager_missing_ancestors_by_authority",
698 "The number of missing ancestors by ancestor authority across received blocks",
699 &["authority"],
700 registry,
701 ).unwrap(),
702 block_manager_gced_blocks: register_int_counter_vec_with_registry!(
703 "block_manager_gced_blocks",
704 "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
705 &["authority"],
706 registry,
707 ).unwrap(),
708 block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
709 "block_manager_gc_unsuspended_blocks",
710 "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
711 &["authority"],
712 registry,
713 ).unwrap(),
714 block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
715 "block_manager_skipped_blocks",
716 "The number of blocks skipped by the block manager due to block round being <= gc_round",
717 &["authority"],
718 registry,
719 ).unwrap(),
720 threshold_clock_round: register_int_gauge_with_registry!(
721 "threshold_clock_round",
722 "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
723 registry,
724 ).unwrap(),
725 subscriber_connection_attempts: register_int_counter_vec_with_registry!(
726 "subscriber_connection_attempts",
727 "The number of connection attempts per peer",
728 &["authority", "status"],
729 registry,
730 ).unwrap(),
731 subscribed_to: register_int_gauge_vec_with_registry!(
732 "subscribed_to",
733 "Peers that this authority subscribed to for block streams.",
734 &["authority"],
735 registry,
736 ).unwrap(),
737 subscribed_by: register_int_gauge_vec_with_registry!(
738 "subscribed_by",
739 "Peers subscribing for block streams from this authority.",
740 &["authority"],
741 registry,
742 ).unwrap(),
743 commit_sync_inflight_fetches: register_int_gauge_with_registry!(
744 "commit_sync_inflight_fetches",
745 "The number of inflight fetches in commit syncer",
746 registry,
747 ).unwrap(),
748 commit_sync_pending_fetches: register_int_gauge_with_registry!(
749 "commit_sync_pending_fetches",
750 "The number of pending fetches in commit syncer",
751 registry,
752 ).unwrap(),
753 commit_sync_fetched_commits: register_int_counter_with_registry!(
754 "commit_sync_fetched_commits",
755 "The number of commits fetched via commit syncer",
756 registry,
757 ).unwrap(),
758 commit_sync_fetched_blocks: register_int_counter_with_registry!(
759 "commit_sync_fetched_blocks",
760 "The number of blocks fetched via commit syncer",
761 registry,
762 ).unwrap(),
763 commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
764 "commit_sync_total_fetched_blocks_size",
765 "The total size in bytes of blocks fetched via commit syncer",
766 registry,
767 ).unwrap(),
768 commit_sync_quorum_index: register_int_gauge_with_registry!(
769 "commit_sync_quorum_index",
770 "The maximum commit index voted by a quorum of authorities",
771 registry,
772 ).unwrap(),
773 commit_sync_highest_synced_index: register_int_gauge_with_registry!(
774 "commit_sync_fetched_index",
775 "The max commit index among local and fetched commits",
776 registry,
777 ).unwrap(),
778 commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
779 "commit_sync_highest_fetched_index",
780 "The max commit index that has been fetched via network",
781 registry,
782 ).unwrap(),
783 commit_sync_local_index: register_int_gauge_with_registry!(
784 "commit_sync_local_index",
785 "The local commit index",
786 registry,
787 ).unwrap(),
788 commit_sync_gap_on_processing: register_int_counter_with_registry!(
789 "commit_sync_gap_on_processing",
790 "Number of instances where a gap was found in fetched commit processing",
791 registry,
792 ).unwrap(),
793 commit_sync_fetch_loop_latency: register_histogram_with_registry!(
794 "commit_sync_fetch_loop_latency",
795 "The time taken to finish fetching commits and blocks from a given range",
796 LATENCY_SEC_BUCKETS.to_vec(),
797 registry,
798 ).unwrap(),
799 commit_sync_fetch_once_latency: register_histogram_with_registry!(
800 "commit_sync_fetch_once_latency",
801 "The time taken to fetch commits and blocks once",
802 LATENCY_SEC_BUCKETS.to_vec(),
803 registry,
804 ).unwrap(),
805 commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
806 "commit_sync_fetch_once_errors",
807 "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
808 &["authority", "error"],
809 registry
810 ).unwrap(),
811 commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
812 "commit_sync_fetch_missing_blocks",
813 "Number of ancestor blocks that are missing when processing blocks via commit sync.",
814 &["authority"],
815 registry,
816 ).unwrap(),
817 commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
818 "commit_sync_fetch_commits_handler_uncertified_skipped",
819 "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
820 registry,
821 ).unwrap(),
822 round_tracker_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
823 "round_tracker_received_quorum_round_gaps",
824 "Received round gaps among peers for blocks proposed from each authority",
825 &["authority"],
826 registry
827 ).unwrap(),
828 round_tracker_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
829 "round_tracker_accepted_quorum_round_gaps",
830 "Accepted round gaps among peers for blocks proposed & accepted from each authority",
831 &["authority"],
832 registry
833 ).unwrap(),
834 round_tracker_low_received_quorum_round: register_int_gauge_vec_with_registry!(
835 "round_tracker_low_received_quorum_round",
836 "Low quorum round among peers for blocks proposed from each authority",
837 &["authority"],
838 registry
839 ).unwrap(),
840 round_tracker_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
841 "round_tracker_low_accepted_quorum_round",
842 "Low quorum round among peers for blocks proposed & accepted from each authority",
843 &["authority"],
844 registry
845 ).unwrap(),
846 round_tracker_current_received_round_gaps: register_int_gauge_vec_with_registry!(
847 "round_tracker_current_received_round_gaps",
848 "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
849 &["authority"],
850 registry
851 ).unwrap(),
852 round_tracker_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
853 "round_tracker_current_accepted_round_gaps",
854 "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
855 &["authority"],
856 registry
857 ).unwrap(),
858 round_tracker_propagation_delays: register_histogram_with_registry!(
859 "round_tracker_propagation_delays",
860 "Round gaps between the last proposed block round and the lower bound of own quorum round",
861 NUM_BUCKETS.to_vec(),
862 registry
863 ).unwrap(),
864 round_tracker_last_propagation_delay: register_int_gauge_with_registry!(
865 "round_tracker_last_propagation_delay",
866 "Most recent propagation delay observed by RoundTracker",
867 registry
868 ).unwrap(),
869 round_prober_request_errors: register_int_counter_vec_with_registry!(
870 "round_prober_request_errors",
871 "Number of errors when probing against peers per error type",
872 &["error_type"],
873 registry
874 ).unwrap(),
875 certifier_gc_round: register_int_gauge_with_registry!(
876 "certifier_gc_round",
877 "The current GC round of the certifier",
878 registry
879 ).unwrap(),
880 certifier_own_reject_votes: register_int_counter_vec_with_registry!(
881 "certifier_own_reject_votes",
882 "Number of own reject votes against each peer authority",
883 &["authority"],
884 registry
885 ).unwrap(),
886 certifier_output_blocks: register_int_counter_vec_with_registry!(
887 "certifier_output_blocks",
888 "Number of output blocks certified by the certifier, grouped by type.",
889 &["type"],
890 registry
891 ).unwrap(),
892 finalizer_buffered_commits: register_int_gauge_with_registry!(
893 "finalizer_buffered_commits",
894 "The number of commits buffered in the finalizer",
895 registry,
896 ).unwrap(),
897 finalizer_round_delay: register_histogram_with_registry!(
898 "finalizer_round_delay",
899 "The delay between the round of the last committed block and the round of the finalized commit.",
900 ROUND_DELAY_BUCKETS.to_vec(),
901 registry,
902 ).unwrap(),
903 finalizer_transaction_status: register_int_counter_vec_with_registry!(
904 "finalizer_transaction_status",
905 "Number of transactions finalized by the finalizer, grouped by status.",
906 &["status"],
907 registry
908 ).unwrap(),
909 finalizer_reject_votes: register_int_counter_vec_with_registry!(
910 "finalizer_reject_votes",
911 "Number of reject votes casted by each authority observed by the finalizer.",
912 &["authority"],
913 registry
914 ).unwrap(),
915 finalizer_output_commits: register_int_counter_vec_with_registry!(
916 "finalizer_output_commits",
917 "Number of output commits finalized by the finalizer, grouped by type.",
918 &["type"],
919 registry
920 ).unwrap(),
921 finalizer_skipped_voting_blocks: register_int_counter_vec_with_registry!(
922 "finalizer_skipped_voting_blocks",
923 "Number of times where another block skipped voting due to potentially out of GC bound. Authority is from the voted block.",
924 &["authority", "type"],
925 registry
926 ).unwrap(),
927 uptime: register_histogram_with_registry!(
928 "uptime",
929 "Total node uptime",
930 LATENCY_SEC_BUCKETS.to_vec(),
931 registry,
932 ).unwrap(),
933 }
934 }
935}