1use std::sync::Arc;
5
6use prometheus::{
7 Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
8 Registry, exponential_buckets, register_gauge_vec_with_registry, register_gauge_with_registry,
9 register_histogram_vec_with_registry, register_histogram_with_registry,
10 register_int_counter_vec_with_registry, register_int_counter_with_registry,
11 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
12};
13
14use crate::network::metrics::NetworkMetrics;
15
16const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
18 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
19 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
20 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
21];
22
23const NUM_BUCKETS: &[f64] = &[
24 1.0,
25 2.0,
26 4.0,
27 8.0,
28 10.0,
29 20.0,
30 40.0,
31 80.0,
32 100.0,
33 150.0,
34 200.0,
35 250.0,
36 300.0,
37 400.0,
38 500.0,
39 600.0,
40 700.0,
41 800.0,
42 900.0,
43 1000.0,
44 1500.0,
45 2000.0,
46 2500.0,
47 3000.0,
48 4000.0,
49 5000.0,
50 6000.0,
51 7000.0,
52 8000.0,
53 9000.0,
54 10000.0,
55 15000.0,
56 20000.0,
57 25000.0,
58 30000.0,
59 40000.0,
60 50000.0,
61 100_000.0,
62 200_000.0,
63 300_000.0,
64 500_000.0,
65 1_000_000.0,
66];
67
68const LATENCY_SEC_BUCKETS: &[f64] = &[
69 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
70 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
71 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
72];
73
74const SIZE_BUCKETS: &[f64] = &[
75 100.,
76 400.,
77 800.,
78 1_000.,
79 2_000.,
80 5_000.,
81 10_000.,
82 20_000.,
83 50_000.,
84 100_000.,
85 150_000.,
86 200_000.0,
87 250_000.0,
88 300_000.0,
89 350_000.0,
90 400_000.0,
91 450_000.0,
92 500_000.0,
93 600_000.0,
94 700_000.0,
95 800_000.0,
96 900_000.0,
97 1_000_000.0,
98 2_000_000.0,
99 3_000_000.0,
100 5_000_000.0,
101 10_000_000.0,
102]; const ROUND_DELAY_BUCKETS: &[f64] = &[0.0, 0.5, 1.0, 2.0, 3.0, 4.0];
106
107pub struct Metrics {
108 pub(crate) node_metrics: NodeMetrics,
109 pub(crate) network_metrics: NetworkMetrics,
110}
111
112pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
113 let node_metrics = NodeMetrics::new(®istry);
114 let network_metrics = NetworkMetrics::new(®istry);
115
116 Arc::new(Metrics {
117 node_metrics,
118 network_metrics,
119 })
120}
121
122pub(crate) fn test_metrics() -> Arc<Metrics> {
123 initialise_metrics(Registry::new())
124}
125
126pub(crate) struct NodeMetrics {
127 pub(crate) authority_index: IntGaugeVec,
128 pub(crate) protocol_version: IntGauge,
129 pub(crate) block_commit_latency: Histogram,
130 pub(crate) proposed_blocks: IntCounterVec,
131 pub(crate) proposed_block_commit_latency: Histogram,
132 pub(crate) proposed_block_finalization_latency: Histogram,
133 pub(crate) proposed_block_size: Histogram,
134 pub(crate) proposed_block_transactions: Histogram,
135 pub(crate) proposed_block_transaction_bytes: Histogram,
136 pub(crate) proposed_block_transaction_vote_blocks: Histogram,
137 pub(crate) proposed_block_transaction_vote_entries: Histogram,
138 pub(crate) proposed_block_ancestors: Histogram,
139 pub(crate) proposed_block_ancestors_depth: HistogramVec,
140 pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec,
141 pub(crate) highest_verified_authority_round: IntGaugeVec,
142 pub(crate) lowest_verified_authority_round: IntGaugeVec,
143 pub(crate) block_proposal_interval: Histogram,
144 pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
145 pub(crate) block_proposal_leader_wait_count: IntCounterVec,
146 pub(crate) block_timestamp_drift_ms: IntCounterVec,
147 pub(crate) blocks_per_commit_count: Histogram,
148 pub(crate) blocks_pruned_on_commit: IntCounterVec,
149 pub(crate) block_sync_service_read_blocks_count: IntCounter,
150 pub(crate) commit_observer_last_recovered_commit_index: IntGauge,
151 pub(crate) core_add_blocks_batch_size: Histogram,
152 pub(crate) core_check_block_refs_batch_size: Histogram,
153 pub(crate) core_lock_dequeued: IntCounter,
154 pub(crate) core_lock_enqueued: IntCounter,
155 pub(crate) priority_submission_backpressure: IntCounter,
156 pub(crate) core_skipped_proposals: IntCounterVec,
157 pub(crate) handler_received_block_missing_ancestors: IntCounterVec,
158 pub(crate) highest_accepted_authority_round: IntGaugeVec,
159 pub(crate) highest_accepted_round: IntGauge,
160 pub(crate) accepted_block_time_drift_ms: IntCounterVec,
161 pub(crate) accepted_blocks: IntCounterVec,
162 pub(crate) dag_state_recent_blocks: IntGauge,
163 pub(crate) dag_state_recent_refs: IntGauge,
164 pub(crate) dag_state_store_read_count: IntCounterVec,
165 pub(crate) dag_state_store_write_count: IntCounter,
166 pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
167 pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
168 pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
169 pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
170 pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
171 pub(crate) synchronizer_fetch_failures: IntCounterVec,
172 pub(crate) synchronizer_skipped_fetch_requests: IntCounterVec,
173 pub(crate) synchronizer_process_fetched_failures: IntCounterVec,
174 pub(crate) synchronizer_periodic_sync_decision: IntCounterVec,
175 pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
176 pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
177 pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
178 pub(crate) invalid_blocks: IntCounterVec,
179 pub(crate) rejected_blocks: IntCounterVec,
180 pub(crate) subscribed_blocks: IntCounterVec,
181 pub(crate) observer_subscribed_blocks_batch_size: Histogram,
182 pub(crate) observer_subscription_suspended: IntGauge,
183 pub(crate) verified_blocks: IntCounterVec,
184 pub(crate) committed_leaders_total: IntCounterVec,
185 pub(crate) last_committed_authority_round: IntGaugeVec,
186 pub(crate) last_committed_leader_round: IntGauge,
187 pub(crate) last_commit_index: IntGauge,
188 pub(crate) last_commit_time_diff: Histogram,
189 pub(crate) last_known_own_block_round: IntGauge,
190 pub(crate) sync_last_known_own_block_retries: IntCounter,
191 pub(crate) commit_round_advancement_interval: Histogram,
192 pub(crate) last_decided_leader_round: IntGauge,
193 pub(crate) leader_timeout_total: IntCounterVec,
194 pub(crate) smart_selection_wait: IntCounter,
195 pub(crate) ancestor_state_change_by_authority: IntCounterVec,
196 pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
197 pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
198 pub(crate) missing_blocks_total: IntCounter,
199 pub(crate) missing_blocks_after_fetch_total: IntCounter,
200 pub(crate) num_of_bad_nodes: IntGauge,
201 pub(crate) quorum_receive_latency: Histogram,
202 pub(crate) block_receive_delay: IntCounterVec,
203 pub(crate) reputation_scores: IntGaugeVec,
204 pub(crate) leader_schedule_total_scores: IntGaugeVec,
205 pub(crate) leader_schedule_normalized_scores: GaugeVec,
206 pub(crate) leader_schedule_last_num_leaders: IntGauge,
207 pub(crate) leader_schedule_average_num_leaders: Gauge,
208 pub(crate) scope_processing_time: HistogramVec,
209 pub(crate) sub_dags_per_commit_count: Histogram,
210 pub(crate) block_suspensions: IntCounterVec,
211 pub(crate) block_unsuspensions: IntCounterVec,
212 pub(crate) suspended_block_time: HistogramVec,
213 pub(crate) block_manager_suspended_blocks: IntGauge,
214 pub(crate) block_manager_missing_ancestors: IntGauge,
215 pub(crate) block_manager_missing_blocks: IntGauge,
216 pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
217 pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
218 pub(crate) block_manager_gced_blocks: IntCounterVec,
219 pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
220 pub(crate) block_manager_skipped_blocks: IntCounterVec,
221 pub(crate) threshold_clock_round: IntGauge,
222 pub(crate) subscriber_connection_attempts: IntCounterVec,
223 pub(crate) subscribed_to: IntGaugeVec,
224 pub(crate) subscribed_by: IntGaugeVec,
225 pub(crate) commit_sync_inflight_fetches: IntGauge,
226 pub(crate) commit_sync_pending_fetches: IntGauge,
227 pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
228 pub(crate) commit_sync_fetched_commits: IntCounter,
229 pub(crate) commit_sync_fetched_blocks: IntCounter,
230 pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
231 pub(crate) commit_sync_quorum_index: IntGauge,
232 pub(crate) commit_sync_highest_synced_index: IntGauge,
233 pub(crate) commit_sync_highest_fetched_index: IntGauge,
234 pub(crate) commit_sync_local_index: IntGauge,
235 pub(crate) commit_sync_gap_on_processing: IntCounter,
236 pub(crate) commit_sync_fetch_loop_latency: Histogram,
237 pub(crate) commit_sync_fetch_once_latency: Histogram,
238 pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
239 pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
240 pub(crate) round_tracker_received_quorum_round_gaps: IntGaugeVec,
241 pub(crate) round_tracker_accepted_quorum_round_gaps: IntGaugeVec,
242 pub(crate) round_tracker_low_received_quorum_round: IntGaugeVec,
243 pub(crate) round_tracker_low_accepted_quorum_round: IntGaugeVec,
244 pub(crate) round_tracker_current_received_round_gaps: IntGaugeVec,
245 pub(crate) round_tracker_current_accepted_round_gaps: IntGaugeVec,
246 pub(crate) round_tracker_propagation_delays: Histogram,
247 pub(crate) round_tracker_last_propagation_delay: IntGauge,
248 pub(crate) round_prober_request_errors: IntCounterVec,
249 pub(crate) certifier_gc_round: IntGauge,
250 pub(crate) certifier_own_reject_votes: IntCounterVec,
251 pub(crate) finalizer_buffered_commits: IntGauge,
252 pub(crate) finalizer_round_delay: Histogram,
253 pub(crate) finalizer_transaction_status: IntCounterVec,
254 pub(crate) finalizer_reject_votes: IntCounterVec,
255 pub(crate) finalizer_output_commits: IntCounterVec,
256 pub(crate) finalizer_skipped_voting_blocks: IntCounterVec,
257 pub(crate) uptime: Histogram,
258}
259
260impl NodeMetrics {
261 pub(crate) fn new(registry: &Registry) -> Self {
262 Self {
263 authority_index: register_int_gauge_vec_with_registry!(
264 "authority_index",
265 "The index of the authority",
266 &["name"],
267 registry,
268 ).unwrap(),
269 protocol_version: register_int_gauge_with_registry!(
270 "protocol_version",
271 "The protocol version used in this epoch",
272 registry,
273 ).unwrap(),
274 block_commit_latency: register_histogram_with_registry!(
275 "block_commit_latency",
276 "The time taken between block creation and block commit.",
277 LATENCY_SEC_BUCKETS.to_vec(),
278 registry,
279 ).unwrap(),
280 proposed_blocks: register_int_counter_vec_with_registry!(
281 "proposed_blocks",
282 "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
283 &["force"],
284 registry,
285 ).unwrap(),
286 proposed_block_commit_latency: register_histogram_with_registry!(
287 "proposed_block_commit_latency",
288 "The time taken between block proposal and commit.",
289 LATENCY_SEC_BUCKETS.to_vec(),
290 registry,
291 ).unwrap(),
292 proposed_block_finalization_latency: register_histogram_with_registry!(
293 "proposed_block_finalization_latency",
294 "The time taken between block proposal and finalization.",
295 LATENCY_SEC_BUCKETS.to_vec(),
296 registry,
297 ).unwrap(),
298 proposed_block_size: register_histogram_with_registry!(
299 "proposed_block_size",
300 "The size (in bytes) of proposed blocks",
301 SIZE_BUCKETS.to_vec(),
302 registry
303 ).unwrap(),
304 proposed_block_transactions: register_histogram_with_registry!(
305 "proposed_block_transactions",
306 "# of transactions contained in proposed blocks",
307 NUM_BUCKETS.to_vec(),
308 registry
309 ).unwrap(),
310 proposed_block_transaction_bytes: register_histogram_with_registry!(
311 "proposed_block_transaction_bytes",
312 "Total size (in bytes) of transactions contained in proposed blocks",
313 SIZE_BUCKETS.to_vec(),
314 registry
315 ).unwrap(),
316 proposed_block_transaction_vote_blocks: register_histogram_with_registry!(
317 "proposed_block_transaction_vote_blocks",
318 "Number of blocks with reject votes contained in proposed blocks",
319 NUM_BUCKETS.to_vec(),
320 registry
321 ).unwrap(),
322 proposed_block_transaction_vote_entries: register_histogram_with_registry!(
323 "proposed_block_transaction_vote_entries",
324 "Number of reject vote entries contained in proposed blocks",
325 NUM_BUCKETS.to_vec(),
326 registry
327 ).unwrap(),
328 proposed_block_ancestors: register_histogram_with_registry!(
329 "proposed_block_ancestors",
330 "Number of ancestors in proposed blocks",
331 exponential_buckets(1.0, 1.4, 20).unwrap(),
332 registry,
333 ).unwrap(),
334 proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!(
335 "proposed_block_ancestors_timestamp_drift_ms",
336 "The drift in ms of ancestors' timestamps included in newly proposed blocks",
337 &["authority"],
338 registry,
339 ).unwrap(),
340 proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
341 "proposed_block_ancestors_depth",
342 "The depth in rounds of ancestors included in newly proposed blocks",
343 &["authority"],
344 exponential_buckets(1.0, 2.0, 14).unwrap(),
345 registry,
346 ).unwrap(),
347 highest_verified_authority_round: register_int_gauge_vec_with_registry!(
348 "highest_verified_authority_round",
349 "The highest round of verified block for the corresponding authority",
350 &["authority"],
351 registry,
352 ).unwrap(),
353 lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
354 "lowest_verified_authority_round",
355 "The lowest round of verified block for the corresponding authority",
356 &["authority"],
357 registry,
358 ).unwrap(),
359 block_proposal_interval: register_histogram_with_registry!(
360 "block_proposal_interval",
361 "Intervals (in secs) between block proposals.",
362 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
363 registry,
364 ).unwrap(),
365 block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
366 "block_proposal_leader_wait_ms",
367 "Total time in ms spent waiting for a leader when proposing blocks.",
368 &["authority"],
369 registry,
370 ).unwrap(),
371 block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
372 "block_proposal_leader_wait_count",
373 "Total times waiting for a leader when proposing blocks.",
374 &["authority"],
375 registry,
376 ).unwrap(),
377 block_timestamp_drift_ms: register_int_counter_vec_with_registry!(
378 "block_timestamp_drift_ms",
379 "The clock drift time between a received block and the current node's time.",
380 &["authority", "source"],
381 registry,
382 ).unwrap(),
383 blocks_per_commit_count: register_histogram_with_registry!(
384 "blocks_per_commit_count",
385 "The number of blocks per commit.",
386 NUM_BUCKETS.to_vec(),
387 registry,
388 ).unwrap(),
389 blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
390 "blocks_pruned_on_commit",
391 "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
392 &["authority", "commit_status"],
393 registry,
394 ).unwrap(),
395 block_sync_service_read_blocks_count: register_int_counter_with_registry!(
396 "block_sync_service_read_blocks_count",
397 "Number of times the block sync service needs to read blocks from the store",
398 registry,
399 ).unwrap(),
400 commit_observer_last_recovered_commit_index: register_int_gauge_with_registry!(
401 "commit_observer_last_recovered_commit_index",
402 "The last commit index recovered by the commit observer",
403 registry,
404 ).unwrap(),
405 core_add_blocks_batch_size: register_histogram_with_registry!(
406 "core_add_blocks_batch_size",
407 "The number of blocks received from Core for processing on a single batch",
408 NUM_BUCKETS.to_vec(),
409 registry,
410 ).unwrap(),
411 core_check_block_refs_batch_size: register_histogram_with_registry!(
412 "core_check_block_refs_batch_size",
413 "The number of excluded blocks received from Core for search on a single batch",
414 NUM_BUCKETS.to_vec(),
415 registry,
416 ).unwrap(),
417 core_lock_dequeued: register_int_counter_with_registry!(
418 "core_lock_dequeued",
419 "Number of dequeued core requests",
420 registry,
421 ).unwrap(),
422 core_lock_enqueued: register_int_counter_with_registry!(
423 "core_lock_enqueued",
424 "Number of enqueued core requests",
425 registry,
426 ).unwrap(),
427 priority_submission_backpressure: register_int_counter_with_registry!(
428 "priority_submission_backpressure",
429 "Number of high-priority submissions that found the reserved priority input channel full and had to wait for capacity",
430 registry,
431 ).unwrap(),
432 core_skipped_proposals: register_int_counter_vec_with_registry!(
433 "core_skipped_proposals",
434 "Number of proposals skipped in the Core, per reason",
435 &["reason"],
436 registry,
437 ).unwrap(),
438 handler_received_block_missing_ancestors: register_int_counter_vec_with_registry!(
439 "handler_received_block_missing_ancestors",
440 "Number of missing ancestors in blocks received in the handler, separated by peer",
441 &["authority"],
442 registry,
443 ).unwrap(),
444 highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
445 "highest_accepted_authority_round",
446 "The highest round where a block has been accepted per authority. Resets on restart.",
447 &["authority"],
448 registry,
449 ).unwrap(),
450 highest_accepted_round: register_int_gauge_with_registry!(
451 "highest_accepted_round",
452 "The highest round where a block has been accepted. Resets on restart.",
453 registry,
454 ).unwrap(),
455 accepted_block_time_drift_ms: register_int_counter_vec_with_registry!(
456 "accepted_block_time_drift_ms",
457 "The time drift in ms of an accepted block compared to local time",
458 &["authority"],
459 registry,
460 ).unwrap(),
461 accepted_blocks: register_int_counter_vec_with_registry!(
462 "accepted_blocks",
463 "Number of accepted blocks by source (own, others)",
464 &["source"],
465 registry,
466 ).unwrap(),
467 dag_state_recent_blocks: register_int_gauge_with_registry!(
468 "dag_state_recent_blocks",
469 "Number of recent blocks cached in the DagState",
470 registry,
471 ).unwrap(),
472 dag_state_recent_refs: register_int_gauge_with_registry!(
473 "dag_state_recent_refs",
474 "Number of recent refs cached in the DagState",
475 registry,
476 ).unwrap(),
477 dag_state_store_read_count: register_int_counter_vec_with_registry!(
478 "dag_state_store_read_count",
479 "Number of times DagState needs to read from store per operation type",
480 &["type"],
481 registry,
482 ).unwrap(),
483 dag_state_store_write_count: register_int_counter_with_registry!(
484 "dag_state_store_write_count",
485 "Number of times DagState needs to write to store",
486 registry,
487 ).unwrap(),
488 fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
489 "fetch_blocks_scheduler_inflight",
490 "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
491 registry,
492 ).unwrap(),
493 synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
494 "synchronizer_fetched_blocks_by_peer",
495 "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
496 &["peer", "type"],
497 registry,
498 ).unwrap(),
499 synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
500 "synchronizer_missing_blocks_by_authority",
501 "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
502 &["authority"],
503 registry,
504 ).unwrap(),
505 synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
506 "synchronizer_current_missing_blocks_by_authority",
507 "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
508 &["authority"],
509 registry,
510 ).unwrap(),
511 synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
512 "synchronizer_fetched_blocks_by_authority",
513 "Number of fetched blocks per block author via the synchronizer",
514 &["authority", "type"],
515 registry,
516 ).unwrap(),
517 synchronizer_fetch_failures: register_int_counter_vec_with_registry!(
518 "synchronizer_fetch_failures",
519 "Number of fetch failures against each peer",
520 &["peer", "type"],
521 registry,
522 ).unwrap(),
523 synchronizer_skipped_fetch_requests: register_int_counter_vec_with_registry!(
524 "synchronizer_skipped_fetch_requests",
525 "Number of fetch requests skipped against each peer, because the peer is saturated",
526 &["peer"],
527 registry,
528 ).unwrap(),
529 synchronizer_process_fetched_failures: register_int_counter_vec_with_registry!(
530 "synchronizer_process_fetched_failures",
531 "Number of failures for processing fetched blocks against each peer",
532 &["peer", "type"],
533 registry,
534 ).unwrap(),
535 synchronizer_periodic_sync_decision: register_int_counter_vec_with_registry!(
536 "synchronizer_periodic_sync_decision",
537 "Decision to run periodic sync",
538 &["decision", "reason"],
539 registry,
540 ).unwrap(),
541 network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
542 "network_received_excluded_ancestors_from_authority",
543 "Number of excluded ancestors received from each authority.",
544 &["authority"],
545 registry,
546 ).unwrap(),
547 network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
548 "network_excluded_ancestors_count_by_authority",
549 "Total number of excluded ancestors per authority.",
550 &["authority"],
551 registry,
552 ).unwrap(),
553 network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
554 "network_excluded_ancestors_sent_to_fetch",
555 "Number of excluded ancestors sent to fetch.",
556 &["authority"],
557 registry,
558 ).unwrap(),
559 last_known_own_block_round: register_int_gauge_with_registry!(
560 "last_known_own_block_round",
561 "The highest round of our own block as this has been synced from peers during an amnesia recovery",
562 registry,
563 ).unwrap(),
564 sync_last_known_own_block_retries: register_int_counter_with_registry!(
565 "sync_last_known_own_block_retries",
566 "Number of times this node tried to fetch the last own block from peers",
567 registry,
568 ).unwrap(),
569 invalid_blocks: register_int_counter_vec_with_registry!(
570 "invalid_blocks",
571 "Number of invalid blocks per peer authority",
572 &["authority", "source", "error"],
573 registry,
574 ).unwrap(),
575 rejected_blocks: register_int_counter_vec_with_registry!(
576 "rejected_blocks",
577 "Number of blocks rejected before verifications",
578 &["reason"],
579 registry,
580 ).unwrap(),
581 subscribed_blocks: register_int_counter_vec_with_registry!(
582 "subscribed_blocks",
583 "Number of blocks received from each peer before verification",
584 &["authority"],
585 registry,
586 ).unwrap(),
587 observer_subscribed_blocks_batch_size: register_histogram_with_registry!(
588 "observer_subscribed_blocks_batch_size",
589 "The number of blocks received from a peer before verification in a single batch",
590 NUM_BUCKETS.to_vec(),
591 registry,
592 ).unwrap(),
593 observer_subscription_suspended: register_int_gauge_with_registry!(
594 "observer_subscription_suspended",
595 "Whether the observer block stream subscription is currently suspended due to commit lag (1) or active (0)",
596 registry,
597 ).unwrap(),
598 verified_blocks: register_int_counter_vec_with_registry!(
599 "verified_blocks",
600 "Number of blocks received from each peer that are verified",
601 &["authority"],
602 registry,
603 ).unwrap(),
604 committed_leaders_total: register_int_counter_vec_with_registry!(
605 "committed_leaders_total",
606 "Total number of (direct or indirect) committed leaders per authority",
607 &["authority", "commit_type"],
608 registry,
609 ).unwrap(),
610 last_committed_authority_round: register_int_gauge_vec_with_registry!(
611 "last_committed_authority_round",
612 "The last round committed by authority.",
613 &["authority"],
614 registry,
615 ).unwrap(),
616 last_committed_leader_round: register_int_gauge_with_registry!(
617 "last_committed_leader_round",
618 "The last round where a leader was committed to store and sent to commit consumer.",
619 registry,
620 ).unwrap(),
621 last_commit_index: register_int_gauge_with_registry!(
622 "last_commit_index",
623 "Index of the last commit.",
624 registry,
625 ).unwrap(),
626 last_commit_time_diff: register_histogram_with_registry!(
627 "last_commit_time_diff",
628 "The time diff between the last commit and previous one.",
629 LATENCY_SEC_BUCKETS.to_vec(),
630 registry,
631 ).unwrap(),
632 commit_round_advancement_interval: register_histogram_with_registry!(
633 "commit_round_advancement_interval",
634 "Intervals (in secs) between commit round advancements.",
635 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
636 registry,
637 ).unwrap(),
638 last_decided_leader_round: register_int_gauge_with_registry!(
639 "last_decided_leader_round",
640 "The last round where a commit decision was made.",
641 registry,
642 ).unwrap(),
643 leader_timeout_total: register_int_counter_vec_with_registry!(
644 "leader_timeout_total",
645 "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
646 &["timeout_type"],
647 registry,
648 ).unwrap(),
649 smart_selection_wait: register_int_counter_with_registry!(
650 "smart_selection_wait",
651 "Number of times we waited for smart ancestor selection.",
652 registry,
653 ).unwrap(),
654 ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
655 "ancestor_state_change_by_authority",
656 "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
657 &["authority", "state"],
658 registry,
659 ).unwrap(),
660 excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
661 "excluded_proposal_ancestors_count_by_authority",
662 "Total number of excluded ancestors per authority during proposal.",
663 &["authority"],
664 registry,
665 ).unwrap(),
666 included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
667 "included_excluded_proposal_ancestors_count_by_authority",
668 "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
669 &["authority", "type"],
670 registry,
671 ).unwrap(),
672 missing_blocks_total: register_int_counter_with_registry!(
673 "missing_blocks_total",
674 "Total cumulative number of missing blocks",
675 registry,
676 ).unwrap(),
677 missing_blocks_after_fetch_total: register_int_counter_with_registry!(
678 "missing_blocks_after_fetch_total",
679 "Total number of missing blocks after fetching blocks from peer",
680 registry,
681 ).unwrap(),
682 num_of_bad_nodes: register_int_gauge_with_registry!(
683 "num_of_bad_nodes",
684 "The number of bad nodes in the new leader schedule",
685 registry
686 ).unwrap(),
687 quorum_receive_latency: register_histogram_with_registry!(
688 "quorum_receive_latency",
689 "The time it took to receive a new round quorum of blocks",
690 registry
691 ).unwrap(),
692 block_receive_delay: register_int_counter_vec_with_registry!(
693 "block_receive_delay",
694 "Total delay from the start of the round to receiving the block, in milliseconds per authority",
695 &["authority"],
696 registry,
697 ).unwrap(),
698 reputation_scores: register_int_gauge_vec_with_registry!(
699 "reputation_scores",
700 "Reputation scores for each authority",
701 &["authority"],
702 registry,
703 ).unwrap(),
704 leader_schedule_total_scores: register_int_gauge_vec_with_registry!(
705 "leader_schedule_total_scores",
706 "LeaderScheduleV3 running per-authority score over the scoring window. Each commit contributes voted_for_stake * certified_by_stake (stake^2 units)",
707 &["authority"],
708 registry,
709 ).unwrap(),
710 leader_schedule_normalized_scores: register_gauge_vec_with_registry!(
711 "leader_schedule_normalized_scores",
712 "LeaderScheduleV3 per-authority total score divided by (running sum of per-commit leader stake) * committee total stake. Stake^2 over stake^2; a fraction with no stake dimension",
713 &["authority"],
714 registry,
715 ).unwrap(),
716 leader_schedule_last_num_leaders: register_int_gauge_with_registry!(
717 "leader_schedule_last_num_leaders",
718 "LeaderScheduleV3 number of leader-round blocks in the most recently scored commit",
719 registry,
720 ).unwrap(),
721 leader_schedule_average_num_leaders: register_gauge_with_registry!(
722 "leader_schedule_average_num_leaders",
723 "LeaderScheduleV3 moving average of the number of leaders per commit across the scoring window",
724 registry,
725 ).unwrap(),
726 scope_processing_time: register_histogram_vec_with_registry!(
727 "scope_processing_time",
728 "The processing time of a specific code scope",
729 &["scope"],
730 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
731 registry
732 ).unwrap(),
733 sub_dags_per_commit_count: register_histogram_with_registry!(
734 "sub_dags_per_commit_count",
735 "The number of subdags per commit.",
736 registry,
737 ).unwrap(),
738 block_suspensions: register_int_counter_vec_with_registry!(
739 "block_suspensions",
740 "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
741 &["authority"],
742 registry,
743 ).unwrap(),
744 block_unsuspensions: register_int_counter_vec_with_registry!(
745 "block_unsuspensions",
746 "The number of block unsuspensions.",
747 &["authority"],
748 registry,
749 ).unwrap(),
750 suspended_block_time: register_histogram_vec_with_registry!(
751 "suspended_block_time",
752 "The time for which a block remains suspended",
753 &["authority"],
754 registry,
755 ).unwrap(),
756 block_manager_suspended_blocks: register_int_gauge_with_registry!(
757 "block_manager_suspended_blocks",
758 "The number of blocks currently suspended in the block manager",
759 registry,
760 ).unwrap(),
761 block_manager_missing_ancestors: register_int_gauge_with_registry!(
762 "block_manager_missing_ancestors",
763 "The number of missing ancestors tracked in the block manager",
764 registry,
765 ).unwrap(),
766 block_manager_missing_blocks: register_int_gauge_with_registry!(
767 "block_manager_missing_blocks",
768 "The number of blocks missing content tracked in the block manager",
769 registry,
770 ).unwrap(),
771 block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
772 "block_manager_missing_blocks_by_authority",
773 "The number of new missing blocks by block authority",
774 &["authority"],
775 registry,
776 ).unwrap(),
777 block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
778 "block_manager_missing_ancestors_by_authority",
779 "The number of missing ancestors by ancestor authority across received blocks",
780 &["authority"],
781 registry,
782 ).unwrap(),
783 block_manager_gced_blocks: register_int_counter_vec_with_registry!(
784 "block_manager_gced_blocks",
785 "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
786 &["authority"],
787 registry,
788 ).unwrap(),
789 block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
790 "block_manager_gc_unsuspended_blocks",
791 "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
792 &["authority"],
793 registry,
794 ).unwrap(),
795 block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
796 "block_manager_skipped_blocks",
797 "The number of blocks skipped by the block manager due to block round being <= gc_round",
798 &["authority"],
799 registry,
800 ).unwrap(),
801 threshold_clock_round: register_int_gauge_with_registry!(
802 "threshold_clock_round",
803 "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
804 registry,
805 ).unwrap(),
806 subscriber_connection_attempts: register_int_counter_vec_with_registry!(
807 "subscriber_connection_attempts",
808 "The number of connection attempts per peer",
809 &["authority", "status"],
810 registry,
811 ).unwrap(),
812 subscribed_to: register_int_gauge_vec_with_registry!(
813 "subscribed_to",
814 "Peers that this authority subscribed to for block streams.",
815 &["authority"],
816 registry,
817 ).unwrap(),
818 subscribed_by: register_int_gauge_vec_with_registry!(
819 "subscribed_by",
820 "Peers subscribing for block streams from this authority.",
821 &["authority"],
822 registry,
823 ).unwrap(),
824 commit_sync_inflight_fetches: register_int_gauge_with_registry!(
825 "commit_sync_inflight_fetches",
826 "The number of inflight fetches in commit syncer",
827 registry,
828 ).unwrap(),
829 commit_sync_pending_fetches: register_int_gauge_with_registry!(
830 "commit_sync_pending_fetches",
831 "The number of pending fetches in commit syncer",
832 registry,
833 ).unwrap(),
834 commit_sync_fetched_commits: register_int_counter_with_registry!(
835 "commit_sync_fetched_commits",
836 "The number of commits fetched via commit syncer",
837 registry,
838 ).unwrap(),
839 commit_sync_fetched_blocks: register_int_counter_with_registry!(
840 "commit_sync_fetched_blocks",
841 "The number of blocks fetched via commit syncer",
842 registry,
843 ).unwrap(),
844 commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
845 "commit_sync_total_fetched_blocks_size",
846 "The total size in bytes of blocks fetched via commit syncer",
847 registry,
848 ).unwrap(),
849 commit_sync_quorum_index: register_int_gauge_with_registry!(
850 "commit_sync_quorum_index",
851 "The maximum commit index voted by a quorum of authorities",
852 registry,
853 ).unwrap(),
854 commit_sync_highest_synced_index: register_int_gauge_with_registry!(
855 "commit_sync_fetched_index",
856 "The max commit index among local and fetched commits",
857 registry,
858 ).unwrap(),
859 commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
860 "commit_sync_highest_fetched_index",
861 "The max commit index that has been fetched via network",
862 registry,
863 ).unwrap(),
864 commit_sync_local_index: register_int_gauge_with_registry!(
865 "commit_sync_local_index",
866 "The local commit index",
867 registry,
868 ).unwrap(),
869 commit_sync_gap_on_processing: register_int_counter_with_registry!(
870 "commit_sync_gap_on_processing",
871 "Number of instances where a gap was found in fetched commit processing",
872 registry,
873 ).unwrap(),
874 commit_sync_fetch_loop_latency: register_histogram_with_registry!(
875 "commit_sync_fetch_loop_latency",
876 "The time taken to finish fetching commits and blocks from a given range",
877 LATENCY_SEC_BUCKETS.to_vec(),
878 registry,
879 ).unwrap(),
880 commit_sync_fetch_once_latency: register_histogram_with_registry!(
881 "commit_sync_fetch_once_latency",
882 "The time taken to fetch commits and blocks once",
883 LATENCY_SEC_BUCKETS.to_vec(),
884 registry,
885 ).unwrap(),
886 commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
887 "commit_sync_fetch_once_errors",
888 "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
889 &["authority", "error"],
890 registry
891 ).unwrap(),
892 commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
893 "commit_sync_fetch_missing_blocks",
894 "Number of ancestor blocks that are missing when processing blocks via commit sync.",
895 &["authority"],
896 registry,
897 ).unwrap(),
898 commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
899 "commit_sync_fetch_commits_handler_uncertified_skipped",
900 "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
901 registry,
902 ).unwrap(),
903 round_tracker_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
904 "round_tracker_received_quorum_round_gaps",
905 "Received round gaps among peers for blocks proposed from each authority",
906 &["authority"],
907 registry
908 ).unwrap(),
909 round_tracker_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
910 "round_tracker_accepted_quorum_round_gaps",
911 "Accepted round gaps among peers for blocks proposed & accepted from each authority",
912 &["authority"],
913 registry
914 ).unwrap(),
915 round_tracker_low_received_quorum_round: register_int_gauge_vec_with_registry!(
916 "round_tracker_low_received_quorum_round",
917 "Low quorum round among peers for blocks proposed from each authority",
918 &["authority"],
919 registry
920 ).unwrap(),
921 round_tracker_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
922 "round_tracker_low_accepted_quorum_round",
923 "Low quorum round among peers for blocks proposed & accepted from each authority",
924 &["authority"],
925 registry
926 ).unwrap(),
927 round_tracker_current_received_round_gaps: register_int_gauge_vec_with_registry!(
928 "round_tracker_current_received_round_gaps",
929 "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
930 &["authority"],
931 registry
932 ).unwrap(),
933 round_tracker_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
934 "round_tracker_current_accepted_round_gaps",
935 "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
936 &["authority"],
937 registry
938 ).unwrap(),
939 round_tracker_propagation_delays: register_histogram_with_registry!(
940 "round_tracker_propagation_delays",
941 "Round gaps between the last proposed block round and the lower bound of own quorum round",
942 NUM_BUCKETS.to_vec(),
943 registry
944 ).unwrap(),
945 round_tracker_last_propagation_delay: register_int_gauge_with_registry!(
946 "round_tracker_last_propagation_delay",
947 "Most recent propagation delay observed by RoundTracker",
948 registry
949 ).unwrap(),
950 round_prober_request_errors: register_int_counter_vec_with_registry!(
951 "round_prober_request_errors",
952 "Number of errors when probing against peers per error type",
953 &["error_type"],
954 registry
955 ).unwrap(),
956 certifier_gc_round: register_int_gauge_with_registry!(
957 "certifier_gc_round",
958 "The current GC round of the certifier",
959 registry
960 ).unwrap(),
961 certifier_own_reject_votes: register_int_counter_vec_with_registry!(
962 "certifier_own_reject_votes",
963 "Number of own reject votes against each peer authority",
964 &["authority"],
965 registry
966 ).unwrap(),
967 finalizer_buffered_commits: register_int_gauge_with_registry!(
968 "finalizer_buffered_commits",
969 "The number of commits buffered in the finalizer",
970 registry,
971 ).unwrap(),
972 finalizer_round_delay: register_histogram_with_registry!(
973 "finalizer_round_delay",
974 "The delay between the round of the last committed block and the round of the finalized commit.",
975 ROUND_DELAY_BUCKETS.to_vec(),
976 registry,
977 ).unwrap(),
978 finalizer_transaction_status: register_int_counter_vec_with_registry!(
979 "finalizer_transaction_status",
980 "Number of transactions finalized by the finalizer, grouped by status.",
981 &["status"],
982 registry
983 ).unwrap(),
984 finalizer_reject_votes: register_int_counter_vec_with_registry!(
985 "finalizer_reject_votes",
986 "Number of reject votes casted by each authority observed by the finalizer.",
987 &["authority"],
988 registry
989 ).unwrap(),
990 finalizer_output_commits: register_int_counter_vec_with_registry!(
991 "finalizer_output_commits",
992 "Number of output commits finalized by the finalizer, grouped by type.",
993 &["type"],
994 registry
995 ).unwrap(),
996 finalizer_skipped_voting_blocks: register_int_counter_vec_with_registry!(
997 "finalizer_skipped_voting_blocks",
998 "Number of times where another block skipped voting due to potentially out of GC bound. Authority is from the voted block.",
999 &["authority", "type"],
1000 registry
1001 ).unwrap(),
1002 uptime: register_histogram_with_registry!(
1003 "uptime",
1004 "Total node uptime",
1005 LATENCY_SEC_BUCKETS.to_vec(),
1006 registry,
1007 ).unwrap(),
1008 }
1009 }
1010}