1use std::sync::Arc;
5
6use prometheus::{
7 Gauge, GaugeVec, Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
8 Registry, exponential_buckets, register_gauge_vec_with_registry, register_gauge_with_registry,
9 register_histogram_vec_with_registry, register_histogram_with_registry,
10 register_int_counter_vec_with_registry, register_int_counter_with_registry,
11 register_int_gauge_vec_with_registry, register_int_gauge_with_registry,
12};
13
14use crate::network::metrics::NetworkMetrics;
15
16const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[
18 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25,
19 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5,
20 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120.,
21];
22
23const NUM_BUCKETS: &[f64] = &[
24 1.0,
25 2.0,
26 4.0,
27 8.0,
28 10.0,
29 20.0,
30 40.0,
31 80.0,
32 100.0,
33 150.0,
34 200.0,
35 250.0,
36 300.0,
37 400.0,
38 500.0,
39 600.0,
40 700.0,
41 800.0,
42 900.0,
43 1000.0,
44 1500.0,
45 2000.0,
46 2500.0,
47 3000.0,
48 4000.0,
49 5000.0,
50 6000.0,
51 7000.0,
52 8000.0,
53 9000.0,
54 10000.0,
55 15000.0,
56 20000.0,
57 25000.0,
58 30000.0,
59 40000.0,
60 50000.0,
61 100_000.0,
62 200_000.0,
63 300_000.0,
64 500_000.0,
65 1_000_000.0,
66];
67
68const LATENCY_SEC_BUCKETS: &[f64] = &[
69 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9,
70 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5,
71 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300.,
72];
73
74const SIZE_BUCKETS: &[f64] = &[
75 100.,
76 400.,
77 800.,
78 1_000.,
79 2_000.,
80 5_000.,
81 10_000.,
82 20_000.,
83 50_000.,
84 100_000.,
85 150_000.,
86 200_000.0,
87 250_000.0,
88 300_000.0,
89 350_000.0,
90 400_000.0,
91 450_000.0,
92 500_000.0,
93 600_000.0,
94 700_000.0,
95 800_000.0,
96 900_000.0,
97 1_000_000.0,
98 2_000_000.0,
99 3_000_000.0,
100 5_000_000.0,
101 10_000_000.0,
102]; const ROUND_DELAY_BUCKETS: &[f64] = &[0.0, 0.5, 1.0, 2.0, 3.0, 4.0];
106
107pub struct Metrics {
108 pub(crate) node_metrics: NodeMetrics,
109 pub(crate) network_metrics: NetworkMetrics,
110}
111
112pub(crate) fn initialise_metrics(registry: Registry) -> Arc<Metrics> {
113 let node_metrics = NodeMetrics::new(®istry);
114 let network_metrics = NetworkMetrics::new(®istry);
115
116 Arc::new(Metrics {
117 node_metrics,
118 network_metrics,
119 })
120}
121
122pub(crate) fn test_metrics() -> Arc<Metrics> {
123 initialise_metrics(Registry::new())
124}
125
126pub(crate) struct NodeMetrics {
127 pub(crate) authority_index: IntGaugeVec,
128 pub(crate) protocol_version: IntGauge,
129 pub(crate) block_commit_latency: Histogram,
130 pub(crate) proposed_blocks: IntCounterVec,
131 pub(crate) subscribe_blocks_response_bytes: IntCounterVec,
132 pub(crate) subscribe_stream_form_failures: IntCounterVec,
133 pub(crate) proposed_block_commit_latency: Histogram,
134 pub(crate) proposed_block_finalization_latency: Histogram,
135 pub(crate) proposed_block_size: Histogram,
136 pub(crate) proposed_block_transactions: Histogram,
137 pub(crate) proposed_block_transaction_bytes: Histogram,
138 pub(crate) proposed_block_transaction_vote_blocks: Histogram,
139 pub(crate) proposed_block_transaction_vote_entries: Histogram,
140 pub(crate) proposed_block_ancestors: Histogram,
141 pub(crate) proposed_block_ancestors_depth: HistogramVec,
142 pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec,
143 pub(crate) highest_verified_authority_round: IntGaugeVec,
144 pub(crate) lowest_verified_authority_round: IntGaugeVec,
145 pub(crate) block_proposal_interval: Histogram,
146 pub(crate) block_proposal_leader_wait_ms: IntCounterVec,
147 pub(crate) block_proposal_leader_wait_count: IntCounterVec,
148 pub(crate) block_timestamp_drift_ms: IntCounterVec,
149 pub(crate) blocks_per_commit_count: Histogram,
150 pub(crate) blocks_pruned_on_commit: IntCounterVec,
151 pub(crate) block_sync_service_read_blocks_count: IntCounter,
152 pub(crate) commit_observer_last_recovered_commit_index: IntGauge,
153 pub(crate) core_add_blocks_batch_size: Histogram,
154 pub(crate) core_check_block_refs_batch_size: Histogram,
155 pub(crate) core_lock_dequeued: IntCounter,
156 pub(crate) core_lock_enqueued: IntCounter,
157 pub(crate) priority_submission_backpressure: IntCounter,
158 pub(crate) core_skipped_proposals: IntCounterVec,
159 pub(crate) core_certified_commits_processed: IntCounterVec,
160 pub(crate) handler_received_block_missing_ancestors: IntCounterVec,
161 pub(crate) highest_accepted_authority_round: IntGaugeVec,
162 pub(crate) highest_accepted_round: IntGauge,
163 pub(crate) accepted_block_time_drift_ms: IntCounterVec,
164 pub(crate) accepted_blocks: IntCounterVec,
165 pub(crate) dag_state_recent_blocks: IntGauge,
166 pub(crate) dag_state_recent_refs: IntGauge,
167 pub(crate) dag_state_store_read_count: IntCounterVec,
168 pub(crate) dag_state_store_write_count: IntCounter,
169 pub(crate) fetch_blocks_scheduler_inflight: IntGauge,
170 pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec,
171 pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec,
172 pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec,
173 pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec,
174 pub(crate) synchronizer_fetch_failures: IntCounterVec,
175 pub(crate) synchronizer_skipped_fetch_requests: IntCounterVec,
176 pub(crate) synchronizer_process_fetched_failures: IntCounterVec,
177 pub(crate) synchronizer_periodic_sync_decision: IntCounterVec,
178 pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec,
179 pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec,
180 pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec,
181 pub(crate) invalid_blocks: IntCounterVec,
182 pub(crate) rejected_blocks: IntCounterVec,
183 pub(crate) subscribed_blocks: IntCounterVec,
184 pub(crate) observer_subscribed_blocks_batch_size: Histogram,
185 pub(crate) observer_subscription_suspended: IntGauge,
186 pub(crate) verified_blocks: IntCounterVec,
187 pub(crate) committed_leaders_total: IntCounterVec,
188 pub(crate) last_committed_authority_round: IntGaugeVec,
189 pub(crate) last_committed_leader_round: IntGauge,
190 pub(crate) last_commit_index: IntGauge,
191 pub(crate) last_commit_time_diff: Histogram,
192 pub(crate) last_known_own_block_round: IntGauge,
193 pub(crate) sync_last_known_own_block_retries: IntCounter,
194 pub(crate) commit_round_advancement_interval: Histogram,
195 pub(crate) last_decided_leader_round: IntGauge,
196 pub(crate) leader_timeout_total: IntCounterVec,
197 pub(crate) smart_selection_wait: IntCounter,
198 pub(crate) ancestor_state_change_by_authority: IntCounterVec,
199 pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec,
200 pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec,
201 pub(crate) missing_blocks_total: IntCounter,
202 pub(crate) missing_blocks_after_fetch_total: IntCounter,
203 pub(crate) num_of_bad_nodes: IntGauge,
204 pub(crate) quorum_receive_latency: Histogram,
205 pub(crate) block_receive_delay: IntCounterVec,
206 pub(crate) reputation_scores: IntGaugeVec,
207 pub(crate) leader_schedule_total_scores: IntGaugeVec,
208 pub(crate) leader_schedule_normalized_scores: GaugeVec,
209 pub(crate) leader_schedule_last_num_leaders: IntGauge,
210 pub(crate) leader_schedule_average_num_leaders: Gauge,
211 pub(crate) scope_processing_time: HistogramVec,
212 pub(crate) sub_dags_per_commit_count: Histogram,
213 pub(crate) block_suspensions: IntCounterVec,
214 pub(crate) block_unsuspensions: IntCounterVec,
215 pub(crate) suspended_block_time: HistogramVec,
216 pub(crate) block_manager_suspended_blocks: IntGauge,
217 pub(crate) block_manager_missing_ancestors: IntGauge,
218 pub(crate) block_manager_missing_blocks: IntGauge,
219 pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec,
220 pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec,
221 pub(crate) block_manager_gced_blocks: IntCounterVec,
222 pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec,
223 pub(crate) block_manager_skipped_blocks: IntCounterVec,
224 pub(crate) threshold_clock_round: IntGauge,
225 pub(crate) subscriber_connection_attempts: IntCounterVec,
226 pub(crate) subscribed_to: IntGaugeVec,
227 pub(crate) subscribed_by: IntGaugeVec,
228 pub(crate) commit_sync_inflight_fetches: IntGauge,
229 pub(crate) commit_sync_pending_fetches: IntGauge,
230 pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter,
231 pub(crate) commit_sync_fetched_commits: IntCounter,
232 pub(crate) commit_sync_fetched_blocks: IntCounter,
233 pub(crate) commit_sync_total_fetched_blocks_size: IntCounter,
234 pub(crate) commit_sync_quorum_index: IntGauge,
235 pub(crate) commit_sync_highest_synced_index: IntGauge,
236 pub(crate) commit_sync_highest_fetched_index: IntGauge,
237 pub(crate) commit_sync_local_index: IntGauge,
238 pub(crate) commit_sync_gap_on_processing: IntCounter,
239 pub(crate) commit_sync_fetch_loop_latency: Histogram,
240 pub(crate) commit_sync_fetch_once_latency: Histogram,
241 pub(crate) commit_sync_fetch_once_errors: IntCounterVec,
242 pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec,
243 pub(crate) round_tracker_received_quorum_round_gaps: IntGaugeVec,
244 pub(crate) round_tracker_accepted_quorum_round_gaps: IntGaugeVec,
245 pub(crate) round_tracker_low_received_quorum_round: IntGaugeVec,
246 pub(crate) round_tracker_low_accepted_quorum_round: IntGaugeVec,
247 pub(crate) round_tracker_current_received_round_gaps: IntGaugeVec,
248 pub(crate) round_tracker_current_accepted_round_gaps: IntGaugeVec,
249 pub(crate) round_tracker_propagation_delays: Histogram,
250 pub(crate) round_tracker_last_propagation_delay: IntGauge,
251 pub(crate) round_prober_request_errors: IntCounterVec,
252 pub(crate) certifier_gc_round: IntGauge,
253 pub(crate) certifier_own_reject_votes: IntCounterVec,
254 pub(crate) finalizer_buffered_commits: IntGauge,
255 pub(crate) finalizer_round_delay: Histogram,
256 pub(crate) finalizer_transaction_status: IntCounterVec,
257 pub(crate) finalizer_reject_votes: IntCounterVec,
258 pub(crate) finalizer_output_commits: IntCounterVec,
259 pub(crate) finalizer_skipped_voting_blocks: IntCounterVec,
260 pub(crate) uptime: Histogram,
261}
262
263impl NodeMetrics {
264 pub(crate) fn new(registry: &Registry) -> Self {
265 Self {
266 authority_index: register_int_gauge_vec_with_registry!(
267 "authority_index",
268 "The index of the authority",
269 &["name"],
270 registry,
271 ).unwrap(),
272 protocol_version: register_int_gauge_with_registry!(
273 "protocol_version",
274 "The protocol version used in this epoch",
275 registry,
276 ).unwrap(),
277 block_commit_latency: register_histogram_with_registry!(
278 "block_commit_latency",
279 "The time taken between block creation and block commit.",
280 LATENCY_SEC_BUCKETS.to_vec(),
281 registry,
282 ).unwrap(),
283 subscribe_blocks_response_bytes: register_int_counter_vec_with_registry!(
284 "subscribe_blocks_response_bytes",
285 "Encoded protobuf length of subscription responses, by subscriber and block form. Measured at message construction -- upstream of the stream throttle, the gRPC length prefix and framing, and zstd -- so it tracks payload form, not bytes on the socket.",
286 &["authority", "form"],
287 registry,
288 ).unwrap(),
289 subscribe_stream_form_failures: register_int_counter_vec_with_registry!(
290 "subscribe_stream_form_failures",
291 "Subscription payloads whose wire form was unusable, per peer and reason",
292 &["authority", "reason"],
293 registry,
294 ).unwrap(),
295 proposed_blocks: register_int_counter_vec_with_registry!(
296 "proposed_blocks",
297 "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.",
298 &["force"],
299 registry,
300 ).unwrap(),
301 proposed_block_commit_latency: register_histogram_with_registry!(
302 "proposed_block_commit_latency",
303 "The time taken between block proposal and commit.",
304 LATENCY_SEC_BUCKETS.to_vec(),
305 registry,
306 ).unwrap(),
307 proposed_block_finalization_latency: register_histogram_with_registry!(
308 "proposed_block_finalization_latency",
309 "The time taken between block proposal and finalization.",
310 LATENCY_SEC_BUCKETS.to_vec(),
311 registry,
312 ).unwrap(),
313 proposed_block_size: register_histogram_with_registry!(
314 "proposed_block_size",
315 "The size (in bytes) of proposed blocks",
316 SIZE_BUCKETS.to_vec(),
317 registry
318 ).unwrap(),
319 proposed_block_transactions: register_histogram_with_registry!(
320 "proposed_block_transactions",
321 "# of transactions contained in proposed blocks",
322 NUM_BUCKETS.to_vec(),
323 registry
324 ).unwrap(),
325 proposed_block_transaction_bytes: register_histogram_with_registry!(
326 "proposed_block_transaction_bytes",
327 "Total size (in bytes) of transactions contained in proposed blocks",
328 SIZE_BUCKETS.to_vec(),
329 registry
330 ).unwrap(),
331 proposed_block_transaction_vote_blocks: register_histogram_with_registry!(
332 "proposed_block_transaction_vote_blocks",
333 "Number of blocks with reject votes contained in proposed blocks",
334 NUM_BUCKETS.to_vec(),
335 registry
336 ).unwrap(),
337 proposed_block_transaction_vote_entries: register_histogram_with_registry!(
338 "proposed_block_transaction_vote_entries",
339 "Number of reject vote entries contained in proposed blocks",
340 NUM_BUCKETS.to_vec(),
341 registry
342 ).unwrap(),
343 proposed_block_ancestors: register_histogram_with_registry!(
344 "proposed_block_ancestors",
345 "Number of ancestors in proposed blocks",
346 exponential_buckets(1.0, 1.4, 20).unwrap(),
347 registry,
348 ).unwrap(),
349 proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!(
350 "proposed_block_ancestors_timestamp_drift_ms",
351 "The drift in ms of ancestors' timestamps included in newly proposed blocks",
352 &["authority"],
353 registry,
354 ).unwrap(),
355 proposed_block_ancestors_depth: register_histogram_vec_with_registry!(
356 "proposed_block_ancestors_depth",
357 "The depth in rounds of ancestors included in newly proposed blocks",
358 &["authority"],
359 exponential_buckets(1.0, 2.0, 14).unwrap(),
360 registry,
361 ).unwrap(),
362 highest_verified_authority_round: register_int_gauge_vec_with_registry!(
363 "highest_verified_authority_round",
364 "The highest round of verified block for the corresponding authority",
365 &["authority"],
366 registry,
367 ).unwrap(),
368 lowest_verified_authority_round: register_int_gauge_vec_with_registry!(
369 "lowest_verified_authority_round",
370 "The lowest round of verified block for the corresponding authority",
371 &["authority"],
372 registry,
373 ).unwrap(),
374 block_proposal_interval: register_histogram_with_registry!(
375 "block_proposal_interval",
376 "Intervals (in secs) between block proposals.",
377 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
378 registry,
379 ).unwrap(),
380 block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!(
381 "block_proposal_leader_wait_ms",
382 "Total time in ms spent waiting for a leader when proposing blocks.",
383 &["authority"],
384 registry,
385 ).unwrap(),
386 block_proposal_leader_wait_count: register_int_counter_vec_with_registry!(
387 "block_proposal_leader_wait_count",
388 "Total times waiting for a leader when proposing blocks.",
389 &["authority"],
390 registry,
391 ).unwrap(),
392 block_timestamp_drift_ms: register_int_counter_vec_with_registry!(
393 "block_timestamp_drift_ms",
394 "The clock drift time between a received block and the current node's time.",
395 &["authority", "source"],
396 registry,
397 ).unwrap(),
398 blocks_per_commit_count: register_histogram_with_registry!(
399 "blocks_per_commit_count",
400 "The number of blocks per commit.",
401 NUM_BUCKETS.to_vec(),
402 registry,
403 ).unwrap(),
404 blocks_pruned_on_commit: register_int_counter_vec_with_registry!(
405 "blocks_pruned_on_commit",
406 "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.",
407 &["authority", "commit_status"],
408 registry,
409 ).unwrap(),
410 block_sync_service_read_blocks_count: register_int_counter_with_registry!(
411 "block_sync_service_read_blocks_count",
412 "Number of times the block sync service needs to read blocks from the store",
413 registry,
414 ).unwrap(),
415 commit_observer_last_recovered_commit_index: register_int_gauge_with_registry!(
416 "commit_observer_last_recovered_commit_index",
417 "The last commit index recovered by the commit observer",
418 registry,
419 ).unwrap(),
420 core_add_blocks_batch_size: register_histogram_with_registry!(
421 "core_add_blocks_batch_size",
422 "The number of blocks received from Core for processing on a single batch",
423 NUM_BUCKETS.to_vec(),
424 registry,
425 ).unwrap(),
426 core_check_block_refs_batch_size: register_histogram_with_registry!(
427 "core_check_block_refs_batch_size",
428 "The number of excluded blocks received from Core for search on a single batch",
429 NUM_BUCKETS.to_vec(),
430 registry,
431 ).unwrap(),
432 core_lock_dequeued: register_int_counter_with_registry!(
433 "core_lock_dequeued",
434 "Number of dequeued core requests",
435 registry,
436 ).unwrap(),
437 core_lock_enqueued: register_int_counter_with_registry!(
438 "core_lock_enqueued",
439 "Number of enqueued core requests",
440 registry,
441 ).unwrap(),
442 priority_submission_backpressure: register_int_counter_with_registry!(
443 "priority_submission_backpressure",
444 "Number of high-priority submissions that found the reserved priority input channel full and had to wait for capacity",
445 registry,
446 ).unwrap(),
447 core_skipped_proposals: register_int_counter_vec_with_registry!(
448 "core_skipped_proposals",
449 "Number of proposals skipped in the Core, per reason",
450 &["reason"],
451 registry,
452 ).unwrap(),
453 core_certified_commits_processed: register_int_counter_vec_with_registry!(
454 "core_certified_commits_processed",
455 "Number of certified commits processed by the Core",
456 &["result"],
457 registry,
458 ).unwrap(),
459 handler_received_block_missing_ancestors: register_int_counter_vec_with_registry!(
460 "handler_received_block_missing_ancestors",
461 "Number of missing ancestors in blocks received in the handler, separated by peer",
462 &["authority"],
463 registry,
464 ).unwrap(),
465 highest_accepted_authority_round: register_int_gauge_vec_with_registry!(
466 "highest_accepted_authority_round",
467 "The highest round where a block has been accepted per authority. Resets on restart.",
468 &["authority"],
469 registry,
470 ).unwrap(),
471 highest_accepted_round: register_int_gauge_with_registry!(
472 "highest_accepted_round",
473 "The highest round where a block has been accepted. Resets on restart.",
474 registry,
475 ).unwrap(),
476 accepted_block_time_drift_ms: register_int_counter_vec_with_registry!(
477 "accepted_block_time_drift_ms",
478 "The time drift in ms of an accepted block compared to local time",
479 &["authority"],
480 registry,
481 ).unwrap(),
482 accepted_blocks: register_int_counter_vec_with_registry!(
483 "accepted_blocks",
484 "Number of accepted blocks by source (own, others)",
485 &["source"],
486 registry,
487 ).unwrap(),
488 dag_state_recent_blocks: register_int_gauge_with_registry!(
489 "dag_state_recent_blocks",
490 "Number of recent blocks cached in the DagState",
491 registry,
492 ).unwrap(),
493 dag_state_recent_refs: register_int_gauge_with_registry!(
494 "dag_state_recent_refs",
495 "Number of recent refs cached in the DagState",
496 registry,
497 ).unwrap(),
498 dag_state_store_read_count: register_int_counter_vec_with_registry!(
499 "dag_state_store_read_count",
500 "Number of times DagState needs to read from store per operation type",
501 &["type"],
502 registry,
503 ).unwrap(),
504 dag_state_store_write_count: register_int_counter_with_registry!(
505 "dag_state_store_write_count",
506 "Number of times DagState needs to write to store",
507 registry,
508 ).unwrap(),
509 fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!(
510 "fetch_blocks_scheduler_inflight",
511 "Designates whether the synchronizer scheduler task to fetch blocks is currently running",
512 registry,
513 ).unwrap(),
514 synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!(
515 "synchronizer_fetched_blocks_by_peer",
516 "Number of fetched blocks per peer authority via the synchronizer and also by block authority",
517 &["peer", "type"],
518 registry,
519 ).unwrap(),
520 synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
521 "synchronizer_missing_blocks_by_authority",
522 "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
523 &["authority"],
524 registry,
525 ).unwrap(),
526 synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!(
527 "synchronizer_current_missing_blocks_by_authority",
528 "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.",
529 &["authority"],
530 registry,
531 ).unwrap(),
532 synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!(
533 "synchronizer_fetched_blocks_by_authority",
534 "Number of fetched blocks per block author via the synchronizer",
535 &["authority", "type"],
536 registry,
537 ).unwrap(),
538 synchronizer_fetch_failures: register_int_counter_vec_with_registry!(
539 "synchronizer_fetch_failures",
540 "Number of fetch failures against each peer",
541 &["peer", "type"],
542 registry,
543 ).unwrap(),
544 synchronizer_skipped_fetch_requests: register_int_counter_vec_with_registry!(
545 "synchronizer_skipped_fetch_requests",
546 "Number of fetch requests skipped against each peer, because the peer is saturated",
547 &["peer"],
548 registry,
549 ).unwrap(),
550 synchronizer_process_fetched_failures: register_int_counter_vec_with_registry!(
551 "synchronizer_process_fetched_failures",
552 "Number of failures for processing fetched blocks against each peer",
553 &["peer", "type"],
554 registry,
555 ).unwrap(),
556 synchronizer_periodic_sync_decision: register_int_counter_vec_with_registry!(
557 "synchronizer_periodic_sync_decision",
558 "Decision to run periodic sync",
559 &["decision", "reason"],
560 registry,
561 ).unwrap(),
562 network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!(
563 "network_received_excluded_ancestors_from_authority",
564 "Number of excluded ancestors received from each authority.",
565 &["authority"],
566 registry,
567 ).unwrap(),
568 network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
569 "network_excluded_ancestors_count_by_authority",
570 "Total number of excluded ancestors per authority.",
571 &["authority"],
572 registry,
573 ).unwrap(),
574 network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!(
575 "network_excluded_ancestors_sent_to_fetch",
576 "Number of excluded ancestors sent to fetch.",
577 &["authority"],
578 registry,
579 ).unwrap(),
580 last_known_own_block_round: register_int_gauge_with_registry!(
581 "last_known_own_block_round",
582 "The highest round of our own block as this has been synced from peers during an amnesia recovery",
583 registry,
584 ).unwrap(),
585 sync_last_known_own_block_retries: register_int_counter_with_registry!(
586 "sync_last_known_own_block_retries",
587 "Number of times this node tried to fetch the last own block from peers",
588 registry,
589 ).unwrap(),
590 invalid_blocks: register_int_counter_vec_with_registry!(
591 "invalid_blocks",
592 "Number of invalid blocks per peer authority",
593 &["authority", "source", "error"],
594 registry,
595 ).unwrap(),
596 rejected_blocks: register_int_counter_vec_with_registry!(
597 "rejected_blocks",
598 "Number of blocks rejected before verifications",
599 &["reason"],
600 registry,
601 ).unwrap(),
602 subscribed_blocks: register_int_counter_vec_with_registry!(
603 "subscribed_blocks",
604 "Number of blocks received from each peer before verification",
605 &["authority"],
606 registry,
607 ).unwrap(),
608 observer_subscribed_blocks_batch_size: register_histogram_with_registry!(
609 "observer_subscribed_blocks_batch_size",
610 "The number of blocks received from a peer before verification in a single batch",
611 NUM_BUCKETS.to_vec(),
612 registry,
613 ).unwrap(),
614 observer_subscription_suspended: register_int_gauge_with_registry!(
615 "observer_subscription_suspended",
616 "Whether the observer block stream subscription is currently suspended due to commit lag (1) or active (0)",
617 registry,
618 ).unwrap(),
619 verified_blocks: register_int_counter_vec_with_registry!(
620 "verified_blocks",
621 "Number of blocks received from each peer that are verified",
622 &["authority"],
623 registry,
624 ).unwrap(),
625 committed_leaders_total: register_int_counter_vec_with_registry!(
626 "committed_leaders_total",
627 "Total number of (direct or indirect) committed leaders per authority",
628 &["authority", "commit_type"],
629 registry,
630 ).unwrap(),
631 last_committed_authority_round: register_int_gauge_vec_with_registry!(
632 "last_committed_authority_round",
633 "The last round committed by authority.",
634 &["authority"],
635 registry,
636 ).unwrap(),
637 last_committed_leader_round: register_int_gauge_with_registry!(
638 "last_committed_leader_round",
639 "The last round where a leader was committed to store and sent to commit consumer.",
640 registry,
641 ).unwrap(),
642 last_commit_index: register_int_gauge_with_registry!(
643 "last_commit_index",
644 "Index of the last commit.",
645 registry,
646 ).unwrap(),
647 last_commit_time_diff: register_histogram_with_registry!(
648 "last_commit_time_diff",
649 "The time diff between the last commit and previous one.",
650 LATENCY_SEC_BUCKETS.to_vec(),
651 registry,
652 ).unwrap(),
653 commit_round_advancement_interval: register_histogram_with_registry!(
654 "commit_round_advancement_interval",
655 "Intervals (in secs) between commit round advancements.",
656 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
657 registry,
658 ).unwrap(),
659 last_decided_leader_round: register_int_gauge_with_registry!(
660 "last_decided_leader_round",
661 "The last round where a commit decision was made.",
662 registry,
663 ).unwrap(),
664 leader_timeout_total: register_int_counter_vec_with_registry!(
665 "leader_timeout_total",
666 "Total number of leader timeouts, either when the min round time has passed, or max leader timeout",
667 &["timeout_type"],
668 registry,
669 ).unwrap(),
670 smart_selection_wait: register_int_counter_with_registry!(
671 "smart_selection_wait",
672 "Number of times we waited for smart ancestor selection.",
673 registry,
674 ).unwrap(),
675 ancestor_state_change_by_authority: register_int_counter_vec_with_registry!(
676 "ancestor_state_change_by_authority",
677 "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.",
678 &["authority", "state"],
679 registry,
680 ).unwrap(),
681 excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
682 "excluded_proposal_ancestors_count_by_authority",
683 "Total number of excluded ancestors per authority during proposal.",
684 &["authority"],
685 registry,
686 ).unwrap(),
687 included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!(
688 "included_excluded_proposal_ancestors_count_by_authority",
689 "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.",
690 &["authority", "type"],
691 registry,
692 ).unwrap(),
693 missing_blocks_total: register_int_counter_with_registry!(
694 "missing_blocks_total",
695 "Total cumulative number of missing blocks",
696 registry,
697 ).unwrap(),
698 missing_blocks_after_fetch_total: register_int_counter_with_registry!(
699 "missing_blocks_after_fetch_total",
700 "Total number of missing blocks after fetching blocks from peer",
701 registry,
702 ).unwrap(),
703 num_of_bad_nodes: register_int_gauge_with_registry!(
704 "num_of_bad_nodes",
705 "The number of bad nodes in the new leader schedule",
706 registry
707 ).unwrap(),
708 quorum_receive_latency: register_histogram_with_registry!(
709 "quorum_receive_latency",
710 "The time it took to receive a new round quorum of blocks",
711 registry
712 ).unwrap(),
713 block_receive_delay: register_int_counter_vec_with_registry!(
714 "block_receive_delay",
715 "Total delay from the start of the round to receiving the block, in milliseconds per authority",
716 &["authority"],
717 registry,
718 ).unwrap(),
719 reputation_scores: register_int_gauge_vec_with_registry!(
720 "reputation_scores",
721 "Reputation scores for each authority",
722 &["authority"],
723 registry,
724 ).unwrap(),
725 leader_schedule_total_scores: register_int_gauge_vec_with_registry!(
726 "leader_schedule_total_scores",
727 "LeaderScheduleV3 running per-authority score over the scoring window. Each commit contributes voted_for_stake * certified_by_stake (stake^2 units)",
728 &["authority"],
729 registry,
730 ).unwrap(),
731 leader_schedule_normalized_scores: register_gauge_vec_with_registry!(
732 "leader_schedule_normalized_scores",
733 "LeaderScheduleV3 per-authority total score divided by (running sum of per-commit leader stake) * committee total stake. Stake^2 over stake^2; a fraction with no stake dimension",
734 &["authority"],
735 registry,
736 ).unwrap(),
737 leader_schedule_last_num_leaders: register_int_gauge_with_registry!(
738 "leader_schedule_last_num_leaders",
739 "LeaderScheduleV3 number of leader-round blocks in the most recently scored commit",
740 registry,
741 ).unwrap(),
742 leader_schedule_average_num_leaders: register_gauge_with_registry!(
743 "leader_schedule_average_num_leaders",
744 "LeaderScheduleV3 moving average of the number of leaders per commit across the scoring window",
745 registry,
746 ).unwrap(),
747 scope_processing_time: register_histogram_vec_with_registry!(
748 "scope_processing_time",
749 "The processing time of a specific code scope",
750 &["scope"],
751 FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(),
752 registry
753 ).unwrap(),
754 sub_dags_per_commit_count: register_histogram_with_registry!(
755 "sub_dags_per_commit_count",
756 "The number of subdags per commit.",
757 registry,
758 ).unwrap(),
759 block_suspensions: register_int_counter_vec_with_registry!(
760 "block_suspensions",
761 "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted",
762 &["authority"],
763 registry,
764 ).unwrap(),
765 block_unsuspensions: register_int_counter_vec_with_registry!(
766 "block_unsuspensions",
767 "The number of block unsuspensions.",
768 &["authority"],
769 registry,
770 ).unwrap(),
771 suspended_block_time: register_histogram_vec_with_registry!(
772 "suspended_block_time",
773 "The time for which a block remains suspended",
774 &["authority"],
775 registry,
776 ).unwrap(),
777 block_manager_suspended_blocks: register_int_gauge_with_registry!(
778 "block_manager_suspended_blocks",
779 "The number of blocks currently suspended in the block manager",
780 registry,
781 ).unwrap(),
782 block_manager_missing_ancestors: register_int_gauge_with_registry!(
783 "block_manager_missing_ancestors",
784 "The number of missing ancestors tracked in the block manager",
785 registry,
786 ).unwrap(),
787 block_manager_missing_blocks: register_int_gauge_with_registry!(
788 "block_manager_missing_blocks",
789 "The number of blocks missing content tracked in the block manager",
790 registry,
791 ).unwrap(),
792 block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!(
793 "block_manager_missing_blocks_by_authority",
794 "The number of new missing blocks by block authority",
795 &["authority"],
796 registry,
797 ).unwrap(),
798 block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!(
799 "block_manager_missing_ancestors_by_authority",
800 "The number of missing ancestors by ancestor authority across received blocks",
801 &["authority"],
802 registry,
803 ).unwrap(),
804 block_manager_gced_blocks: register_int_counter_vec_with_registry!(
805 "block_manager_gced_blocks",
806 "The number of blocks that garbage collected and did not get accepted, counted by block's source authority",
807 &["authority"],
808 registry,
809 ).unwrap(),
810 block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!(
811 "block_manager_gc_unsuspended_blocks",
812 "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority",
813 &["authority"],
814 registry,
815 ).unwrap(),
816 block_manager_skipped_blocks: register_int_counter_vec_with_registry!(
817 "block_manager_skipped_blocks",
818 "The number of blocks skipped by the block manager due to block round being <= gc_round",
819 &["authority"],
820 registry,
821 ).unwrap(),
822 threshold_clock_round: register_int_gauge_with_registry!(
823 "threshold_clock_round",
824 "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.",
825 registry,
826 ).unwrap(),
827 subscriber_connection_attempts: register_int_counter_vec_with_registry!(
828 "subscriber_connection_attempts",
829 "The number of connection attempts per peer",
830 &["authority", "status"],
831 registry,
832 ).unwrap(),
833 subscribed_to: register_int_gauge_vec_with_registry!(
834 "subscribed_to",
835 "Peers that this authority subscribed to for block streams.",
836 &["authority"],
837 registry,
838 ).unwrap(),
839 subscribed_by: register_int_gauge_vec_with_registry!(
840 "subscribed_by",
841 "Peers subscribing for block streams from this authority.",
842 &["authority"],
843 registry,
844 ).unwrap(),
845 commit_sync_inflight_fetches: register_int_gauge_with_registry!(
846 "commit_sync_inflight_fetches",
847 "The number of inflight fetches in commit syncer",
848 registry,
849 ).unwrap(),
850 commit_sync_pending_fetches: register_int_gauge_with_registry!(
851 "commit_sync_pending_fetches",
852 "The number of pending fetches in commit syncer",
853 registry,
854 ).unwrap(),
855 commit_sync_fetched_commits: register_int_counter_with_registry!(
856 "commit_sync_fetched_commits",
857 "The number of commits fetched via commit syncer",
858 registry,
859 ).unwrap(),
860 commit_sync_fetched_blocks: register_int_counter_with_registry!(
861 "commit_sync_fetched_blocks",
862 "The number of blocks fetched via commit syncer",
863 registry,
864 ).unwrap(),
865 commit_sync_total_fetched_blocks_size: register_int_counter_with_registry!(
866 "commit_sync_total_fetched_blocks_size",
867 "The total size in bytes of blocks fetched via commit syncer",
868 registry,
869 ).unwrap(),
870 commit_sync_quorum_index: register_int_gauge_with_registry!(
871 "commit_sync_quorum_index",
872 "The maximum commit index voted by a quorum of authorities",
873 registry,
874 ).unwrap(),
875 commit_sync_highest_synced_index: register_int_gauge_with_registry!(
876 "commit_sync_fetched_index",
877 "The max commit index among local and fetched commits",
878 registry,
879 ).unwrap(),
880 commit_sync_highest_fetched_index: register_int_gauge_with_registry!(
881 "commit_sync_highest_fetched_index",
882 "The max commit index that has been fetched via network",
883 registry,
884 ).unwrap(),
885 commit_sync_local_index: register_int_gauge_with_registry!(
886 "commit_sync_local_index",
887 "The local commit index",
888 registry,
889 ).unwrap(),
890 commit_sync_gap_on_processing: register_int_counter_with_registry!(
891 "commit_sync_gap_on_processing",
892 "Number of instances where a gap was found in fetched commit processing",
893 registry,
894 ).unwrap(),
895 commit_sync_fetch_loop_latency: register_histogram_with_registry!(
896 "commit_sync_fetch_loop_latency",
897 "The time taken to finish fetching commits and blocks from a given range",
898 LATENCY_SEC_BUCKETS.to_vec(),
899 registry,
900 ).unwrap(),
901 commit_sync_fetch_once_latency: register_histogram_with_registry!(
902 "commit_sync_fetch_once_latency",
903 "The time taken to fetch commits and blocks once",
904 LATENCY_SEC_BUCKETS.to_vec(),
905 registry,
906 ).unwrap(),
907 commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!(
908 "commit_sync_fetch_once_errors",
909 "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.",
910 &["authority", "error"],
911 registry
912 ).unwrap(),
913 commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!(
914 "commit_sync_fetch_missing_blocks",
915 "Number of ancestor blocks that are missing when processing blocks via commit sync.",
916 &["authority"],
917 registry,
918 ).unwrap(),
919 commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!(
920 "commit_sync_fetch_commits_handler_uncertified_skipped",
921 "Number of uncertified commits that got skipped when fetching commits due to lack of votes",
922 registry,
923 ).unwrap(),
924 round_tracker_received_quorum_round_gaps: register_int_gauge_vec_with_registry!(
925 "round_tracker_received_quorum_round_gaps",
926 "Received round gaps among peers for blocks proposed from each authority",
927 &["authority"],
928 registry
929 ).unwrap(),
930 round_tracker_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!(
931 "round_tracker_accepted_quorum_round_gaps",
932 "Accepted round gaps among peers for blocks proposed & accepted from each authority",
933 &["authority"],
934 registry
935 ).unwrap(),
936 round_tracker_low_received_quorum_round: register_int_gauge_vec_with_registry!(
937 "round_tracker_low_received_quorum_round",
938 "Low quorum round among peers for blocks proposed from each authority",
939 &["authority"],
940 registry
941 ).unwrap(),
942 round_tracker_low_accepted_quorum_round: register_int_gauge_vec_with_registry!(
943 "round_tracker_low_accepted_quorum_round",
944 "Low quorum round among peers for blocks proposed & accepted from each authority",
945 &["authority"],
946 registry
947 ).unwrap(),
948 round_tracker_current_received_round_gaps: register_int_gauge_vec_with_registry!(
949 "round_tracker_current_received_round_gaps",
950 "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.",
951 &["authority"],
952 registry
953 ).unwrap(),
954 round_tracker_current_accepted_round_gaps: register_int_gauge_vec_with_registry!(
955 "round_tracker_current_accepted_round_gaps",
956 "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.",
957 &["authority"],
958 registry
959 ).unwrap(),
960 round_tracker_propagation_delays: register_histogram_with_registry!(
961 "round_tracker_propagation_delays",
962 "Round gaps between the last proposed block round and the lower bound of own quorum round",
963 NUM_BUCKETS.to_vec(),
964 registry
965 ).unwrap(),
966 round_tracker_last_propagation_delay: register_int_gauge_with_registry!(
967 "round_tracker_last_propagation_delay",
968 "Most recent propagation delay observed by RoundTracker",
969 registry
970 ).unwrap(),
971 round_prober_request_errors: register_int_counter_vec_with_registry!(
972 "round_prober_request_errors",
973 "Number of errors when probing against peers per error type",
974 &["error_type"],
975 registry
976 ).unwrap(),
977 certifier_gc_round: register_int_gauge_with_registry!(
978 "certifier_gc_round",
979 "The current GC round of the certifier",
980 registry
981 ).unwrap(),
982 certifier_own_reject_votes: register_int_counter_vec_with_registry!(
983 "certifier_own_reject_votes",
984 "Number of own reject votes against each peer authority",
985 &["authority"],
986 registry
987 ).unwrap(),
988 finalizer_buffered_commits: register_int_gauge_with_registry!(
989 "finalizer_buffered_commits",
990 "The number of commits buffered in the finalizer",
991 registry,
992 ).unwrap(),
993 finalizer_round_delay: register_histogram_with_registry!(
994 "finalizer_round_delay",
995 "The delay between the round of the last committed block and the round of the finalized commit.",
996 ROUND_DELAY_BUCKETS.to_vec(),
997 registry,
998 ).unwrap(),
999 finalizer_transaction_status: register_int_counter_vec_with_registry!(
1000 "finalizer_transaction_status",
1001 "Number of transactions finalized by the finalizer, grouped by status.",
1002 &["status"],
1003 registry
1004 ).unwrap(),
1005 finalizer_reject_votes: register_int_counter_vec_with_registry!(
1006 "finalizer_reject_votes",
1007 "Number of reject votes casted by each authority observed by the finalizer.",
1008 &["authority"],
1009 registry
1010 ).unwrap(),
1011 finalizer_output_commits: register_int_counter_vec_with_registry!(
1012 "finalizer_output_commits",
1013 "Number of output commits finalized by the finalizer, grouped by type.",
1014 &["type"],
1015 registry
1016 ).unwrap(),
1017 finalizer_skipped_voting_blocks: register_int_counter_vec_with_registry!(
1018 "finalizer_skipped_voting_blocks",
1019 "Number of times where another block skipped voting due to potentially out of GC bound. Authority is from the voted block.",
1020 &["authority", "type"],
1021 registry
1022 ).unwrap(),
1023 uptime: register_histogram_with_registry!(
1024 "uptime",
1025 "Total node uptime",
1026 LATENCY_SEC_BUCKETS.to_vec(),
1027 registry,
1028 ).unwrap(),
1029 }
1030 }
1031}