1use axum::{Router, extract::Extension, http::StatusCode, routing::get};
5use mysten_metrics::RegistryService;
6use prometheus::{
7 Histogram, IntCounter, IntGauge, register_histogram_with_registry,
8 register_int_counter_with_registry, register_int_gauge_with_registry,
9};
10use prometheus::{Registry, TextEncoder};
11use std::net::SocketAddr;
12use tracing::info;
13
14const METRICS_ROUTE: &str = "/metrics";
15
16pub fn start_prometheus_server(
17 addr: SocketAddr,
18) -> Result<(RegistryService, Registry), anyhow::Error> {
19 info!(address =% addr, "Starting prometheus server");
20 let registry = Registry::new_custom(Some("indexer".to_string()), None)?;
21 let registry_service = RegistryService::new(registry.clone());
22
23 let app = Router::new()
24 .route(METRICS_ROUTE, get(metrics))
25 .layer(Extension(registry_service.clone()));
26
27 tokio::spawn(async move {
28 let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
29 axum::serve(listener, app).await.unwrap();
30 });
31 Ok((registry_service, registry))
32}
33
34async fn metrics(Extension(registry_service): Extension<RegistryService>) -> (StatusCode, String) {
35 let metrics_families = registry_service.gather_all();
36 match TextEncoder.encode_to_string(&metrics_families) {
37 Ok(metrics) => (StatusCode::OK, metrics),
38 Err(error) => (
39 StatusCode::INTERNAL_SERVER_ERROR,
40 format!("unable to encode metrics: {error}"),
41 ),
42 }
43}
44
45const DATA_INGESTION_LATENCY_SEC_BUCKETS: &[f64] = &[
48 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0,
49];
50const DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS: &[f64] = &[
53 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0,
54 10000.0,
55];
56const JSON_RPC_LATENCY_SEC_BUCKETS: &[f64] = &[
59 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0,
60];
61
62#[derive(Clone)]
63pub struct IndexerMetrics {
64 pub total_checkpoint_received: IntCounter,
65 pub total_tx_checkpoint_committed: IntCounter,
66 pub total_object_checkpoint_committed: IntCounter,
67 pub total_transaction_committed: IntCounter,
68 pub total_object_change_committed: IntCounter,
69 pub total_transaction_chunk_committed: IntCounter,
70 pub total_object_change_chunk_committed: IntCounter,
71 pub total_epoch_committed: IntCounter,
72 pub latest_fullnode_checkpoint_sequence_number: IntGauge,
73 pub latest_tx_checkpoint_sequence_number: IntGauge,
74 pub latest_indexer_object_checkpoint_sequence_number: IntGauge,
75 pub latest_object_snapshot_sequence_number: IntGauge,
76 pub max_downloaded_checkpoint_sequence_number: IntGauge,
78 pub max_indexed_checkpoint_sequence_number: IntGauge,
79 pub max_committed_checkpoint_sequence_number: IntGauge,
80 pub downloaded_checkpoint_timestamp_ms: IntGauge,
82 pub indexed_checkpoint_timestamp_ms: IntGauge,
83 pub committed_checkpoint_timestamp_ms: IntGauge,
84 pub download_lag_ms: IntGauge,
86 pub index_lag_ms: IntGauge,
87 pub db_commit_lag_ms: IntGauge,
88 pub checkpoint_download_bytes_size: IntGauge,
91 pub tokio_blocking_task_wait_latency: Histogram,
92 pub fullnode_checkpoint_data_download_latency: Histogram,
93 pub fullnode_checkpoint_wait_and_download_latency: Histogram,
94 pub fullnode_transaction_download_latency: Histogram,
95 pub fullnode_object_download_latency: Histogram,
96 pub checkpoint_index_latency: Histogram,
97 pub indexing_batch_size: IntGauge,
98 pub indexing_tx_object_changes_latency: Histogram,
99 pub indexing_objects_latency: Histogram,
100 pub indexing_get_object_in_mem_hit: IntCounter,
101 pub indexing_get_object_db_hit: IntCounter,
102 pub indexing_module_resolver_in_mem_hit: IntCounter,
103 pub indexing_package_resolver_in_mem_hit: IntCounter,
104 pub indexing_packages_latency: Histogram,
105 pub checkpoint_objects_index_latency: Histogram,
106 pub checkpoint_db_commit_latency: Histogram,
107 pub checkpoint_db_commit_latency_step_1: Histogram,
108 pub checkpoint_db_commit_latency_transactions: Histogram,
109 pub checkpoint_db_commit_latency_transactions_chunks: Histogram,
110 pub checkpoint_db_commit_latency_transactions_chunks_transformation: Histogram,
111 pub checkpoint_db_commit_latency_objects: Histogram,
112 pub checkpoint_db_commit_latency_objects_snapshot: Histogram,
113 pub checkpoint_db_commit_latency_objects_version: Histogram,
114 pub checkpoint_db_commit_latency_objects_history: Histogram,
115 pub checkpoint_db_commit_latency_full_objects_history: Histogram,
116 pub checkpoint_db_commit_latency_objects_chunks: Histogram,
117 pub checkpoint_db_commit_latency_objects_snapshot_chunks: Histogram,
118 pub checkpoint_db_commit_latency_objects_version_chunks: Histogram,
119 pub checkpoint_db_commit_latency_objects_history_chunks: Histogram,
120 pub checkpoint_db_commit_latency_full_objects_history_chunks: Histogram,
121 pub checkpoint_db_commit_latency_events: Histogram,
122 pub checkpoint_db_commit_latency_events_chunks: Histogram,
123 pub checkpoint_db_commit_latency_event_indices: Histogram,
124 pub checkpoint_db_commit_latency_event_indices_chunks: Histogram,
125 pub checkpoint_db_commit_latency_packages: Histogram,
126 pub checkpoint_db_commit_latency_tx_indices: Histogram,
127 pub checkpoint_db_commit_latency_tx_indices_chunks: Histogram,
128 pub checkpoint_db_commit_latency_checkpoints: Histogram,
129 pub checkpoint_db_commit_latency_epoch: Histogram,
130 pub checkpoint_db_commit_latency_watermarks: Histogram,
131 pub thousand_transaction_avg_db_commit_latency: Histogram,
132 pub object_db_commit_latency: Histogram,
133 pub object_mutation_db_commit_latency: Histogram,
134 pub object_deletion_db_commit_latency: Histogram,
135 pub epoch_db_commit_latency: Histogram,
136 pub advance_epoch_latency: Histogram,
138 pub get_transaction_block_latency: Histogram,
140 pub multi_get_transaction_blocks_latency: Histogram,
141 pub get_object_latency: Histogram,
142 pub multi_get_objects_latency: Histogram,
143 pub try_get_past_object_latency: Histogram,
144 pub try_multi_get_past_objects_latency: Histogram,
145 pub get_checkpoint_latency: Histogram,
146 pub get_checkpoints_latency: Histogram,
147 pub get_events_latency: Histogram,
148 pub get_loaded_child_objects_latency: Histogram,
149 pub get_total_transaction_blocks_latency: Histogram,
150 pub get_latest_checkpoint_sequence_number_latency: Histogram,
151 pub get_owned_objects_latency: Histogram,
153 pub query_transaction_blocks_latency: Histogram,
154 pub query_events_latency: Histogram,
155 pub get_dynamic_fields_latency: Histogram,
156 pub get_dynamic_field_object_latency: Histogram,
157 pub get_protocol_config_latency: Histogram,
158 pub subscription_process_latency: Histogram,
160 pub transaction_per_checkpoint: Histogram,
161 pub db_conn_pool_size: IntGauge,
163 pub idle_db_conn: IntGauge,
164 pub address_processor_failure: IntCounter,
165 pub checkpoint_metrics_processor_failure: IntCounter,
166 pub last_pruned_epoch: IntGauge,
168 pub last_pruned_checkpoint: IntGauge,
169 pub last_pruned_transaction: IntGauge,
170 pub epoch_pruning_latency: Histogram,
171}
172
173impl IndexerMetrics {
174 pub fn new(registry: &Registry) -> Self {
175 Self {
176 total_checkpoint_received: register_int_counter_with_registry!(
177 "total_checkpoint_received",
178 "Total number of checkpoint received",
179 registry,
180 )
181 .unwrap(),
182 total_tx_checkpoint_committed: register_int_counter_with_registry!(
183 "total_checkpoint_committed",
184 "Total number of checkpoint committed",
185 registry,
186 )
187 .unwrap(),
188 total_object_checkpoint_committed: register_int_counter_with_registry!(
189 "total_object_checkpoint_committed",
190 "Total number of object checkpoint committed",
191 registry,
192 )
193 .unwrap(),
194 total_transaction_committed: register_int_counter_with_registry!(
195 "total_transaction_committed",
196 "Total number of transaction committed",
197 registry,
198 )
199 .unwrap(),
200 total_object_change_committed: register_int_counter_with_registry!(
201 "total_object_change_committed",
202 "Total number of object change committed",
203 registry,
204 )
205 .unwrap(),
206 total_transaction_chunk_committed: register_int_counter_with_registry!(
207 "total_transaction_chunk_committed",
208 "Total number of transaction chunk committed",
209 registry,
210 )
211 .unwrap(),
212 total_object_change_chunk_committed: register_int_counter_with_registry!(
213 "total_object_change_chunk_committed",
214 "Total number of object change chunk committed",
215 registry,
216 )
217 .unwrap(),
218 total_epoch_committed: register_int_counter_with_registry!(
219 "total_epoch_committed",
220 "Total number of epoch committed",
221 registry,
222 )
223 .unwrap(),
224 latest_fullnode_checkpoint_sequence_number: register_int_gauge_with_registry!(
225 "latest_fullnode_checkpoint_sequence_number",
226 "Latest checkpoint sequence number from the Full Node",
227 registry,
228 )
229 .unwrap(),
230 latest_tx_checkpoint_sequence_number: register_int_gauge_with_registry!(
231 "latest_indexer_checkpoint_sequence_number",
232 "Latest checkpoint sequence number from the Indexer",
233 registry,
234 )
235 .unwrap(),
236 latest_indexer_object_checkpoint_sequence_number: register_int_gauge_with_registry!(
237 "latest_indexer_object_checkpoint_sequence_number",
238 "Latest object checkpoint sequence number from the Indexer",
239 registry,
240 )
241 .unwrap(),
242 latest_object_snapshot_sequence_number: register_int_gauge_with_registry!(
243 "latest_object_snapshot_sequence_number",
244 "Latest object snapshot sequence number from the Indexer",
245 registry,
246 ).unwrap(),
247 max_downloaded_checkpoint_sequence_number: register_int_gauge_with_registry!(
248 "max_downloaded_checkpoint_sequence_number",
249 "Max downloaded checkpoint sequence number",
250 registry,
251 ).unwrap(),
252 max_indexed_checkpoint_sequence_number: register_int_gauge_with_registry!(
253 "max_indexed_checkpoint_sequence_number",
254 "Max indexed checkpoint sequence number",
255 registry,
256 ).unwrap(),
257 max_committed_checkpoint_sequence_number: register_int_gauge_with_registry!(
258 "max_committed_checkpoint_sequence_number",
259 "Max committed checkpoint sequence number",
260 registry,
261 ).unwrap(),
262 downloaded_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
263 "downloaded_checkpoint_timestamp_ms",
264 "Timestamp of the downloaded checkpoint",
265 registry,
266 ).unwrap(),
267 indexed_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
268 "indexed_checkpoint_timestamp_ms",
269 "Timestamp of the indexed checkpoint",
270 registry,
271 ).unwrap(),
272 committed_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
273 "committed_checkpoint_timestamp_ms",
274 "Timestamp of the committed checkpoint",
275 registry,
276 ).unwrap(),
277 download_lag_ms: register_int_gauge_with_registry!(
278 "download_lag_ms",
279 "Lag of the latest checkpoint in milliseconds",
280 registry,
281 ).unwrap(),
282 index_lag_ms: register_int_gauge_with_registry!(
283 "index_lag_ms",
284 "Lag of the latest checkpoint in milliseconds",
285 registry,
286 ).unwrap(),
287 db_commit_lag_ms: register_int_gauge_with_registry!(
288 "db_commit_lag_ms",
289 "Lag of the latest checkpoint in milliseconds",
290 registry,
291 ).unwrap(),
292 checkpoint_download_bytes_size: register_int_gauge_with_registry!(
293 "checkpoint_download_bytes_size",
294 "Size of the downloaded checkpoint in bytes",
295 registry,
296 ).unwrap(),
297 fullnode_checkpoint_data_download_latency: register_histogram_with_registry!(
298 "fullnode_checkpoint_data_download_latency",
299 "Time spent in downloading checkpoint and transaction for a new checkpoint from the Full Node",
300 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
301 registry,
302 )
303 .unwrap(),
304 fullnode_checkpoint_wait_and_download_latency: register_histogram_with_registry!(
305 "fullnode_checkpoint_wait_and_download_latency",
306 "Time spent in waiting for a new checkpoint from the Full Node",
307 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
308 registry,
309 )
310 .unwrap(),
311
312 fullnode_transaction_download_latency: register_histogram_with_registry!(
313 "fullnode_transaction_download_latency",
314 "Time spent in waiting for a new transaction from the Full Node",
315 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
316 registry,
317 )
318 .unwrap(),
319 fullnode_object_download_latency: register_histogram_with_registry!(
320 "fullnode_object_download_latency",
321 "Time spent in waiting for a new epoch from the Full Node",
322 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
323 registry,
324 )
325 .unwrap(),
326 checkpoint_index_latency: register_histogram_with_registry!(
327 "checkpoint_index_latency",
328 "Time spent in indexing a checkpoint",
329 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
330 registry,
331 )
332 .unwrap(),
333 indexing_batch_size: register_int_gauge_with_registry!(
334 "indexing_batch_size",
335 "Size of the indexing batch",
336 registry,
337 ).unwrap(),
338 indexing_tx_object_changes_latency: register_histogram_with_registry!(
339 "indexing_tx_object_changes_latency",
340 "Time spent in indexing object changes for a transaction",
341 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
342 registry,
343 )
344 .unwrap(),
345 indexing_objects_latency: register_histogram_with_registry!(
346 "indexing_objects_latency",
347 "Time spent in indexing objects",
348 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
349 registry,
350 )
351 .unwrap(),
352 indexing_packages_latency: register_histogram_with_registry!(
353 "indexing_packages_latency",
354 "Time spent in indexing packages",
355 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
356 registry,
357 )
358 .unwrap(),
359 indexing_get_object_in_mem_hit: register_int_counter_with_registry!(
360 "indexing_get_object_in_mem_hit",
361 "Total number get object hit in mem",
362 registry,
363 )
364 .unwrap(),
365 indexing_get_object_db_hit: register_int_counter_with_registry!(
366 "indexing_get_object_db_hit",
367 "Total number get object hit in db",
368 registry,
369 )
370 .unwrap(),
371 indexing_module_resolver_in_mem_hit: register_int_counter_with_registry!(
372 "indexing_module_resolver_in_mem_hit",
373 "Total number module resolver hit in mem",
374 registry,
375 )
376 .unwrap(),
377 indexing_package_resolver_in_mem_hit: register_int_counter_with_registry!(
378 "indexing_package_resolver_in_mem_hit",
379 "Total number package resolver hit in mem",
380 registry,
381 )
382 .unwrap(),
383 checkpoint_objects_index_latency: register_histogram_with_registry!(
384 "checkpoint_object_index_latency",
385 "Time spent in indexing a checkpoint objects",
386 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
387 registry,
388 )
389 .unwrap(),
390 checkpoint_db_commit_latency: register_histogram_with_registry!(
391 "checkpoint_db_commit_latency",
392 "Time spent committing a checkpoint to the db",
393 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
394 registry,
395 )
396 .unwrap(),
397
398 checkpoint_db_commit_latency_step_1: register_histogram_with_registry!(
399 "checkpoint_db_commit_latency_step_1",
400 "Time spent committing a checkpoint to the db, step 1",
401 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
402 registry,
403 )
404 .unwrap(),
405 checkpoint_db_commit_latency_transactions: register_histogram_with_registry!(
406 "checkpoint_db_commit_latency_transactions",
407 "Time spent committing transactions",
408 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
409 registry,
410 )
411 .unwrap(),
412 checkpoint_db_commit_latency_transactions_chunks: register_histogram_with_registry!(
413 "checkpoint_db_commit_latency_transactions_chunks",
414 "Time spent committing transactions chunks",
415 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
416 registry,
417 )
418 .unwrap(),
419 checkpoint_db_commit_latency_transactions_chunks_transformation: register_histogram_with_registry!(
420 "checkpoint_db_commit_latency_transactions_transaformation",
421 "Time spent in transactions chunks transformation prior to commit",
422 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
423 registry,
424 )
425 .unwrap(),
426 checkpoint_db_commit_latency_objects: register_histogram_with_registry!(
427 "checkpoint_db_commit_latency_objects",
428 "Time spent committing objects",
429 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
430 registry,
431 )
432 .unwrap(),
433 checkpoint_db_commit_latency_objects_snapshot: register_histogram_with_registry!(
434 "checkpoint_db_commit_latency_objects_snapshot",
435 "Time spent committing objects snapshots",
436 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
437 registry,
438 )
439 .unwrap(),
440 checkpoint_db_commit_latency_objects_version: register_histogram_with_registry!(
441 "checkpoint_db_commit_latency_objects_version",
442 "Time spent committing objects version",
443 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
444 registry,
445 ).unwrap(),
446 checkpoint_db_commit_latency_objects_history: register_histogram_with_registry!(
447 "checkpoint_db_commit_latency_objects_history",
448 "Time spent committing objects history",
449 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
450 registry,
451 ).unwrap(),
452 checkpoint_db_commit_latency_full_objects_history: register_histogram_with_registry!(
453 "checkpoint_db_commit_latency_full_objects_history",
454 "Time spent committing full objects history",
455 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
456 registry,
457 ).unwrap(),
458 checkpoint_db_commit_latency_objects_chunks: register_histogram_with_registry!(
459 "checkpoint_db_commit_latency_objects_chunks",
460 "Time spent committing objects chunks",
461 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
462 registry,
463 )
464 .unwrap(),
465 checkpoint_db_commit_latency_objects_snapshot_chunks: register_histogram_with_registry!(
466 "checkpoint_db_commit_latency_objects_snapshot_chunks",
467 "Time spent committing objects snapshot chunks",
468 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
469 registry,
470 )
471 .unwrap(),
472 checkpoint_db_commit_latency_objects_version_chunks: register_histogram_with_registry!(
473 "checkpoint_db_commit_latency_objects_version_chunks",
474 "Time spent committing objects version chunks",
475 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
476 registry,
477 ).unwrap(),
478 checkpoint_db_commit_latency_objects_history_chunks: register_histogram_with_registry!(
479 "checkpoint_db_commit_latency_objects_history_chunks",
480 "Time spent committing objects history chunks",
481 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
482 registry,
483 ).unwrap(),
484 checkpoint_db_commit_latency_full_objects_history_chunks: register_histogram_with_registry!(
485 "checkpoint_db_commit_latency_full_objects_history_chunks",
486 "Time spent committing full objects history chunks",
487 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
488 registry,
489 )
490 .unwrap(),
491 checkpoint_db_commit_latency_events: register_histogram_with_registry!(
492 "checkpoint_db_commit_latency_events",
493 "Time spent committing events",
494 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
495 registry,
496 )
497 .unwrap(),
498 checkpoint_db_commit_latency_events_chunks: register_histogram_with_registry!(
499 "checkpoint_db_commit_latency_events_chunks",
500 "Time spent committing events chunks",
501 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
502 registry,
503 )
504 .unwrap(),
505 checkpoint_db_commit_latency_event_indices: register_histogram_with_registry!(
506 "checkpoint_db_commit_latency_event_indices",
507 "Time spent committing event indices",
508 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
509 registry,
510 )
511 .unwrap(),
512 checkpoint_db_commit_latency_event_indices_chunks: register_histogram_with_registry!(
513 "checkpoint_db_commit_latency_event_indices_chunks",
514 "Time spent committing event indices chunks",
515 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
516 registry,
517 )
518 .unwrap(),
519 checkpoint_db_commit_latency_packages: register_histogram_with_registry!(
520 "checkpoint_db_commit_latency_packages",
521 "Time spent committing packages",
522 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
523 registry,
524 )
525 .unwrap(),
526 checkpoint_db_commit_latency_tx_indices: register_histogram_with_registry!(
527 "checkpoint_db_commit_latency_tx_indices",
528 "Time spent committing tx indices",
529 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
530 registry,
531 )
532 .unwrap(),
533 checkpoint_db_commit_latency_tx_indices_chunks: register_histogram_with_registry!(
534 "checkpoint_db_commit_latency_tx_indices_chunks",
535 "Time spent committing tx_indices chunks",
536 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
537 registry,
538 )
539 .unwrap(),
540 checkpoint_db_commit_latency_checkpoints: register_histogram_with_registry!(
541 "checkpoint_db_commit_latency_checkpoints",
542 "Time spent committing checkpoints",
543 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
544 registry,
545 )
546 .unwrap(),
547 checkpoint_db_commit_latency_epoch: register_histogram_with_registry!(
548 "checkpoint_db_commit_latency_epochs",
549 "Time spent committing epochs",
550 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
551 registry,
552 )
553 .unwrap(),
554 checkpoint_db_commit_latency_watermarks: register_histogram_with_registry!(
555 "checkpoint_db_commit_latency_watermarks",
556 "Time spent committing watermarks",
557 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
558 registry,
559 )
560 .unwrap(),
561 tokio_blocking_task_wait_latency: register_histogram_with_registry!(
562 "tokio_blocking_task_wait_latency",
563 "Time spent to wait for tokio blocking task pool",
564 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
565 registry,
566 ).unwrap(),
567 thousand_transaction_avg_db_commit_latency: register_histogram_with_registry!(
568 "transaction_db_commit_latency",
569 "Average time spent committing 1000 transactions to the db",
570 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
571 registry,
572 )
573 .unwrap(),
574 object_db_commit_latency: register_histogram_with_registry!(
575 "object_db_commit_latency",
576 "Time spent committing a object to the db",
577 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
578 registry,
579 )
580 .unwrap(),
581 object_mutation_db_commit_latency: register_histogram_with_registry!(
582 "object_mutation_db_commit_latency",
583 "Time spent committing a object mutation to the db",
584 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
585 registry,
586 )
587 .unwrap(),
588 object_deletion_db_commit_latency: register_histogram_with_registry!(
589 "object_deletion_db_commit_latency",
590 "Time spent committing a object deletion to the db",
591 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
592 registry,
593 )
594 .unwrap(),
595 epoch_db_commit_latency: register_histogram_with_registry!(
596 "epoch_db_commit_latency",
597 "Time spent committing a epoch to the db",
598 DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
599 registry,
600 )
601 .unwrap(),
602 advance_epoch_latency: register_histogram_with_registry!(
603 "advance_epoch_latency",
604 "Time spent in advancing epoch",
605 DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(),
606 registry,
607 ).unwrap(),
608 subscription_process_latency: register_histogram_with_registry!(
609 "subscription_process_latency",
610 "Time spent in process Websocket subscription",
611 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
612 registry,
613 )
614 .unwrap(),
615 transaction_per_checkpoint: register_histogram_with_registry!(
616 "transaction_per_checkpoint",
617 "Number of transactions per checkpoint",
618 vec![1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0],
619 registry,
620 )
621 .unwrap(),
622 get_transaction_block_latency: register_histogram_with_registry!(
623 "get_transaction_block_latency",
624 "Time spent in get_transaction_block on the fullnode behind.",
625 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
626 registry
627 )
628 .unwrap(),
629 multi_get_transaction_blocks_latency: register_histogram_with_registry!(
630 "multi_get_transaction_blocks_latency",
631 "Time spent in multi_get_transaction_blocks on the fullnode behind.",
632 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
633 registry
634 )
635 .unwrap(),
636 get_object_latency: register_histogram_with_registry!(
637 "get_object_latency",
638 "Time spent in get_object on the fullnode behind.",
639 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
640 registry
641 )
642 .unwrap(),
643 multi_get_objects_latency: register_histogram_with_registry!(
644 "multi_get_objects_latency",
645 "Time spent in multi_get_objects on the fullnode behind.",
646 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
647 registry
648 )
649 .unwrap(),
650 try_get_past_object_latency: register_histogram_with_registry!(
651 "try_get_past_object_latency",
652 "Time spent in try_get_past_object on the fullnode behind.",
653 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
654 registry
655 )
656 .unwrap(),
657 try_multi_get_past_objects_latency: register_histogram_with_registry!(
658 "try_multi_get_past_objects_latency",
659 "Time spent in try_multi_get_past_objects on the fullnode behind.",
660 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
661 registry
662 )
663 .unwrap(),
664 get_checkpoint_latency: register_histogram_with_registry!(
665 "get_checkpoint_latency",
666 "Time spent in get_checkpoint on the fullnode behind.",
667 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
668 registry
669 )
670 .unwrap(),
671 get_checkpoints_latency: register_histogram_with_registry!(
672 "get_checkpoints_latency",
673 "Time spent in get_checkpoints on the fullnode behind.",
674 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
675 registry
676 )
677 .unwrap(),
678 get_events_latency: register_histogram_with_registry!(
679 "get_events_latency",
680 "Time spent in get_events on the fullnode behind.",
681 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
682 registry
683 )
684 .unwrap(),
685 get_total_transaction_blocks_latency: register_histogram_with_registry!(
686 "get_total_transaction_blocks_latency",
687 "Time spent in get_total_transaction_blocks on the fullnode behind.",
688 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
689 registry
690 )
691 .unwrap(),
692 get_latest_checkpoint_sequence_number_latency: register_histogram_with_registry!(
693 "get_latest_checkpoint_sequence_number_latency",
694 "Time spent in get_latest_checkpoint_sequence_number on the fullnode behind.",
695 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
696 registry
697 )
698 .unwrap(),
699 get_owned_objects_latency: register_histogram_with_registry!(
700 "get_owned_objects_latency",
701 "Time spent in get_owned_objects on the fullnode behind.",
702 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
703 registry
704 )
705 .unwrap(),
706 query_transaction_blocks_latency: register_histogram_with_registry!(
707 "query_transaction_blocks_latency",
708 "Time spent in query_transaction_blocks on the fullnode behind.",
709 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
710 registry
711 )
712 .unwrap(),
713 query_events_latency: register_histogram_with_registry!(
714 "query_events_latency",
715 "Time spent in query_events on the fullnode behind.",
716 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
717 registry
718 )
719 .unwrap(),
720 get_dynamic_fields_latency: register_histogram_with_registry!(
721 "get_dynamic_fields_latency",
722 "Time spent in get_dynamic_fields on the fullnode behind.",
723 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
724 registry
725 )
726 .unwrap(),
727 get_dynamic_field_object_latency: register_histogram_with_registry!(
728 "get_dynamic_field_object_latency",
729 "Time spent in get_dynamic_field_object on the fullnode behind.",
730 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
731 registry
732 )
733 .unwrap(),
734 get_loaded_child_objects_latency: register_histogram_with_registry!(
735 "get_loaded_child_objects_latency",
736 "Time spent in get_loaded_child_objects_latency on the fullnode behind.",
737 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
738 registry
739 )
740 .unwrap(),
741 get_protocol_config_latency: register_histogram_with_registry!(
742 "get_protocol_config_latency",
743 "Time spent in get_protocol_config_latency on the fullnode behind.",
744 JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
745 registry
746 )
747 .unwrap(),
748 db_conn_pool_size: register_int_gauge_with_registry!(
749 "db_conn_pool_size",
750 "Size of the database connection pool",
751 registry
752 ).unwrap(),
753 idle_db_conn: register_int_gauge_with_registry!(
754 "idle_db_conn",
755 "Number of idle database connections",
756 registry
757 ).unwrap(),
758 address_processor_failure: register_int_counter_with_registry!(
759 "address_processor_failure",
760 "Total number of address processor failure",
761 registry,
762 )
763 .unwrap(),
764 checkpoint_metrics_processor_failure: register_int_counter_with_registry!(
765 "checkpoint_metrics_processor_failure",
766 "Total number of checkpoint metrics processor failure",
767 registry,
768 )
769 .unwrap(),
770 last_pruned_epoch: register_int_gauge_with_registry!(
771 "last_pruned_epoch",
772 "Last pruned epoch number",
773 registry,
774 )
775 .unwrap(),
776 last_pruned_checkpoint: register_int_gauge_with_registry!(
777 "last_pruned_checkpoint",
778 "Last pruned checkpoint sequence number",
779 registry,
780 )
781 .unwrap(),
782 last_pruned_transaction: register_int_gauge_with_registry!(
783 "last_pruned_transaction",
784 "Last pruned transaction sequence number",
785 registry,
786 ).unwrap(),
787 epoch_pruning_latency: register_histogram_with_registry!(
788 "epoch_pruning_latency",
789 "Time spent in pruning one epoch",
790 DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(),
791 registry
792 ).unwrap(),
793 }
794 }
795}
796
797pub fn spawn_connection_pool_metric_collector(
798 metrics: IndexerMetrics,
799 connection_pool: crate::database::ConnectionPool,
800) {
801 tokio::spawn(async move {
802 loop {
803 let cp_state = connection_pool.state();
804 tracing::debug!(
805 connection_pool_size =% cp_state.connections,
806 idle_connections =% cp_state.idle_connections,
807 );
808 metrics.db_conn_pool_size.set(cp_state.connections as i64);
809 metrics.idle_db_conn.set(cp_state.idle_connections as i64);
810 tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
811 }
812 });
813}