sui_indexer/
metrics.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
// Copyright (c) Mysten Labs, Inc.
// SPDX-License-Identifier: Apache-2.0

use axum::{extract::Extension, http::StatusCode, routing::get, Router};
use mysten_metrics::RegistryService;
use prometheus::{
    register_histogram_with_registry, register_int_counter_with_registry,
    register_int_gauge_with_registry, Histogram, IntCounter, IntGauge,
};
use prometheus::{Registry, TextEncoder};
use std::net::SocketAddr;
use tracing::info;

const METRICS_ROUTE: &str = "/metrics";

pub fn start_prometheus_server(
    addr: SocketAddr,
) -> Result<(RegistryService, Registry), anyhow::Error> {
    info!(address =% addr, "Starting prometheus server");
    let registry = Registry::new_custom(Some("indexer".to_string()), None)?;
    let registry_service = RegistryService::new(registry.clone());

    let app = Router::new()
        .route(METRICS_ROUTE, get(metrics))
        .layer(Extension(registry_service.clone()));

    tokio::spawn(async move {
        let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
        axum::serve(listener, app).await.unwrap();
    });
    Ok((registry_service, registry))
}

async fn metrics(Extension(registry_service): Extension<RegistryService>) -> (StatusCode, String) {
    let metrics_families = registry_service.gather_all();
    match TextEncoder.encode_to_string(&metrics_families) {
        Ok(metrics) => (StatusCode::OK, metrics),
        Err(error) => (
            StatusCode::INTERNAL_SERVER_ERROR,
            format!("unable to encode metrics: {error}"),
        ),
    }
}

/// NOTE: for various data ingestion steps, which are expected to be within [0.001, 100] seconds,
/// and high double digits usually means something is broken.
const DATA_INGESTION_LATENCY_SEC_BUCKETS: &[f64] = &[
    0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0,
];
/// NOTE: for objects_snapshot update and advance_epoch, which are expected to be within [0.1, 100] seconds,
/// and can go up to high hundreds of seconds when things go wrong.
const DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS: &[f64] = &[
    0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0,
    10000.0,
];
/// NOTE: for json_rpc calls, which are expected to be within [0.01, 100] seconds,
/// high hundreds of seconds usually means something is broken.
const JSON_RPC_LATENCY_SEC_BUCKETS: &[f64] = &[
    0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0,
];

#[derive(Clone)]
pub struct IndexerMetrics {
    pub total_checkpoint_received: IntCounter,
    pub total_tx_checkpoint_committed: IntCounter,
    pub total_object_checkpoint_committed: IntCounter,
    pub total_transaction_committed: IntCounter,
    pub total_object_change_committed: IntCounter,
    pub total_transaction_chunk_committed: IntCounter,
    pub total_object_change_chunk_committed: IntCounter,
    pub total_epoch_committed: IntCounter,
    pub latest_fullnode_checkpoint_sequence_number: IntGauge,
    pub latest_tx_checkpoint_sequence_number: IntGauge,
    pub latest_indexer_object_checkpoint_sequence_number: IntGauge,
    pub latest_object_snapshot_sequence_number: IntGauge,
    // max checkpoint sequence numbers on various stages of indexer data ingestion
    pub max_downloaded_checkpoint_sequence_number: IntGauge,
    pub max_indexed_checkpoint_sequence_number: IntGauge,
    pub max_committed_checkpoint_sequence_number: IntGauge,
    // the related timestamps of max checkpoint ^ on various stages
    pub downloaded_checkpoint_timestamp_ms: IntGauge,
    pub indexed_checkpoint_timestamp_ms: IntGauge,
    pub committed_checkpoint_timestamp_ms: IntGauge,
    // lag starting from the timestamp of the latest checkpoint to the current time
    pub download_lag_ms: IntGauge,
    pub index_lag_ms: IntGauge,
    pub db_commit_lag_ms: IntGauge,
    // latencies of various steps of data ingestion.
    // checkpoint E2E latency is: fullnode_download_latency + checkpoint_index_latency + db_commit_latency
    pub checkpoint_download_bytes_size: IntGauge,
    pub tokio_blocking_task_wait_latency: Histogram,
    pub fullnode_checkpoint_data_download_latency: Histogram,
    pub fullnode_checkpoint_wait_and_download_latency: Histogram,
    pub fullnode_transaction_download_latency: Histogram,
    pub fullnode_object_download_latency: Histogram,
    pub checkpoint_index_latency: Histogram,
    pub indexing_batch_size: IntGauge,
    pub indexing_tx_object_changes_latency: Histogram,
    pub indexing_objects_latency: Histogram,
    pub indexing_get_object_in_mem_hit: IntCounter,
    pub indexing_get_object_db_hit: IntCounter,
    pub indexing_module_resolver_in_mem_hit: IntCounter,
    pub indexing_package_resolver_in_mem_hit: IntCounter,
    pub indexing_packages_latency: Histogram,
    pub checkpoint_objects_index_latency: Histogram,
    pub checkpoint_db_commit_latency: Histogram,
    pub checkpoint_db_commit_latency_step_1: Histogram,
    pub checkpoint_db_commit_latency_transactions: Histogram,
    pub checkpoint_db_commit_latency_transactions_chunks: Histogram,
    pub checkpoint_db_commit_latency_transactions_chunks_transformation: Histogram,
    pub checkpoint_db_commit_latency_objects: Histogram,
    pub checkpoint_db_commit_latency_objects_snapshot: Histogram,
    pub checkpoint_db_commit_latency_objects_version: Histogram,
    pub checkpoint_db_commit_latency_objects_history: Histogram,
    pub checkpoint_db_commit_latency_full_objects_history: Histogram,
    pub checkpoint_db_commit_latency_objects_chunks: Histogram,
    pub checkpoint_db_commit_latency_objects_snapshot_chunks: Histogram,
    pub checkpoint_db_commit_latency_objects_version_chunks: Histogram,
    pub checkpoint_db_commit_latency_objects_history_chunks: Histogram,
    pub checkpoint_db_commit_latency_full_objects_history_chunks: Histogram,
    pub checkpoint_db_commit_latency_events: Histogram,
    pub checkpoint_db_commit_latency_events_chunks: Histogram,
    pub checkpoint_db_commit_latency_event_indices: Histogram,
    pub checkpoint_db_commit_latency_event_indices_chunks: Histogram,
    pub checkpoint_db_commit_latency_packages: Histogram,
    pub checkpoint_db_commit_latency_tx_indices: Histogram,
    pub checkpoint_db_commit_latency_tx_indices_chunks: Histogram,
    pub checkpoint_db_commit_latency_checkpoints: Histogram,
    pub checkpoint_db_commit_latency_epoch: Histogram,
    pub checkpoint_db_commit_latency_watermarks: Histogram,
    pub thousand_transaction_avg_db_commit_latency: Histogram,
    pub object_db_commit_latency: Histogram,
    pub object_mutation_db_commit_latency: Histogram,
    pub object_deletion_db_commit_latency: Histogram,
    pub epoch_db_commit_latency: Histogram,
    // latencies of slow DB update queries, now only advance epoch and objects_snapshot update
    pub advance_epoch_latency: Histogram,
    // latencies of RPC endpoints in read.rs
    pub get_transaction_block_latency: Histogram,
    pub multi_get_transaction_blocks_latency: Histogram,
    pub get_object_latency: Histogram,
    pub multi_get_objects_latency: Histogram,
    pub try_get_past_object_latency: Histogram,
    pub try_multi_get_past_objects_latency: Histogram,
    pub get_checkpoint_latency: Histogram,
    pub get_checkpoints_latency: Histogram,
    pub get_events_latency: Histogram,
    pub get_loaded_child_objects_latency: Histogram,
    pub get_total_transaction_blocks_latency: Histogram,
    pub get_latest_checkpoint_sequence_number_latency: Histogram,
    // latencies of RPC endpoints in indexer.rs
    pub get_owned_objects_latency: Histogram,
    pub query_transaction_blocks_latency: Histogram,
    pub query_events_latency: Histogram,
    pub get_dynamic_fields_latency: Histogram,
    pub get_dynamic_field_object_latency: Histogram,
    pub get_protocol_config_latency: Histogram,
    // latency of event websocket subscription
    pub subscription_process_latency: Histogram,
    pub transaction_per_checkpoint: Histogram,
    // indexer state metrics
    pub db_conn_pool_size: IntGauge,
    pub idle_db_conn: IntGauge,
    pub address_processor_failure: IntCounter,
    pub checkpoint_metrics_processor_failure: IntCounter,
    // pruner metrics
    pub last_pruned_epoch: IntGauge,
    pub last_pruned_checkpoint: IntGauge,
    pub last_pruned_transaction: IntGauge,
    pub epoch_pruning_latency: Histogram,
}

impl IndexerMetrics {
    pub fn new(registry: &Registry) -> Self {
        Self {
            total_checkpoint_received: register_int_counter_with_registry!(
                "total_checkpoint_received",
                "Total number of checkpoint received",
                registry,
            )
            .unwrap(),
            total_tx_checkpoint_committed: register_int_counter_with_registry!(
                "total_checkpoint_committed",
                "Total number of checkpoint committed",
                registry,
            )
            .unwrap(),
            total_object_checkpoint_committed: register_int_counter_with_registry!(
                "total_object_checkpoint_committed",
                "Total number of object checkpoint committed",
                registry,
            )
            .unwrap(),
            total_transaction_committed: register_int_counter_with_registry!(
                "total_transaction_committed",
                "Total number of transaction committed",
                registry,
            )
            .unwrap(),
            total_object_change_committed: register_int_counter_with_registry!(
                "total_object_change_committed",
                "Total number of object change committed",
                registry,
            )
            .unwrap(),
            total_transaction_chunk_committed: register_int_counter_with_registry!(
                "total_transaction_chunk_committed",
                "Total number of transaction chunk committed",
                registry,
            )
            .unwrap(),
            total_object_change_chunk_committed: register_int_counter_with_registry!(
                "total_object_change_chunk_committed",
                "Total number of object change chunk committed",
                registry,
            )
            .unwrap(),
            total_epoch_committed: register_int_counter_with_registry!(
                "total_epoch_committed",
                "Total number of epoch committed",
                registry,
            )
            .unwrap(),
            latest_fullnode_checkpoint_sequence_number: register_int_gauge_with_registry!(
                "latest_fullnode_checkpoint_sequence_number",
                "Latest checkpoint sequence number from the Full Node",
                registry,
            )
            .unwrap(),
            latest_tx_checkpoint_sequence_number: register_int_gauge_with_registry!(
                "latest_indexer_checkpoint_sequence_number",
                "Latest checkpoint sequence number from the Indexer",
                registry,
            )
            .unwrap(),
            latest_indexer_object_checkpoint_sequence_number: register_int_gauge_with_registry!(
                "latest_indexer_object_checkpoint_sequence_number",
                "Latest object checkpoint sequence number from the Indexer",
                registry,
            )
            .unwrap(),
            latest_object_snapshot_sequence_number: register_int_gauge_with_registry!(
                "latest_object_snapshot_sequence_number",
                "Latest object snapshot sequence number from the Indexer",
                registry,
            ).unwrap(),
            max_downloaded_checkpoint_sequence_number: register_int_gauge_with_registry!(
                "max_downloaded_checkpoint_sequence_number",
                "Max downloaded checkpoint sequence number",
                registry,
            ).unwrap(),
            max_indexed_checkpoint_sequence_number: register_int_gauge_with_registry!(
                "max_indexed_checkpoint_sequence_number",
                "Max indexed checkpoint sequence number",
                registry,
            ).unwrap(),
            max_committed_checkpoint_sequence_number: register_int_gauge_with_registry!(
                "max_committed_checkpoint_sequence_number",
                "Max committed checkpoint sequence number",
                registry,
            ).unwrap(),
            downloaded_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
                "downloaded_checkpoint_timestamp_ms",
                "Timestamp of the downloaded checkpoint",
                registry,
            ).unwrap(),
            indexed_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
                "indexed_checkpoint_timestamp_ms",
                "Timestamp of the indexed checkpoint",
                registry,
            ).unwrap(),
            committed_checkpoint_timestamp_ms: register_int_gauge_with_registry!(
                "committed_checkpoint_timestamp_ms",
                "Timestamp of the committed checkpoint",
                registry,
            ).unwrap(),
            download_lag_ms: register_int_gauge_with_registry!(
                "download_lag_ms",
                "Lag of the latest checkpoint in milliseconds",
                registry,
            ).unwrap(),
            index_lag_ms: register_int_gauge_with_registry!(
                "index_lag_ms",
                "Lag of the latest checkpoint in milliseconds",
                registry,
            ).unwrap(),
            db_commit_lag_ms: register_int_gauge_with_registry!(
                "db_commit_lag_ms",
                "Lag of the latest checkpoint in milliseconds",
                registry,
            ).unwrap(),
            checkpoint_download_bytes_size: register_int_gauge_with_registry!(
                "checkpoint_download_bytes_size",
                "Size of the downloaded checkpoint in bytes",
                registry,
            ).unwrap(),
            fullnode_checkpoint_data_download_latency: register_histogram_with_registry!(
                "fullnode_checkpoint_data_download_latency",
                "Time spent in downloading checkpoint and transaction for a new checkpoint from the Full Node",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            fullnode_checkpoint_wait_and_download_latency: register_histogram_with_registry!(
                "fullnode_checkpoint_wait_and_download_latency",
                "Time spent in waiting for a new checkpoint from the Full Node",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),

            fullnode_transaction_download_latency: register_histogram_with_registry!(
                "fullnode_transaction_download_latency",
                "Time spent in waiting for a new transaction from the Full Node",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            fullnode_object_download_latency: register_histogram_with_registry!(
                "fullnode_object_download_latency",
                "Time spent in waiting for a new epoch from the Full Node",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_index_latency: register_histogram_with_registry!(
                "checkpoint_index_latency",
                "Time spent in indexing a checkpoint",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            indexing_batch_size: register_int_gauge_with_registry!(
                "indexing_batch_size",
                "Size of the indexing batch",
                registry,
            ).unwrap(),
            indexing_tx_object_changes_latency: register_histogram_with_registry!(
                "indexing_tx_object_changes_latency",
                "Time spent in indexing object changes for a transaction",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            indexing_objects_latency: register_histogram_with_registry!(
                "indexing_objects_latency",
                "Time spent in indexing objects",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            indexing_packages_latency: register_histogram_with_registry!(
                "indexing_packages_latency",
                "Time spent in indexing packages",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            indexing_get_object_in_mem_hit: register_int_counter_with_registry!(
                "indexing_get_object_in_mem_hit",
                "Total number get object hit in mem",
                registry,
            )
            .unwrap(),
            indexing_get_object_db_hit: register_int_counter_with_registry!(
                "indexing_get_object_db_hit",
                "Total number get object hit in db",
                registry,
            )
            .unwrap(),
            indexing_module_resolver_in_mem_hit: register_int_counter_with_registry!(
                "indexing_module_resolver_in_mem_hit",
                "Total number module resolver hit in mem",
                registry,
            )
            .unwrap(),
            indexing_package_resolver_in_mem_hit: register_int_counter_with_registry!(
                "indexing_package_resolver_in_mem_hit",
                "Total number package resolver hit in mem",
                registry,
            )
            .unwrap(),
            checkpoint_objects_index_latency: register_histogram_with_registry!(
                "checkpoint_object_index_latency",
                "Time spent in indexing a checkpoint objects",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency: register_histogram_with_registry!(
                "checkpoint_db_commit_latency",
                "Time spent committing a checkpoint to the db",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),

            checkpoint_db_commit_latency_step_1: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_step_1",
                "Time spent committing a checkpoint to the db, step 1",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_transactions: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_transactions",
                "Time spent committing transactions",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_transactions_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_transactions_chunks",
                "Time spent committing transactions chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_transactions_chunks_transformation: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_transactions_transaformation",
                "Time spent in transactions chunks transformation prior to commit",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_objects: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects",
                "Time spent committing objects",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_objects_snapshot: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects_snapshot",
                "Time spent committing objects snapshots",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_objects_version: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects_version",
                "Time spent committing objects version",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            ).unwrap(),
            checkpoint_db_commit_latency_objects_history: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects_history",
                "Time spent committing objects history",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            ).unwrap(),
            checkpoint_db_commit_latency_full_objects_history: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_full_objects_history",
                "Time spent committing full objects history",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            ).unwrap(),
            checkpoint_db_commit_latency_objects_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects_chunks",
                "Time spent committing objects chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_objects_snapshot_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects_snapshot_chunks",
                "Time spent committing objects snapshot chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_objects_version_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects_version_chunks",
                "Time spent committing objects version chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            ).unwrap(),
            checkpoint_db_commit_latency_objects_history_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_objects_history_chunks",
                "Time spent committing objects history chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            ).unwrap(),
            checkpoint_db_commit_latency_full_objects_history_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_full_objects_history_chunks",
                "Time spent committing full objects history chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_events: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_events",
                "Time spent committing events",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_events_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_events_chunks",
                "Time spent committing events chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_event_indices: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_event_indices",
                "Time spent committing event indices",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_event_indices_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_event_indices_chunks",
                "Time spent committing event indices chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_packages: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_packages",
                "Time spent committing packages",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_tx_indices: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_tx_indices",
                "Time spent committing tx indices",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_tx_indices_chunks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_tx_indices_chunks",
                "Time spent committing tx_indices chunks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_checkpoints: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_checkpoints",
                "Time spent committing checkpoints",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_epoch: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_epochs",
                "Time spent committing epochs",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            checkpoint_db_commit_latency_watermarks: register_histogram_with_registry!(
                "checkpoint_db_commit_latency_watermarks",
                "Time spent committing watermarks",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            tokio_blocking_task_wait_latency: register_histogram_with_registry!(
                "tokio_blocking_task_wait_latency",
                "Time spent to wait for tokio blocking task pool",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            ).unwrap(),
            thousand_transaction_avg_db_commit_latency: register_histogram_with_registry!(
                "transaction_db_commit_latency",
                "Average time spent committing 1000 transactions to the db",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            object_db_commit_latency: register_histogram_with_registry!(
                "object_db_commit_latency",
                "Time spent committing a object to the db",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            object_mutation_db_commit_latency: register_histogram_with_registry!(
                "object_mutation_db_commit_latency",
                "Time spent committing a object mutation to the db",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            object_deletion_db_commit_latency: register_histogram_with_registry!(
                "object_deletion_db_commit_latency",
                "Time spent committing a object deletion to the db",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            epoch_db_commit_latency: register_histogram_with_registry!(
                "epoch_db_commit_latency",
                "Time spent committing a epoch to the db",
                DATA_INGESTION_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            advance_epoch_latency: register_histogram_with_registry!(
                "advance_epoch_latency",
                "Time spent in advancing epoch",
                DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            ).unwrap(),
            subscription_process_latency: register_histogram_with_registry!(
                "subscription_process_latency",
                "Time spent in process Websocket subscription",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry,
            )
            .unwrap(),
            transaction_per_checkpoint: register_histogram_with_registry!(
                "transaction_per_checkpoint",
                "Number of transactions per checkpoint",
                vec![1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0],
                registry,
            )
            .unwrap(),
            get_transaction_block_latency: register_histogram_with_registry!(
                "get_transaction_block_latency",
                "Time spent in get_transaction_block on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            multi_get_transaction_blocks_latency: register_histogram_with_registry!(
                "multi_get_transaction_blocks_latency",
                "Time spent in multi_get_transaction_blocks on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_object_latency: register_histogram_with_registry!(
                "get_object_latency",
                "Time spent in get_object on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            multi_get_objects_latency: register_histogram_with_registry!(
                "multi_get_objects_latency",
                "Time spent in multi_get_objects on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            try_get_past_object_latency: register_histogram_with_registry!(
                "try_get_past_object_latency",
                "Time spent in try_get_past_object on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            try_multi_get_past_objects_latency: register_histogram_with_registry!(
                "try_multi_get_past_objects_latency",
                "Time spent in try_multi_get_past_objects on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_checkpoint_latency: register_histogram_with_registry!(
                "get_checkpoint_latency",
                "Time spent in get_checkpoint on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_checkpoints_latency: register_histogram_with_registry!(
                "get_checkpoints_latency",
                "Time spent in get_checkpoints on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_events_latency: register_histogram_with_registry!(
                "get_events_latency",
                "Time spent in get_events on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_total_transaction_blocks_latency: register_histogram_with_registry!(
                "get_total_transaction_blocks_latency",
                "Time spent in get_total_transaction_blocks on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_latest_checkpoint_sequence_number_latency: register_histogram_with_registry!(
                "get_latest_checkpoint_sequence_number_latency",
                "Time spent in get_latest_checkpoint_sequence_number on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_owned_objects_latency: register_histogram_with_registry!(
                "get_owned_objects_latency",
                "Time spent in get_owned_objects on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            query_transaction_blocks_latency: register_histogram_with_registry!(
                "query_transaction_blocks_latency",
                "Time spent in query_transaction_blocks on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            query_events_latency: register_histogram_with_registry!(
                "query_events_latency",
                "Time spent in query_events on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_dynamic_fields_latency: register_histogram_with_registry!(
                "get_dynamic_fields_latency",
                "Time spent in get_dynamic_fields on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_dynamic_field_object_latency: register_histogram_with_registry!(
                "get_dynamic_field_object_latency",
                "Time spent in get_dynamic_field_object on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_loaded_child_objects_latency: register_histogram_with_registry!(
                "get_loaded_child_objects_latency",
                "Time spent in get_loaded_child_objects_latency on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            get_protocol_config_latency: register_histogram_with_registry!(
                "get_protocol_config_latency",
                "Time spent in get_protocol_config_latency on the fullnode behind.",
                JSON_RPC_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            )
            .unwrap(),
            db_conn_pool_size: register_int_gauge_with_registry!(
                "db_conn_pool_size",
                "Size of the database connection pool",
                registry
            ).unwrap(),
            idle_db_conn: register_int_gauge_with_registry!(
                "idle_db_conn",
                "Number of idle database connections",
                registry
            ).unwrap(),
            address_processor_failure: register_int_counter_with_registry!(
                "address_processor_failure",
                "Total number of address processor failure",
                registry,
            )
            .unwrap(),
            checkpoint_metrics_processor_failure: register_int_counter_with_registry!(
                "checkpoint_metrics_processor_failure",
                "Total number of checkpoint metrics processor failure",
                registry,
            )
            .unwrap(),
            last_pruned_epoch: register_int_gauge_with_registry!(
                "last_pruned_epoch",
                "Last pruned epoch number",
                registry,
            )
            .unwrap(),
            last_pruned_checkpoint: register_int_gauge_with_registry!(
                "last_pruned_checkpoint",
                "Last pruned checkpoint sequence number",
                registry,
            )
            .unwrap(),
            last_pruned_transaction: register_int_gauge_with_registry!(
                "last_pruned_transaction",
                "Last pruned transaction sequence number",
                registry,
            ).unwrap(),
            epoch_pruning_latency: register_histogram_with_registry!(
                "epoch_pruning_latency",
                "Time spent in pruning one epoch",
                DB_UPDATE_QUERY_LATENCY_SEC_BUCKETS.to_vec(),
                registry
            ).unwrap(),
        }
    }
}

pub fn spawn_connection_pool_metric_collector(
    metrics: IndexerMetrics,
    connection_pool: crate::database::ConnectionPool,
) {
    tokio::spawn(async move {
        loop {
            let cp_state = connection_pool.state();
            tracing::debug!(
                connection_pool_size =% cp_state.connections,
                idle_connections =% cp_state.idle_connections,
            );
            metrics.db_conn_pool_size.set(cp_state.connections as i64);
            metrics.idle_db_conn.set(cp_state.idle_connections as i64);
            tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
        }
    });
}