sui_tool/db_tool/
db_dump.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use anyhow::{Ok, anyhow};
5use clap::{Parser, ValueEnum};
6#[cfg(not(tidehunter))]
7use comfy_table::{Cell, ContentArrangement, Row, Table};
8use prometheus::Registry;
9use std::collections::BTreeMap;
10use std::path::PathBuf;
11use std::str;
12use std::sync::Arc;
13use strum_macros::EnumString;
14use sui_config::node::AuthorityStorePruningConfig;
15use sui_core::authority::authority_per_epoch_store::AuthorityEpochTables;
16use sui_core::authority::authority_store_pruner::{
17    AuthorityStorePruner, AuthorityStorePruningMetrics, EPOCH_DURATION_MS_FOR_TESTING,
18    PrunerWatermarks,
19};
20use sui_core::authority::authority_store_tables::AuthorityPerpetualTables;
21use sui_core::checkpoints::CheckpointStore;
22use sui_core::epoch::committee_store::CommitteeStoreTables;
23use sui_core::jsonrpc_index::IndexStoreTables;
24use sui_types::base_types::EpochId;
25use tracing::info;
26use typed_store::rocks::{MetricConf, default_db_options};
27use typed_store::rocksdb::MultiThreaded;
28use typed_store::traits::TableSummary;
29
30#[derive(EnumString, Clone, Parser, Debug, ValueEnum)]
31pub enum StoreName {
32    Validator,
33    Index,
34    Epoch,
35    // TODO: Add the new checkpoint v2 tables.
36}
37impl std::fmt::Display for StoreName {
38    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
39        write!(f, "{:?}", self)
40    }
41}
42
43pub fn list_tables(path: PathBuf) -> anyhow::Result<Vec<String>> {
44    typed_store::rocksdb::DBWithThreadMode::<MultiThreaded>::list_cf(
45        &default_db_options().options,
46        path,
47    )
48    .map_err(|e| e.into())
49    .map(|q| {
50        q.iter()
51            .filter_map(|s| {
52                // The `default` table is not used
53                if s != "default" {
54                    Some(s.clone())
55                } else {
56                    None
57                }
58            })
59            .collect()
60    })
61}
62
63pub fn table_summary(
64    store_name: StoreName,
65    epoch: Option<EpochId>,
66    db_path: PathBuf,
67    table_name: &str,
68) -> anyhow::Result<TableSummary> {
69    match store_name {
70        StoreName::Validator => {
71            let epoch_tables = AuthorityEpochTables::describe_tables();
72            if epoch_tables.contains_key(table_name) {
73                let epoch = epoch.ok_or_else(|| anyhow!("--epoch is required"))?;
74                AuthorityEpochTables::open_readonly(epoch, &db_path).table_summary(table_name)
75            } else {
76                AuthorityPerpetualTables::open_readonly(&db_path).table_summary(table_name)
77            }
78        }
79        StoreName::Index => {
80            IndexStoreTables::get_read_only_handle(db_path, None, None, MetricConf::default())
81                .table_summary(table_name)
82        }
83        StoreName::Epoch => {
84            CommitteeStoreTables::get_read_only_handle(db_path, None, None, MetricConf::default())
85                .table_summary(table_name)
86        }
87    }
88    .map_err(|err| anyhow!(err.to_string()))
89}
90
91pub fn print_table_metadata(
92    store_name: StoreName,
93    epoch: Option<EpochId>,
94    db_path: PathBuf,
95    table_name: &str,
96) -> anyhow::Result<()> {
97    #[cfg(not(tidehunter))]
98    {
99        let db = match store_name {
100            StoreName::Validator => {
101                let epoch_tables = AuthorityEpochTables::describe_tables();
102                if epoch_tables.contains_key(table_name) {
103                    let epoch = epoch.ok_or_else(|| anyhow!("--epoch is required"))?;
104                    AuthorityEpochTables::open_readonly(epoch, &db_path)
105                        .next_shared_object_versions_v2
106                        .db
107                } else {
108                    AuthorityPerpetualTables::open_readonly(&db_path).objects.db
109                }
110            }
111            StoreName::Index => {
112                IndexStoreTables::get_read_only_handle(db_path, None, None, MetricConf::default())
113                    .event_by_move_module
114                    .db
115            }
116            StoreName::Epoch => {
117                CommitteeStoreTables::get_read_only_handle(
118                    db_path,
119                    None,
120                    None,
121                    MetricConf::default(),
122                )
123                .committee_map
124                .db
125            }
126        };
127
128        let mut table = Table::new();
129        table
130            .set_content_arrangement(ContentArrangement::Dynamic)
131            .set_width(200)
132            .set_header(vec![
133                "name",
134                "level",
135                "num_entries",
136                "start_key",
137                "end_key",
138                "num_deletions",
139                "file_size",
140            ]);
141
142        for file in db.live_files()?.iter() {
143            if file.column_family_name != table_name {
144                continue;
145            }
146            let mut row = Row::new();
147            row.add_cell(Cell::new(&file.name));
148            row.add_cell(Cell::new(file.level));
149            row.add_cell(Cell::new(file.num_entries));
150            row.add_cell(Cell::new(hex::encode(
151                file.start_key.as_ref().unwrap_or(&"".as_bytes().to_vec()),
152            )));
153            row.add_cell(Cell::new(hex::encode(
154                file.end_key.as_ref().unwrap_or(&"".as_bytes().to_vec()),
155            )));
156            row.add_cell(Cell::new(file.num_deletions));
157            row.add_cell(Cell::new(file.size));
158            table.add_row(row);
159        }
160
161        eprintln!("{}", table);
162    }
163    // Table metadata is only available for the rocksdb backend.
164    #[cfg(tidehunter)]
165    let _ = (store_name, epoch, db_path, table_name);
166    Ok(())
167}
168
169pub fn compact(db_path: PathBuf) -> anyhow::Result<()> {
170    let perpetual = Arc::new(AuthorityPerpetualTables::open(&db_path, None, None));
171    AuthorityStorePruner::compact(&perpetual)?;
172    Ok(())
173}
174
175pub async fn prune_objects(db_path: PathBuf) -> anyhow::Result<()> {
176    let perpetual_db = Arc::new(AuthorityPerpetualTables::open(
177        &db_path.join("store"),
178        None,
179        None,
180    ));
181    let checkpoint_store = CheckpointStore::new(
182        &db_path.join("checkpoints"),
183        Arc::new(PrunerWatermarks::default()),
184    );
185    let highest_pruned_checkpoint = checkpoint_store
186        .get_highest_pruned_checkpoint_seq_number()?
187        .unwrap_or(0);
188    let latest_checkpoint = checkpoint_store.get_highest_executed_checkpoint()?;
189    info!(
190        "Latest executed checkpoint sequence num: {}",
191        latest_checkpoint.map(|x| x.sequence_number).unwrap_or(0)
192    );
193    info!("Highest pruned checkpoint: {}", highest_pruned_checkpoint);
194    let metrics = AuthorityStorePruningMetrics::new(&Registry::default());
195    info!("Pruning setup for db at path: {:?}", db_path.display());
196    let pruning_config = AuthorityStorePruningConfig {
197        num_epochs_to_retain: 0,
198        ..Default::default()
199    };
200    info!("Starting object pruning");
201    AuthorityStorePruner::prune_objects_for_eligible_epochs(
202        &perpetual_db,
203        &checkpoint_store,
204        None,
205        pruning_config,
206        metrics,
207        EPOCH_DURATION_MS_FOR_TESTING,
208    )
209    .await?;
210    Ok(())
211}
212
213pub async fn prune_checkpoints(db_path: PathBuf) -> anyhow::Result<()> {
214    let perpetual_db = Arc::new(AuthorityPerpetualTables::open(
215        &db_path.join("store"),
216        None,
217        None,
218    ));
219    let checkpoint_store = CheckpointStore::new(
220        &db_path.join("checkpoints"),
221        Arc::new(PrunerWatermarks::default()),
222    );
223    let metrics = AuthorityStorePruningMetrics::new(&Registry::default());
224    info!("Pruning setup for db at path: {:?}", db_path.display());
225    let pruning_config = AuthorityStorePruningConfig {
226        num_epochs_to_retain_for_checkpoints: Some(1),
227        ..Default::default()
228    };
229    info!("Starting txns and effects pruning");
230    use sui_core::authority::authority_store_pruner::PrunerWatermarks;
231    let watermarks = std::sync::Arc::new(PrunerWatermarks::default());
232    AuthorityStorePruner::prune_checkpoints_for_eligible_epochs(
233        &perpetual_db,
234        &checkpoint_store,
235        None,
236        pruning_config,
237        metrics,
238        EPOCH_DURATION_MS_FOR_TESTING,
239        &watermarks,
240    )
241    .await?;
242    Ok(())
243}
244
245// TODO: condense this using macro or trait dyn skills
246pub fn dump_table(
247    store_name: StoreName,
248    epoch: Option<EpochId>,
249    db_path: PathBuf,
250    table_name: &str,
251    page_size: u16,
252    page_number: usize,
253) -> anyhow::Result<BTreeMap<String, String>> {
254    match store_name {
255        StoreName::Validator => {
256            let epoch_tables = AuthorityEpochTables::describe_tables();
257            if epoch_tables.contains_key(table_name) {
258                let epoch = epoch.ok_or_else(|| anyhow!("--epoch is required"))?;
259                AuthorityEpochTables::open_readonly(epoch, &db_path).dump(
260                    table_name,
261                    page_size,
262                    page_number,
263                )
264            } else {
265                let perpetual_tables = AuthorityPerpetualTables::describe_tables();
266                assert!(perpetual_tables.contains_key(table_name));
267                AuthorityPerpetualTables::open_readonly(&db_path).dump(
268                    table_name,
269                    page_size,
270                    page_number,
271                )
272            }
273        }
274        StoreName::Index => {
275            IndexStoreTables::get_read_only_handle(db_path, None, None, MetricConf::default()).dump(
276                table_name,
277                page_size,
278                page_number,
279            )
280        }
281        StoreName::Epoch => {
282            CommitteeStoreTables::get_read_only_handle(db_path, None, None, MetricConf::default())
283                .dump(table_name, page_size, page_number)
284        }
285    }
286    .map_err(|err| anyhow!(err.to_string()))
287}
288
289#[cfg(test)]
290mod test {
291    use sui_core::authority::authority_per_epoch_store::AuthorityEpochTables;
292    use sui_core::authority::authority_store_tables::AuthorityPerpetualTables;
293
294    use crate::db_tool::db_dump::{StoreName, dump_table, list_tables};
295
296    #[tokio::test]
297    async fn db_dump_population() -> Result<(), anyhow::Error> {
298        let primary_path = tempfile::tempdir()?.keep();
299
300        // Open the DB for writing
301        let _: AuthorityEpochTables = AuthorityEpochTables::open(0, &primary_path, None);
302        let _: AuthorityPerpetualTables = AuthorityPerpetualTables::open(&primary_path, None, None);
303
304        // Get all the tables for AuthorityEpochTables
305        let tables = {
306            let mut epoch_tables =
307                list_tables(AuthorityEpochTables::path(0, &primary_path)).unwrap();
308            let mut perpetual_tables =
309                list_tables(AuthorityPerpetualTables::path(&primary_path)).unwrap();
310            epoch_tables.append(&mut perpetual_tables);
311            epoch_tables
312        };
313
314        let mut missing_tables = vec![];
315        for t in tables {
316            println!("{}", t);
317            if dump_table(
318                StoreName::Validator,
319                Some(0),
320                primary_path.clone(),
321                &t,
322                0,
323                0,
324            )
325            .is_err()
326            {
327                missing_tables.push(t);
328            }
329        }
330        if missing_tables.is_empty() {
331            return Ok(());
332        }
333        panic!(
334            "{}",
335            format!(
336                "Missing {} table(s) from DB dump registration function: {:?} \n Update the dump function.",
337                missing_tables.len(),
338                missing_tables
339            )
340        );
341    }
342}
sui_tool/db_tool/db_dump.rs

sui_tool/db_tool/
db_dump.rs
