sui_tool/db_tool/
db_dump.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use anyhow::{Ok, anyhow};
5use clap::{Parser, ValueEnum};
6#[cfg(not(tidehunter))]
7use comfy_table::{Cell, ContentArrangement, Row, Table};
8use prometheus::Registry;
9use std::collections::BTreeMap;
10use std::path::PathBuf;
11use std::str;
12use std::sync::Arc;
13use strum_macros::EnumString;
14use sui_config::node::AuthorityStorePruningConfig;
15use sui_core::authority::authority_per_epoch_store::AuthorityEpochTables;
16use sui_core::authority::authority_store_pruner::{
17    AuthorityStorePruner, AuthorityStorePruningMetrics, EPOCH_DURATION_MS_FOR_TESTING,
18    PrunerWatermarks,
19};
20use sui_core::authority::authority_store_tables::AuthorityPerpetualTables;
21use sui_core::checkpoints::CheckpointStore;
22use sui_core::epoch::committee_store::CommitteeStoreTables;
23use sui_types::base_types::EpochId;
24use tracing::info;
25use typed_store::rocks::{MetricConf, default_db_options};
26use typed_store::rocksdb::MultiThreaded;
27use typed_store::traits::TableSummary;
28
29#[derive(EnumString, Clone, Parser, Debug, ValueEnum)]
30pub enum StoreName {
31    Validator,
32    Epoch,
33    // TODO: Add the new checkpoint v2 tables.
34}
35impl std::fmt::Display for StoreName {
36    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
37        write!(f, "{:?}", self)
38    }
39}
40
41pub fn list_tables(path: PathBuf) -> anyhow::Result<Vec<String>> {
42    typed_store::rocksdb::DBWithThreadMode::<MultiThreaded>::list_cf(
43        &default_db_options().options,
44        path,
45    )
46    .map_err(|e| e.into())
47    .map(|q| {
48        q.iter()
49            .filter_map(|s| {
50                // The `default` table is not used
51                if s != "default" {
52                    Some(s.clone())
53                } else {
54                    None
55                }
56            })
57            .collect()
58    })
59}
60
61pub fn table_summary(
62    store_name: StoreName,
63    epoch: Option<EpochId>,
64    db_path: PathBuf,
65    table_name: &str,
66) -> anyhow::Result<TableSummary> {
67    match store_name {
68        StoreName::Validator => {
69            let epoch_tables = AuthorityEpochTables::describe_tables();
70            if epoch_tables.contains_key(table_name) {
71                let epoch = epoch.ok_or_else(|| anyhow!("--epoch is required"))?;
72                AuthorityEpochTables::open_readonly(epoch, &db_path).table_summary(table_name)
73            } else {
74                AuthorityPerpetualTables::open_readonly(&db_path).table_summary(table_name)
75            }
76        }
77        StoreName::Epoch => {
78            CommitteeStoreTables::get_read_only_handle(db_path, None, None, MetricConf::default())
79                .table_summary(table_name)
80        }
81    }
82    .map_err(|err| anyhow!(err.to_string()))
83}
84
85pub fn print_table_metadata(
86    store_name: StoreName,
87    epoch: Option<EpochId>,
88    db_path: PathBuf,
89    table_name: &str,
90) -> anyhow::Result<()> {
91    #[cfg(not(tidehunter))]
92    {
93        let db = match store_name {
94            StoreName::Validator => {
95                let epoch_tables = AuthorityEpochTables::describe_tables();
96                if epoch_tables.contains_key(table_name) {
97                    let epoch = epoch.ok_or_else(|| anyhow!("--epoch is required"))?;
98                    AuthorityEpochTables::open_readonly(epoch, &db_path)
99                        .next_shared_object_versions_v2
100                        .db
101                } else {
102                    AuthorityPerpetualTables::open_readonly(&db_path).objects.db
103                }
104            }
105            StoreName::Epoch => {
106                CommitteeStoreTables::get_read_only_handle(
107                    db_path,
108                    None,
109                    None,
110                    MetricConf::default(),
111                )
112                .committee_map
113                .db
114            }
115        };
116
117        let mut table = Table::new();
118        table
119            .set_content_arrangement(ContentArrangement::Dynamic)
120            .set_width(200)
121            .set_header(vec![
122                "name",
123                "level",
124                "num_entries",
125                "start_key",
126                "end_key",
127                "num_deletions",
128                "file_size",
129            ]);
130
131        for file in db.live_files()?.iter() {
132            if file.column_family_name != table_name {
133                continue;
134            }
135            let mut row = Row::new();
136            row.add_cell(Cell::new(&file.name));
137            row.add_cell(Cell::new(file.level));
138            row.add_cell(Cell::new(file.num_entries));
139            row.add_cell(Cell::new(hex::encode(
140                file.start_key.as_ref().unwrap_or(&"".as_bytes().to_vec()),
141            )));
142            row.add_cell(Cell::new(hex::encode(
143                file.end_key.as_ref().unwrap_or(&"".as_bytes().to_vec()),
144            )));
145            row.add_cell(Cell::new(file.num_deletions));
146            row.add_cell(Cell::new(file.size));
147            table.add_row(row);
148        }
149
150        eprintln!("{}", table);
151    }
152    // Table metadata is only available for the rocksdb backend.
153    #[cfg(tidehunter)]
154    let _ = (store_name, epoch, db_path, table_name);
155    Ok(())
156}
157
158pub fn compact(db_path: PathBuf) -> anyhow::Result<()> {
159    let perpetual = Arc::new(AuthorityPerpetualTables::open(&db_path, None, None));
160    AuthorityStorePruner::compact(&perpetual)?;
161    Ok(())
162}
163
164pub async fn prune_objects(db_path: PathBuf) -> anyhow::Result<()> {
165    let perpetual_db = Arc::new(AuthorityPerpetualTables::open(
166        &db_path.join("store"),
167        None,
168        None,
169    ));
170    let checkpoint_store = CheckpointStore::new(
171        &db_path.join("checkpoints"),
172        Arc::new(PrunerWatermarks::default()),
173    );
174    let highest_pruned_checkpoint = checkpoint_store
175        .get_highest_pruned_checkpoint_seq_number()?
176        .unwrap_or(0);
177    let latest_checkpoint = checkpoint_store.get_highest_executed_checkpoint()?;
178    info!(
179        "Latest executed checkpoint sequence num: {}",
180        latest_checkpoint.map(|x| x.sequence_number).unwrap_or(0)
181    );
182    info!("Highest pruned checkpoint: {}", highest_pruned_checkpoint);
183    let metrics = AuthorityStorePruningMetrics::new(&Registry::default());
184    info!("Pruning setup for db at path: {:?}", db_path.display());
185    let pruning_config = AuthorityStorePruningConfig {
186        num_epochs_to_retain: 0,
187        ..Default::default()
188    };
189    info!("Starting object pruning");
190    AuthorityStorePruner::prune_objects_for_eligible_epochs(
191        &perpetual_db,
192        &checkpoint_store,
193        None,
194        &mut sui_core::authority::authority_store_pruner::RetractionCursors::default(),
195        pruning_config,
196        metrics,
197        EPOCH_DURATION_MS_FOR_TESTING,
198    )
199    .await?;
200    Ok(())
201}
202
203pub async fn prune_checkpoints(db_path: PathBuf) -> anyhow::Result<()> {
204    let perpetual_db = Arc::new(AuthorityPerpetualTables::open(
205        &db_path.join("store"),
206        None,
207        None,
208    ));
209    let checkpoint_store = CheckpointStore::new(
210        &db_path.join("checkpoints"),
211        Arc::new(PrunerWatermarks::default()),
212    );
213    let metrics = AuthorityStorePruningMetrics::new(&Registry::default());
214    info!("Pruning setup for db at path: {:?}", db_path.display());
215    let pruning_config = AuthorityStorePruningConfig {
216        num_epochs_to_retain_for_checkpoints: Some(1),
217        ..Default::default()
218    };
219    info!("Starting txns and effects pruning");
220    use sui_core::authority::authority_store_pruner::PrunerWatermarks;
221    let watermarks = std::sync::Arc::new(PrunerWatermarks::default());
222    AuthorityStorePruner::prune_checkpoints_for_eligible_epochs(
223        &perpetual_db,
224        &checkpoint_store,
225        None,
226        pruning_config,
227        metrics,
228        EPOCH_DURATION_MS_FOR_TESTING,
229        &watermarks,
230    )
231    .await?;
232    Ok(())
233}
234
235// TODO: condense this using macro or trait dyn skills
236pub fn dump_table(
237    store_name: StoreName,
238    epoch: Option<EpochId>,
239    db_path: PathBuf,
240    table_name: &str,
241    page_size: u16,
242    page_number: usize,
243) -> anyhow::Result<BTreeMap<String, String>> {
244    match store_name {
245        StoreName::Validator => {
246            let epoch_tables = AuthorityEpochTables::describe_tables();
247            if epoch_tables.contains_key(table_name) {
248                let epoch = epoch.ok_or_else(|| anyhow!("--epoch is required"))?;
249                AuthorityEpochTables::open_readonly(epoch, &db_path).dump(
250                    table_name,
251                    page_size,
252                    page_number,
253                )
254            } else {
255                let perpetual_tables = AuthorityPerpetualTables::describe_tables();
256                assert!(perpetual_tables.contains_key(table_name));
257                AuthorityPerpetualTables::open_readonly(&db_path).dump(
258                    table_name,
259                    page_size,
260                    page_number,
261                )
262            }
263        }
264        StoreName::Epoch => {
265            CommitteeStoreTables::get_read_only_handle(db_path, None, None, MetricConf::default())
266                .dump(table_name, page_size, page_number)
267        }
268    }
269    .map_err(|err| anyhow!(err.to_string()))
270}
271
272#[cfg(test)]
273mod test {
274    use sui_core::authority::authority_per_epoch_store::AuthorityEpochTables;
275    use sui_core::authority::authority_store_tables::AuthorityPerpetualTables;
276
277    use crate::db_tool::db_dump::{StoreName, dump_table, list_tables};
278
279    #[tokio::test]
280    async fn db_dump_population() -> Result<(), anyhow::Error> {
281        let primary_path = tempfile::tempdir()?.keep();
282
283        // Open the DB for writing
284        let _: AuthorityEpochTables = AuthorityEpochTables::open(0, &primary_path, None);
285        let _: AuthorityPerpetualTables = AuthorityPerpetualTables::open(&primary_path, None, None);
286
287        // Get all the tables for AuthorityEpochTables
288        let tables = {
289            let mut epoch_tables =
290                list_tables(AuthorityEpochTables::path(0, &primary_path)).unwrap();
291            let mut perpetual_tables =
292                list_tables(AuthorityPerpetualTables::path(&primary_path)).unwrap();
293            epoch_tables.append(&mut perpetual_tables);
294            epoch_tables
295        };
296
297        let mut missing_tables = vec![];
298        for t in tables {
299            println!("{}", t);
300            if dump_table(
301                StoreName::Validator,
302                Some(0),
303                primary_path.clone(),
304                &t,
305                0,
306                0,
307            )
308            .is_err()
309            {
310                missing_tables.push(t);
311            }
312        }
313        if missing_tables.is_empty() {
314            return Ok(());
315        }
316        panic!(
317            "{}",
318            format!(
319                "Missing {} table(s) from DB dump registration function: {:?} \n Update the dump function.",
320                missing_tables.len(),
321                missing_tables
322            )
323        );
324    }
325}
sui_tool/db_tool/db_dump.rs

sui_tool/db_tool/
db_dump.rs
