sui_analytics_indexer/handlers/tables/
mod.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Table-specific processors for analytics pipelines.
5//!
6//! Each module implements a `Processor` that extracts data from checkpoints
7//! and produces typed entries for a specific analytics table.
8
9use std::collections::BTreeMap;
10use std::collections::BTreeSet;
11
12use anyhow::Result;
13use anyhow::anyhow;
14use move_core_types::annotated_value::MoveStruct;
15use move_core_types::annotated_value::MoveTypeLayout;
16use move_core_types::annotated_value::MoveValue;
17use move_core_types::language_storage::StructTag;
18use move_core_types::language_storage::TypeTag;
19use sui_package_resolver::PackageStore;
20use sui_package_resolver::Resolver;
21use sui_types::base_types::ObjectID;
22use sui_types::effects::TransactionEffects;
23use sui_types::effects::TransactionEffectsAPI;
24use sui_types::object::Object;
25use sui_types::object::Owner;
26use sui_types::object::bounded_visitor::BoundedVisitor;
27use sui_types::transaction::TransactionData;
28use sui_types::transaction::TransactionDataAPI;
29
30use crate::tables::InputObjectKind;
31use crate::tables::ObjectStatus;
32use crate::tables::OwnerType;
33
34pub mod checkpoint;
35pub mod df;
36pub mod event;
37pub mod move_call;
38pub mod object;
39pub mod package;
40pub mod package_bcs;
41pub mod transaction;
42pub mod transaction_bcs;
43pub mod transaction_objects;
44pub mod wrapped_object;
45
46pub use checkpoint::CheckpointProcessor;
47pub use df::DynamicFieldProcessor;
48pub use event::EventProcessor;
49pub use move_call::MoveCallProcessor;
50pub use object::ObjectProcessor;
51pub use package::PackageProcessor;
52pub use package_bcs::PackageBCSProcessor;
53pub use transaction::TransactionProcessor;
54pub use transaction_bcs::TransactionBCSProcessor;
55pub use transaction_objects::TransactionObjectsProcessor;
56pub use wrapped_object::WrappedObjectProcessor;
57
58const WRAPPED_INDEXING_DISALLOW_LIST: [&str; 4] = [
59    "0x1::string::String",
60    "0x1::ascii::String",
61    "0x2::url::Url",
62    "0x2::object::ID",
63];
64
65#[derive(Debug, Default)]
66pub struct WrappedStruct {
67    pub object_id: Option<ObjectID>,
68    pub struct_tag: Option<StructTag>,
69}
70
71pub struct InputObjectTracker {
72    shared: BTreeSet<ObjectID>,
73    coins: BTreeSet<ObjectID>,
74    input: BTreeSet<ObjectID>,
75}
76
77pub struct ObjectStatusTracker {
78    created: BTreeSet<ObjectID>,
79    mutated: BTreeSet<ObjectID>,
80    deleted: BTreeSet<ObjectID>,
81}
82
83impl InputObjectTracker {
84    pub fn new(txn_data: &TransactionData) -> Self {
85        let shared: BTreeSet<ObjectID> = txn_data
86            .shared_input_objects()
87            .iter()
88            .map(|shared_io| shared_io.id())
89            .collect();
90        let coins: BTreeSet<ObjectID> = txn_data.gas().iter().map(|obj_ref| obj_ref.0).collect();
91        let input: BTreeSet<ObjectID> = txn_data
92            .input_objects()
93            .expect("Input objects must be valid")
94            .iter()
95            .map(|io_kind| io_kind.object_id())
96            .collect();
97        Self {
98            shared,
99            coins,
100            input,
101        }
102    }
103
104    pub fn get_input_object_kind(&self, object_id: &ObjectID) -> Option<InputObjectKind> {
105        if self.coins.contains(object_id) {
106            Some(InputObjectKind::GasCoin)
107        } else if self.shared.contains(object_id) {
108            Some(InputObjectKind::SharedInput)
109        } else if self.input.contains(object_id) {
110            Some(InputObjectKind::Input)
111        } else {
112            None
113        }
114    }
115}
116
117impl ObjectStatusTracker {
118    pub fn new(effects: &TransactionEffects) -> Self {
119        let created: BTreeSet<ObjectID> = effects
120            .created()
121            .iter()
122            .map(|(obj_ref, _)| obj_ref.0)
123            .collect();
124        let mutated: BTreeSet<ObjectID> = effects
125            .mutated()
126            .iter()
127            .chain(effects.unwrapped().iter())
128            .map(|(obj_ref, _)| obj_ref.0)
129            .collect();
130        let deleted: BTreeSet<ObjectID> = effects
131            .all_tombstones()
132            .into_iter()
133            .map(|(id, _)| id)
134            .collect();
135        Self {
136            created,
137            mutated,
138            deleted,
139        }
140    }
141
142    pub fn get_object_status(&self, object_id: &ObjectID) -> Option<ObjectStatus> {
143        if self.mutated.contains(object_id) {
144            Some(ObjectStatus::Mutated)
145        } else if self.deleted.contains(object_id) {
146            Some(ObjectStatus::Deleted)
147        } else if self.created.contains(object_id) {
148            Some(ObjectStatus::Created)
149        } else {
150            None
151        }
152    }
153}
154
155pub fn initial_shared_version(object: &Object) -> Option<u64> {
156    match object.owner {
157        Owner::Shared {
158            initial_shared_version,
159        } => Some(initial_shared_version.value()),
160        _ => None,
161    }
162}
163
164pub fn get_owner_type(object: &Object) -> OwnerType {
165    match object.owner {
166        Owner::AddressOwner(_) => OwnerType::AddressOwner,
167        Owner::ObjectOwner(_) => OwnerType::ObjectOwner,
168        Owner::Shared { .. } => OwnerType::Shared,
169        Owner::Immutable => OwnerType::Immutable,
170        Owner::ConsensusAddressOwner { .. } => OwnerType::AddressOwner,
171    }
172}
173
174pub fn get_owner_address(object: &Object) -> Option<String> {
175    match object.owner {
176        Owner::AddressOwner(address) => Some(address.to_string()),
177        Owner::ObjectOwner(address) => Some(address.to_string()),
178        Owner::Shared { .. } => None,
179        Owner::Immutable => None,
180        Owner::ConsensusAddressOwner { owner, .. } => Some(owner.to_string()),
181    }
182}
183
184pub fn get_is_consensus(object: &Object) -> bool {
185    match object.owner {
186        Owner::AddressOwner(_) => false,
187        Owner::ObjectOwner(_) => false,
188        Owner::Shared { .. } => true,
189        Owner::Immutable => false,
190        Owner::ConsensusAddressOwner { .. } => true,
191    }
192}
193
194pub async fn get_move_struct<T: PackageStore>(
195    struct_tag: &StructTag,
196    contents: &[u8],
197    resolver: &Resolver<T>,
198) -> Result<MoveStruct> {
199    let move_struct = match resolver
200        .type_layout(TypeTag::Struct(Box::new(struct_tag.clone())))
201        .await?
202    {
203        MoveTypeLayout::Struct(move_struct_layout) => {
204            BoundedVisitor::deserialize_struct(contents, &move_struct_layout)
205        }
206        _ => Err(anyhow!("Object is not a move struct")),
207    }?;
208    Ok(move_struct)
209}
210
211pub fn parse_struct(
212    path: &str,
213    move_struct: MoveStruct,
214    all_structs: &mut BTreeMap<String, WrappedStruct>,
215) {
216    let mut wrapped_struct = WrappedStruct {
217        struct_tag: Some(move_struct.type_),
218        ..Default::default()
219    };
220    for (k, v) in move_struct.fields {
221        parse_struct_field(
222            &format!("{}.{}", path, &k),
223            v,
224            &mut wrapped_struct,
225            all_structs,
226        );
227    }
228    all_structs.insert(path.to_string(), wrapped_struct);
229}
230
231fn parse_struct_field(
232    path: &str,
233    move_value: MoveValue,
234    curr_struct: &mut WrappedStruct,
235    all_structs: &mut BTreeMap<String, WrappedStruct>,
236) {
237    match move_value {
238        MoveValue::Struct(move_struct) => {
239            let values = move_struct
240                .fields
241                .iter()
242                .map(|(id, value)| (id.to_string(), value))
243                .collect::<BTreeMap<_, _>>();
244            let struct_name = format!(
245                "0x{}::{}::{}",
246                move_struct.type_.address.short_str_lossless(),
247                move_struct.type_.module,
248                move_struct.type_.name
249            );
250            if "0x2::object::UID" == struct_name {
251                if let Some(MoveValue::Struct(id_struct)) = values.get("id").cloned() {
252                    let id_values = id_struct
253                        .fields
254                        .iter()
255                        .map(|(id, value)| (id.to_string(), value))
256                        .collect::<BTreeMap<_, _>>();
257                    if let Some(MoveValue::Address(address) | MoveValue::Signer(address)) =
258                        id_values.get("bytes").cloned()
259                    {
260                        curr_struct.object_id = Some(ObjectID::from_address(*address))
261                    }
262                }
263            } else if "0x1::option::Option" == struct_name {
264                // Option in sui move is implemented as vector of size 1
265                if let Some(MoveValue::Vector(vec_values)) = values.get("vec").cloned()
266                    && let Some(first_value) = vec_values.first()
267                {
268                    parse_struct_field(
269                        &format!("{}[0]", path),
270                        first_value.clone(),
271                        curr_struct,
272                        all_structs,
273                    );
274                }
275            } else if !WRAPPED_INDEXING_DISALLOW_LIST.contains(&&*struct_name) {
276                // Do not index most common struct types i.e. string, url, etc
277                parse_struct(path, move_struct, all_structs)
278            }
279        }
280        MoveValue::Variant(v) => {
281            for (k, field) in v.fields.iter() {
282                parse_struct_field(
283                    &format!("{}.{}", path, k),
284                    field.clone(),
285                    curr_struct,
286                    all_structs,
287                );
288            }
289        }
290        MoveValue::Vector(fields) => {
291            for (index, field) in fields.iter().enumerate() {
292                parse_struct_field(
293                    &format!("{}[{}]", path, &index),
294                    field.clone(),
295                    curr_struct,
296                    all_structs,
297                );
298            }
299        }
300        _ => {}
301    }
302}
303
304#[cfg(test)]
305mod tests {
306    use super::parse_struct;
307    use move_core_types::account_address::AccountAddress;
308    use move_core_types::annotated_value::MoveStruct;
309    use move_core_types::annotated_value::MoveValue;
310    use move_core_types::annotated_value::MoveVariant;
311    use move_core_types::identifier::Identifier;
312    use move_core_types::language_storage::StructTag;
313    use std::collections::BTreeMap;
314    use std::str::FromStr;
315    use sui_types::base_types::ObjectID;
316
317    #[tokio::test]
318    async fn test_wrapped_object_parsing() -> anyhow::Result<()> {
319        let uid_field = MoveValue::Struct(MoveStruct {
320            type_: StructTag::from_str("0x2::object::UID")?,
321            fields: vec![(
322                Identifier::from_str("id")?,
323                MoveValue::Struct(MoveStruct {
324                    type_: StructTag::from_str("0x2::object::ID")?,
325                    fields: vec![(
326                        Identifier::from_str("bytes")?,
327                        MoveValue::Signer(AccountAddress::from_hex_literal("0x300")?),
328                    )],
329                }),
330            )],
331        });
332        let balance_field = MoveValue::Struct(MoveStruct {
333            type_: StructTag::from_str("0x2::balance::Balance")?,
334            fields: vec![(Identifier::from_str("value")?, MoveValue::U32(10))],
335        });
336        let move_struct = MoveStruct {
337            type_: StructTag::from_str("0x2::test::Test")?,
338            fields: vec![
339                (Identifier::from_str("id")?, uid_field),
340                (Identifier::from_str("principal")?, balance_field),
341            ],
342        };
343        let mut all_structs = BTreeMap::new();
344        parse_struct("$", move_struct, &mut all_structs);
345        assert_eq!(
346            all_structs.get("$").unwrap().object_id,
347            Some(ObjectID::from_hex_literal("0x300")?)
348        );
349        assert_eq!(
350            all_structs.get("$.principal").unwrap().struct_tag,
351            Some(StructTag::from_str("0x2::balance::Balance")?)
352        );
353        Ok(())
354    }
355
356    #[tokio::test]
357    async fn test_wrapped_object_parsing_within_enum() -> anyhow::Result<()> {
358        let uid_field = MoveValue::Struct(MoveStruct {
359            type_: StructTag::from_str("0x2::object::UID")?,
360            fields: vec![(
361                Identifier::from_str("id")?,
362                MoveValue::Struct(MoveStruct {
363                    type_: StructTag::from_str("0x2::object::ID")?,
364                    fields: vec![(
365                        Identifier::from_str("bytes")?,
366                        MoveValue::Signer(AccountAddress::from_hex_literal("0x300")?),
367                    )],
368                }),
369            )],
370        });
371        let balance_field = MoveValue::Struct(MoveStruct {
372            type_: StructTag::from_str("0x2::balance::Balance")?,
373            fields: vec![(Identifier::from_str("value")?, MoveValue::U32(10))],
374        });
375        let move_enum = MoveVariant {
376            type_: StructTag::from_str("0x2::test::TestEnum")?,
377            variant_name: Identifier::from_str("TestVariant")?,
378            tag: 0,
379            fields: vec![
380                (Identifier::from_str("field0")?, MoveValue::U64(10)),
381                (Identifier::from_str("principal")?, balance_field),
382            ],
383        };
384        let move_struct = MoveStruct {
385            type_: StructTag::from_str("0x2::test::Test")?,
386            fields: vec![
387                (Identifier::from_str("id")?, uid_field),
388                (
389                    Identifier::from_str("enum_field")?,
390                    MoveValue::Variant(move_enum),
391                ),
392            ],
393        };
394        let mut all_structs = BTreeMap::new();
395        parse_struct("$", move_struct, &mut all_structs);
396        assert_eq!(
397            all_structs.get("$").unwrap().object_id,
398            Some(ObjectID::from_hex_literal("0x300")?)
399        );
400        assert_eq!(
401            all_structs
402                .get("$.enum_field.principal")
403                .unwrap()
404                .struct_tag,
405            Some(StructTag::from_str("0x2::balance::Balance")?)
406        );
407        Ok(())
408    }
409}
sui_analytics_indexer/handlers/tables/mod.rs

sui_analytics_indexer/handlers/tables/
mod.rs
