sui_move_build/
lib.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{
5    collections::{BTreeMap, BTreeSet, HashSet},
6    io::Write,
7    path::Path,
8};
9
10use fastcrypto::encoding::Base64;
11use serde_reflection::Registry;
12
13use move_binary_format::{
14    CompiledModule,
15    normalized::{self, Type},
16};
17use move_bytecode_utils::{Modules, layout::SerdeLayoutBuilder, module_cache::GetModule};
18use move_compiler::{
19    compiled_unit::AnnotatedCompiledModule,
20    diagnostics::{Diagnostics, report_diagnostics_to_buffer, report_warnings},
21    linters::LINT_WARNING_PREFIX,
22    shared::files::MappedFiles,
23};
24use move_core_types::{
25    account_address::AccountAddress,
26    language_storage::{ModuleId, StructTag},
27};
28use move_package_alt::{MoveFlavor, RootPackage, schema::Environment};
29use move_package_alt_compilation::compiled_package::CompiledPackage as MoveCompiledPackage;
30use move_package_alt_compilation::{
31    build_config::BuildConfig as MoveBuildConfig, build_plan::BuildPlan,
32};
33use move_symbol_pool::Symbol;
34
35use sui_package_alt::{SuiFlavor, testnet_environment};
36use sui_protocol_config::{Chain, ProtocolConfig, ProtocolVersion};
37use sui_types::{
38    BRIDGE_ADDRESS, DEEPBOOK_ADDRESS, MOVE_STDLIB_ADDRESS, SUI_FRAMEWORK_ADDRESS,
39    SUI_SYSTEM_ADDRESS, TypeTag,
40    base_types::ObjectID,
41    error::{SuiError, SuiErrorKind, SuiResult},
42    is_system_package,
43    move_package::{FnInfo, FnInfoKey, FnInfoMap, MovePackage},
44};
45use sui_verifier::verifier as sui_bytecode_verifier;
46
47#[cfg(test)]
48#[path = "unit_tests/build_tests.rs"]
49mod build_tests;
50
51pub mod test_utils {
52    use crate::{BuildConfig, CompiledPackage};
53    use std::path::PathBuf;
54
55    pub async fn compile_basics_package() -> CompiledPackage {
56        compile_example_package("../../examples/move/basics").await
57    }
58
59    pub async fn compile_managed_coin_package() -> CompiledPackage {
60        compile_example_package("../../crates/sui-core/src/unit_tests/data/managed_coin").await
61    }
62
63    pub async fn compile_example_package(relative_path: &str) -> CompiledPackage {
64        let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
65        path.push(relative_path);
66
67        BuildConfig::new_for_testing()
68            .build_async(&path)
69            .await
70            .unwrap()
71    }
72}
73
74/// Wrapper around the core Move `CompiledPackage` with some Sui-specific traits and info
75#[derive(Debug, Clone)]
76pub struct CompiledPackage {
77    pub package: MoveCompiledPackage,
78    /// Address the package is recorded as being published at.
79    pub published_at: Option<ObjectID>,
80    /// The dependency IDs of this package
81    pub dependency_ids: PackageDependencies,
82}
83
84/// Wrapper around the core Move `BuildConfig` with some Sui-specific info
85#[derive(Clone)]
86pub struct BuildConfig {
87    pub config: MoveBuildConfig,
88    /// If true, run the Move bytecode verifier on the bytecode from a successful build
89    pub run_bytecode_verifier: bool,
90    /// If true, print build diagnostics to stderr--no printing if false
91    pub print_diags_to_stderr: bool,
92    /// The environment that compilation is with respect to (e.g., required to resolve
93    /// published dependency IDs).
94    pub environment: Environment,
95}
96
97impl BuildConfig {
98    pub fn new_for_testing() -> Self {
99        let install_dir = mysten_common::tempdir().unwrap().keep();
100        let config = MoveBuildConfig {
101            default_flavor: Some(move_compiler::editions::Flavor::Sui),
102            install_dir: Some(install_dir),
103            silence_warnings: true,
104            lint_flag: move_package_alt_compilation::lint_flag::LintFlag::LEVEL_NONE,
105            ..MoveBuildConfig::default()
106        };
107        BuildConfig {
108            config,
109            run_bytecode_verifier: true,
110            print_diags_to_stderr: false,
111            environment: testnet_environment(),
112        }
113    }
114
115    pub fn new_for_testing_replace_addresses<I, S>(dep_original_addresses: I) -> Self
116    where
117        I: IntoIterator<Item = (S, ObjectID)>,
118        S: Into<String>,
119    {
120        let mut build_config = Self::new_for_testing();
121        for (addr_name, obj_id) in dep_original_addresses {
122            build_config
123                .config
124                .additional_named_addresses
125                .insert(addr_name.into(), AccountAddress::from(obj_id));
126        }
127        build_config
128    }
129
130    fn fn_info(units: &[AnnotatedCompiledModule]) -> FnInfoMap {
131        let mut fn_info_map = BTreeMap::new();
132        for u in units {
133            let mod_addr = u.named_module.address.into_inner();
134            let mod_is_test = u.attributes.is_test_or_test_only();
135            for (_, s, info) in &u.function_infos {
136                let fn_name = s.as_str().to_string();
137                let is_test = mod_is_test || info.attributes.is_test_or_test_only();
138                fn_info_map.insert(FnInfoKey { fn_name, mod_addr }, FnInfo { is_test });
139            }
140        }
141
142        fn_info_map
143    }
144
145    fn compile_package<W: Write + Send, F: MoveFlavor>(
146        &self,
147        root_pkg: &RootPackage<F>,
148        writer: &mut W,
149    ) -> anyhow::Result<(MoveCompiledPackage, FnInfoMap)> {
150        let mut config = self.config.clone();
151        // set the default flavor to Sui if not already set by the user
152        if config.default_flavor.is_none() {
153            config.default_flavor = Some(move_compiler::editions::Flavor::Sui);
154        }
155        let build_plan = BuildPlan::create(root_pkg, &config)?;
156        let mut fn_info = None;
157        let compiled_pkg = build_plan.compile_with_driver(writer, |compiler| {
158            let (files, units_res) = compiler.build()?;
159            match units_res {
160                Ok((units, warning_diags)) => {
161                    decorate_warnings(warning_diags, Some(&files));
162                    fn_info = Some(Self::fn_info(&units));
163                    Ok((files, units))
164                }
165                Err(error_diags) => {
166                    // with errors present don't even try decorating warnings output to avoid
167                    // clutter
168                    assert!(!error_diags.is_empty());
169                    let diags_buf =
170                        report_diagnostics_to_buffer(&files, error_diags, /* color */ true);
171                    if let Err(err) = std::io::stderr().write_all(&diags_buf) {
172                        anyhow::bail!("Cannot output compiler diagnostics: {}", err);
173                    }
174                    anyhow::bail!("Compilation error");
175                }
176            }
177        })?;
178        Ok((compiled_pkg, fn_info.unwrap()))
179    }
180
181    pub async fn build_async(self, path: &Path) -> anyhow::Result<CompiledPackage> {
182        let mut root_pkg = self
183            .config
184            .package_loader(path, &self.environment)
185            .load()
186            .await?;
187
188        self.internal_build(&mut root_pkg)
189    }
190
191    pub async fn build_async_from_root_pkg(
192        self,
193        root_pkg: &mut RootPackage<SuiFlavor>,
194    ) -> anyhow::Result<CompiledPackage> {
195        self.internal_build(root_pkg)
196    }
197
198    /// Given a `path` and a `build_config`, build the package in that path, including its dependencies.
199    /// If we are building the Sui framework, we skip the check that the addresses should be 0
200    pub fn build(self, path: &Path) -> anyhow::Result<CompiledPackage> {
201        // we need to block here to compile the package, which requires to fetch dependencies
202        let mut root_pkg = self
203            .config
204            .package_loader(path, &self.environment)
205            .load_sync()?;
206
207        self.internal_build(&mut root_pkg)
208    }
209
210    fn internal_build(
211        self,
212        root_pkg: &mut RootPackage<SuiFlavor>,
213    ) -> anyhow::Result<CompiledPackage> {
214        let result = if self.print_diags_to_stderr {
215            self.compile_package(root_pkg, &mut std::io::stderr())
216        } else {
217            self.compile_package(root_pkg, &mut std::io::sink())
218        };
219
220        let (package, fn_info) = result.map_err(|error| {
221            SuiError::from(SuiErrorKind::ModuleBuildFailure {
222                // Use [Debug] formatting to capture [anyhow] error context
223                error: format!("{:?}", error),
224            })
225        })?;
226
227        if self.run_bytecode_verifier {
228            verify_bytecode(&package, &fn_info)?;
229        }
230
231        let dependency_ids = PackageDependencies::new(root_pkg)?;
232        let published_at = root_pkg
233            .publication()
234            .map(|p| ObjectID::from_address(p.addresses.published_at.0));
235
236        root_pkg.save_lockfile_to_disk()?;
237
238        Ok(CompiledPackage {
239            package,
240            dependency_ids,
241            published_at,
242        })
243    }
244}
245
246/// There may be additional information that needs to be displayed after diagnostics are reported
247/// (optionally report diagnostics themselves if files argument is provided).
248pub fn decorate_warnings(warning_diags: Diagnostics, files: Option<&MappedFiles>) {
249    let any_linter_warnings = warning_diags.any_with_prefix(LINT_WARNING_PREFIX);
250    let (filtered_diags_num, unique) =
251        warning_diags.filtered_source_diags_with_prefix(LINT_WARNING_PREFIX);
252    if let Some(f) = files {
253        report_warnings(f, warning_diags);
254    }
255    if any_linter_warnings {
256        eprintln!("Please report feedback on the linter warnings at https://forums.sui.io\n");
257    }
258    if filtered_diags_num > 0 {
259        eprintln!(
260            "Total number of linter warnings suppressed: {filtered_diags_num} (unique lints: {unique})"
261        );
262    }
263}
264
265/// Check that the compiled modules in `package` are valid
266fn verify_bytecode(package: &MoveCompiledPackage, fn_info: &FnInfoMap) -> SuiResult<()> {
267    let compiled_modules = package.root_modules_map();
268    let verifier_config = ProtocolConfig::get_for_version(ProtocolVersion::MAX, Chain::Unknown)
269        .verifier_config(/* signing_limits */ None);
270
271    for m in compiled_modules.iter_modules() {
272        move_bytecode_verifier::verify_module_unmetered(m).map_err(|err| {
273            SuiErrorKind::ModuleVerificationFailure {
274                error: err.to_string(),
275            }
276        })?;
277        sui_bytecode_verifier::sui_verify_module_unmetered(m, fn_info, &verifier_config)?;
278    }
279    // TODO(https://github.com/MystenLabs/sui/issues/69): Run Move linker
280
281    Ok(())
282}
283
284impl CompiledPackage {
285    /// Return all of the bytecode modules in this package (not including direct or transitive deps)
286    /// Note: these are not topologically sorted by dependency--use `get_dependency_sorted_modules` to produce a list of modules suitable
287    /// for publishing or static analysis
288    pub fn get_modules(&self) -> impl Iterator<Item = &CompiledModule> {
289        self.package.root_modules().map(|m| &m.unit.module)
290    }
291
292    /// Return all of the bytecode modules in this package (not including direct or transitive deps)
293    /// Note: these are not topologically sorted by dependency--use `get_dependency_sorted_modules` to produce a list of modules suitable
294    /// for publishing or static analysis
295    pub fn into_modules(self) -> Vec<CompiledModule> {
296        self.package
297            .root_compiled_units
298            .into_iter()
299            .map(|m| m.unit.module)
300            .collect()
301    }
302
303    /// Return all of the bytecode modules that this package depends on (both directly and transitively)
304    /// Note: these are not topologically sorted by dependency.
305    pub fn get_dependent_modules(&self) -> impl Iterator<Item = &CompiledModule> {
306        self.package
307            .deps_compiled_units
308            .iter()
309            .map(|(_, m)| &m.unit.module)
310    }
311
312    /// Return all of the bytecode modules in this package and the modules of its direct and transitive dependencies.
313    /// Note: these are not topologically sorted by dependency.
314    pub fn get_modules_and_deps(&self) -> impl Iterator<Item = &CompiledModule> {
315        self.package
316            .all_compiled_units_with_source()
317            .map(|m| &m.unit.module)
318    }
319
320    /// Return the bytecode modules in this package, topologically sorted in dependency order.
321    /// Optionally include dependencies that have not been published (are at address 0x0), if
322    /// `with_unpublished_deps` is true. This is the function to call if you would like to publish
323    /// or statically analyze the modules.
324    pub fn get_dependency_sorted_modules(
325        &self,
326        with_unpublished_deps: bool,
327    ) -> Vec<CompiledModule> {
328        let all_modules = Modules::new(self.get_modules_and_deps());
329
330        // SAFETY: package built successfully
331        let modules = all_modules.compute_topological_order().unwrap();
332
333        if with_unpublished_deps {
334            // For each transitive dependent module, if they are not to be published, they must have
335            // a non-zero address (meaning they are already published on-chain).
336            modules
337                .filter(|module| module.address() == &AccountAddress::ZERO)
338                .cloned()
339                .collect()
340        } else {
341            // Collect all module IDs from the current package to be published (module names are not
342            // sufficient as we may have modules with the same names in user code and in Sui
343            // framework which would result in the latter being pulled into a set of modules to be
344            // published).
345            let self_modules: HashSet<_> = self
346                .package
347                .root_modules_map()
348                .iter_modules()
349                .iter()
350                .map(|m| m.self_id())
351                .collect();
352
353            modules
354                .filter(|module| self_modules.contains(&module.self_id()))
355                .cloned()
356                .collect()
357        }
358    }
359
360    /// Return the set of Object IDs corresponding to this package's transitive dependencies'
361    /// storage package IDs (where to load those packages on-chain).
362    pub fn get_dependency_storage_package_ids(&self) -> Vec<ObjectID> {
363        self.dependency_ids.published.values().cloned().collect()
364    }
365
366    /// Return a digest of the bytecode modules in this package.
367    pub fn get_package_digest(&self, with_unpublished_deps: bool) -> [u8; 32] {
368        let hash_modules = true;
369        MovePackage::compute_digest_for_modules_and_deps(
370            &self.get_package_bytes(with_unpublished_deps),
371            &self.get_dependency_storage_package_ids(),
372            hash_modules,
373        )
374    }
375
376    /// Return a serialized representation of the bytecode modules in this package, topologically sorted in dependency order
377    pub fn get_package_bytes(&self, with_unpublished_deps: bool) -> Vec<Vec<u8>> {
378        self.get_dependency_sorted_modules(with_unpublished_deps)
379            .iter()
380            .map(|m| {
381                let mut bytes = Vec::new();
382                m.serialize_with_version(m.version, &mut bytes).unwrap(); // safe because package built successfully
383                bytes
384            })
385            .collect()
386    }
387
388    /// Return the base64-encoded representation of the bytecode modules in this package, topologically sorted in dependency order
389    pub fn get_package_base64(&self, with_unpublished_deps: bool) -> Vec<Base64> {
390        self.get_package_bytes(with_unpublished_deps)
391            .iter()
392            .map(|b| Base64::from_bytes(b))
393            .collect()
394    }
395
396    /// Get bytecode modules from DeepBook that are used by this package
397    pub fn get_deepbook_modules(&self) -> impl Iterator<Item = &CompiledModule> {
398        self.get_modules_and_deps()
399            .filter(|m| *m.self_id().address() == DEEPBOOK_ADDRESS)
400    }
401
402    /// Get bytecode modules from DeepBook that are used by this package
403    pub fn get_bridge_modules(&self) -> impl Iterator<Item = &CompiledModule> {
404        self.get_modules_and_deps()
405            .filter(|m| *m.self_id().address() == BRIDGE_ADDRESS)
406    }
407
408    /// Get bytecode modules from the Sui System that are used by this package
409    pub fn get_sui_system_modules(&self) -> impl Iterator<Item = &CompiledModule> {
410        self.get_modules_and_deps()
411            .filter(|m| *m.self_id().address() == SUI_SYSTEM_ADDRESS)
412    }
413
414    /// Get bytecode modules from the Sui Framework that are used by this package
415    pub fn get_sui_framework_modules(&self) -> impl Iterator<Item = &CompiledModule> {
416        self.get_modules_and_deps()
417            .filter(|m| *m.self_id().address() == SUI_FRAMEWORK_ADDRESS)
418    }
419
420    /// Get bytecode modules from the Move stdlib that are used by this package
421    pub fn get_stdlib_modules(&self) -> impl Iterator<Item = &CompiledModule> {
422        self.get_modules_and_deps()
423            .filter(|m| *m.self_id().address() == MOVE_STDLIB_ADDRESS)
424    }
425
426    /// Generate layout schemas for all types declared by this package, as well as
427    /// all struct types passed into `entry` functions declared by modules in this package
428    /// (either directly or by reference).
429    /// These layout schemas can be consumed by clients (e.g., the TypeScript SDK) to enable
430    /// BCS serialization/deserialization of the package's objects, tx arguments, and events.
431    pub fn generate_struct_layouts(&self) -> Registry {
432        let pool = &mut normalized::RcPool::new();
433        let mut package_types = BTreeSet::new();
434        for m in self.get_modules() {
435            let normalized_m = normalized::Module::new(pool, m, /* include code */ false);
436            // 1. generate struct layouts for all declared types
437            'structs: for (name, s) in normalized_m.structs {
438                let mut dummy_type_parameters = Vec::new();
439                for t in &s.type_parameters {
440                    if t.is_phantom {
441                        // if all of t's type parameters are phantom, we can generate a type layout
442                        // we make this happen by creating a StructTag with dummy `type_params`, since the layout generator won't look at them.
443                        // we need to do this because SerdeLayoutBuilder will refuse to generate a layout for any open StructTag, but phantom types
444                        // cannot affect the layout of a struct, so we just use dummy values
445                        dummy_type_parameters.push(TypeTag::Signer)
446                    } else {
447                        // open type--do not attempt to generate a layout
448                        // TODO: handle generating layouts for open types?
449                        continue 'structs;
450                    }
451                }
452                debug_assert!(dummy_type_parameters.len() == s.type_parameters.len());
453                package_types.insert(StructTag {
454                    address: *m.address(),
455                    module: m.name().to_owned(),
456                    name: name.as_ident_str().to_owned(),
457                    type_params: dummy_type_parameters,
458                });
459            }
460            // 2. generate struct layouts for all parameters of `entry` funs
461            for (_name, f) in normalized_m.functions {
462                if f.is_entry {
463                    for t in &*f.parameters {
464                        let tag_opt = match &**t {
465                            Type::Address
466                            | Type::Bool
467                            | Type::Signer
468                            | Type::TypeParameter(_)
469                            | Type::U8
470                            | Type::U16
471                            | Type::U32
472                            | Type::U64
473                            | Type::U128
474                            | Type::U256
475                            | Type::Vector(_) => continue,
476                            Type::Reference(_, inner) => inner.to_struct_tag(pool),
477                            Type::Datatype(_) => t.to_struct_tag(pool),
478                        };
479                        if let Some(tag) = tag_opt {
480                            package_types.insert(tag);
481                        }
482                    }
483                }
484            }
485        }
486        let mut layout_builder = SerdeLayoutBuilder::new(self);
487        for typ in &package_types {
488            layout_builder.build_data_layout(typ).unwrap();
489        }
490        layout_builder.into_registry()
491    }
492
493    /// Checks whether this package corresponds to a built-in framework
494    pub fn is_system_package(&self) -> bool {
495        // System packages always have "published-at" addresses
496        let Some(published_at) = self.published_at else {
497            return false;
498        };
499
500        is_system_package(published_at)
501    }
502
503    /// Checks for root modules with non-zero package addresses.  Returns an arbitrary one, if one
504    /// can be found, otherwise returns `None`.
505    pub fn published_root_module(&self) -> Option<&CompiledModule> {
506        self.package.root_compiled_units.iter().find_map(|unit| {
507            if unit.unit.module.self_id().address() != &AccountAddress::ZERO {
508                Some(&unit.unit.module)
509            } else {
510                None
511            }
512        })
513    }
514
515    pub fn verify_unpublished_dependencies(
516        &self,
517        unpublished_deps: &BTreeSet<Symbol>,
518    ) -> SuiResult<()> {
519        if unpublished_deps.is_empty() {
520            return Ok(());
521        }
522
523        let errors = self
524            .package
525            .deps_compiled_units
526            .iter()
527            .filter_map(|(p, m)| {
528                if !unpublished_deps.contains(p) || m.unit.module.address() == &AccountAddress::ZERO
529                {
530                    return None;
531                }
532                Some(format!(
533                    " - {}::{} in dependency {}",
534                    m.unit.module.address(),
535                    m.unit.name,
536                    p
537                ))
538            })
539            .collect::<Vec<String>>();
540
541        if errors.is_empty() {
542            return Ok(());
543        }
544
545        let mut error_message = vec![];
546        error_message.push(
547            "The following modules in package dependencies set a non-zero self-address:".into(),
548        );
549        error_message.extend(errors);
550        error_message.push(
551            "If these packages really are unpublished, their self-addresses should not be \
552            explicitly set when publishing. If they are already published, ensure they specify the \
553            address in the `published-at` of their Published.toml file."
554                .into(),
555        );
556
557        Err(SuiErrorKind::ModulePublishFailure {
558            error: error_message.join("\n"),
559        }
560        .into())
561    }
562
563    pub fn get_published_dependencies_ids(&self) -> Vec<ObjectID> {
564        self.dependency_ids.published.values().cloned().collect()
565    }
566}
567
568impl GetModule for CompiledPackage {
569    type Error = anyhow::Error;
570    // TODO: return ref here for better efficiency? Borrow checker + all_modules_map() make it hard to do this
571    type Item = CompiledModule;
572
573    fn get_module_by_id(&self, id: &ModuleId) -> Result<Option<Self::Item>, Self::Error> {
574        Ok(self.package.all_modules_map().get_module(id).ok().cloned())
575    }
576}
577
578#[derive(thiserror::Error, Debug, Clone)]
579pub enum PublishedAtError {
580    #[error("The 'published-at' field in Move.toml or Move.lock is invalid: {0:?}")]
581    Invalid(String),
582    #[error("The 'published-at' field is not present in Move.toml or Move.lock")]
583    NotPresent,
584}
585
586#[derive(Debug, Clone)]
587pub struct PackageDependencies {
588    /// Set of published dependencies (name and address).
589    pub published: BTreeMap<Symbol, ObjectID>,
590    /// Set of unpublished dependencies (name and address).
591    pub unpublished: BTreeSet<Symbol>,
592    /// Set of dependencies with invalid `published-at` addresses.
593    pub invalid: BTreeMap<Symbol, String>,
594    /// Set of dependencies that have conflicting `published-at` addresses. The key refers to
595    /// the package, and the tuple refers to the address in the (Move.lock, Move.toml) respectively.
596    pub conflicting: BTreeMap<Symbol, (ObjectID, ObjectID)>,
597}
598
599impl PackageDependencies {
600    pub fn new<F: MoveFlavor>(root_pkg: &RootPackage<F>) -> anyhow::Result<Self> {
601        let mut published = BTreeMap::new();
602        let mut unpublished = BTreeSet::new();
603
604        let packages = root_pkg.packages();
605
606        for p in packages {
607            if p.is_root() {
608                continue;
609            }
610            if let Some(addresses) = p.published() {
611                published.insert(
612                    p.display_name().into(),
613                    ObjectID::from_address(addresses.published_at.0),
614                );
615            } else {
616                unpublished.insert(p.display_name().into());
617            }
618        }
619
620        Ok(Self {
621            published,
622            unpublished,
623            invalid: BTreeMap::new(),
624            conflicting: BTreeMap::new(),
625        })
626    }
627}
sui_move_build/lib.rs

sui_move_build/
lib.rs
