sui_source_validation/
toolchain.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{
5    collections::HashMap,
6    ffi::OsStr,
7    fs::File,
8    io::{self, Read},
9    path::{Path, PathBuf},
10    process::Command,
11    str::FromStr,
12};
13
14use anyhow::{Context, Result, anyhow, bail, ensure};
15use colored::Colorize;
16use serde::{Deserialize, Serialize};
17use tar::Archive;
18use tempfile::TempDir;
19use tracing::{debug, info};
20
21use sui_package_alt::SuiFlavor;
22
23use move_binary_format::CompiledModule;
24use move_bytecode_source_map::utils::source_map_from_file;
25use move_command_line_common::{
26    env::MOVE_HOME,
27    files::{
28        DEBUG_INFO_EXTENSION, MOVE_COMPILED_EXTENSION, MOVE_EXTENSION, extension_equals,
29        find_filenames,
30    },
31};
32use move_compiler::{
33    compiled_unit::NamedCompiledModule,
34    editions::{Edition, Flavor},
35    shared::{NumericalAddress, files::FileName},
36};
37use move_package_alt::{
38    package::layout::SourcePackageLayout,
39    schema::{Environment, ParsedPublishedFile},
40};
41use move_package_alt_compilation::compiled_package::CompiledUnitWithSource;
42use move_package_alt_compilation::layout::CompiledPackageLayout;
43use move_symbol_pool::Symbol;
44
45pub(crate) const CURRENT_COMPILER_VERSION: &str = env!("CARGO_PKG_VERSION");
46const LEGACY_COMPILER_VERSION: &str = CURRENT_COMPILER_VERSION; // TODO: update this when Move 2024 is released
47const PRE_TOOLCHAIN_MOVE_LOCK_VERSION: u16 = 0; // Used to detect lockfiles pre-toolchain versioning support
48const CANONICAL_UNIX_BINARY_NAME: &str = "sui";
49const CANONICAL_WIN_BINARY_NAME: &str = "sui.exe";
50
51/// Lock file version written by this version of the compiler.  Backwards compatibility is
52/// guaranteed (the compiler can read lock files with older versions), forward compatibility is not
53/// (the compiler will fail to read lock files at newer versions).
54///
55/// V0: Base version.
56/// V1: Adds toolchain versioning support.
57/// V2: Adds support for managing addresses on package publish and upgrades.
58/// V3: Renames dependency `name` field to `id` and adds a `name` field to store the name from the manifest.
59pub const VERSION: u16 = 3;
60
61// TODO: pkg-alt, maybe we want to reuse the code in move-package-alt to read lockfiles
62#[derive(Serialize, Deserialize)]
63pub struct LockfileHeader {
64    pub version: u16,
65}
66
67// TODO: pkg-alt this needs to work with both old style and new style formats. Particularly, for
68// the new pkg system, the toolchain version is in the Published.toml file, or Pub.env.toml file.
69#[derive(Serialize, Deserialize, Debug)]
70pub struct ToolchainVersion {
71    /// The Move compiler version used to compile this package.
72    #[serde(rename = "compiler-version")]
73    pub compiler_version: String,
74    /// The Move compiler configuration used to compile this package.
75    pub edition: Edition,
76    pub flavor: Flavor,
77}
78
79#[derive(Serialize, Deserialize)]
80struct Schema<T> {
81    #[serde(rename = "move")]
82    move_: T,
83}
84
85// TODO: pkg-alt, maybe we want to reuse the code in move-package-alt to read lockfiles
86impl LockfileHeader {
87    /// Read lock file header after verifying that the version of the lock is not newer than the version
88    /// supported by this library.
89    pub fn read(lock: &mut impl Read) -> Result<Self> {
90        let contents = {
91            let mut buf = String::new();
92            lock.read_to_string(&mut buf).context("Reading lock file")?;
93            buf
94        };
95        Self::from_str(&contents)
96    }
97
98    fn from_str(contents: &str) -> Result<Self> {
99        let Schema { move_: header } =
100            toml::de::from_str::<Schema<Self>>(contents).context("Deserializing lock header")?;
101
102        if header.version != VERSION {
103            bail!(
104                "Lock file format mismatch, expected version {}, found {}",
105                VERSION,
106                header.version
107            );
108        }
109
110        Ok(header)
111    }
112}
113
114impl ToolchainVersion {
115    /// Read toolchain version info from the root project directory. Tries to read Published.toml
116    /// first (new pkg-alt format), then falls back to Move.lock (old format). Returns None if
117    /// neither file exists or if no toolchain info is found.
118    pub fn read(root_path: &Path, env: &Environment) -> anyhow::Result<Option<ToolchainVersion>> {
119        let published_path = root_path.join("Published.toml");
120        let lock_path = root_path.join(SourcePackageLayout::Lock.path());
121
122        if published_path.exists() {
123            let contents =
124                std::fs::read_to_string(&published_path).context("Reading Published.toml file")?;
125            let parsed: ParsedPublishedFile<SuiFlavor> =
126                toml::de::from_str(&contents).context("Deserializing Published.toml")?;
127
128            if let Some((_, publication)) = parsed.published.into_iter().next()
129                && let (Some(compiler_version), Some(build_config)) = (
130                    publication.metadata.toolchain_version,
131                    publication.metadata.build_config,
132                )
133            {
134                // Check that the publication info matches the current environment before returning
135                // the toolchain info. If they don't match, then we don't know which toolchain was
136                // used.
137                if env.id() == &publication.chain_id {
138                    debug!("Found toolchain version in Published.toml file");
139                    return Ok(Some(ToolchainVersion {
140                        compiler_version,
141                        edition: Edition::from_str(&build_config.edition)?,
142                        flavor: Flavor::from_str(&build_config.flavor)?,
143                    }));
144                }
145            }
146
147            debug!("Did not find toolchain version in Published.toml file");
148
149            return Ok(None);
150        }
151
152        if lock_path.exists() {
153            debug!("Found Move.lock file, reading toolchain version from it");
154            let contents = std::fs::read_to_string(&lock_path).context("Reading Move.lock file")?;
155            let _ = LockfileHeader::from_str(&contents)?;
156
157            #[derive(serde::Deserialize)]
158            struct TV {
159                #[serde(rename = "toolchain-version")]
160                toolchain_version: Option<ToolchainVersion>,
161            }
162
163            let Schema { move_: value } = toml::de::from_str::<Schema<TV>>(&contents)
164                .context("Deserializing toolchain version from Move.lock")?;
165
166            debug!(
167                "Toolchain version read from Move.lock file {:?}",
168                value.toolchain_version
169            );
170            return Ok(value.toolchain_version);
171        }
172
173        debug!("Did not find Move.lock nor Published.toml file");
174
175        Ok(None)
176    }
177}
178
179pub(crate) fn current_toolchain() -> ToolchainVersion {
180    ToolchainVersion {
181        compiler_version: CURRENT_COMPILER_VERSION.into(),
182        edition: Edition::LEGACY, /* does not matter, unused for current_toolchain */
183        flavor: Flavor::Sui,      /* does not matter, unused for current_toolchain */
184    }
185}
186
187pub(crate) fn legacy_toolchain() -> ToolchainVersion {
188    ToolchainVersion {
189        compiler_version: LEGACY_COMPILER_VERSION.into(),
190        edition: Edition::LEGACY,
191        flavor: Flavor::Sui,
192    }
193}
194
195// /// Ensures `compiled_units` are compiled with the right compiler version, based on
196// /// Move.lock contents. This works by detecting if a compiled unit requires a prior compiler version:
197// /// - If so, download the compiler, recompile the unit, and return that unit in the result.
198// /// - If not, simply keep the current compiled unit.
199pub(crate) fn units_for_toolchain(
200    compiled_units: &Vec<(Symbol, CompiledUnitWithSource)>,
201    env: &Environment,
202) -> anyhow::Result<Vec<(Symbol, CompiledUnitWithSource)>> {
203    if std::env::var("SUI_RUN_TOOLCHAIN_BUILD").is_err() {
204        return Ok(compiled_units.clone());
205    }
206    let mut package_version_map: HashMap<Symbol, (ToolchainVersion, Vec<CompiledUnitWithSource>)> =
207        HashMap::new();
208    // First iterate over packages, mapping the required version for each package in `package_version_map`.
209    for (package, local_unit) in compiled_units {
210        if let Some((_, units)) = package_version_map.get_mut(package) {
211            // We've processed this package's required version.
212            units.push(local_unit.clone());
213            continue;
214        }
215
216        if sui_types::is_system_package(local_unit.unit.address.into_inner()) {
217            // System packages are always compiled with the current compiler.
218            package_version_map.insert(
219                Symbol::from(package.as_str()),
220                (current_toolchain(), vec![local_unit.clone()]),
221            );
222            continue;
223        }
224
225        let package_root = SourcePackageLayout::try_find_root(&local_unit.source_path)?;
226        let lock_file = package_root.join(SourcePackageLayout::Lock.path());
227        if !lock_file.exists() {
228            // No lock file implies current compiler for this package.
229            package_version_map.insert(*package, (current_toolchain(), vec![local_unit.clone()]));
230            continue;
231        }
232
233        let mut lock_file = File::open(lock_file)?;
234        let lock_version = LockfileHeader::read(&mut lock_file)?.version;
235        if lock_version == PRE_TOOLCHAIN_MOVE_LOCK_VERSION {
236            // No need to attempt reading lock file toolchain
237            debug!("{package} on legacy compiler",);
238            package_version_map.insert(*package, (legacy_toolchain(), vec![local_unit.clone()]));
239            continue;
240        }
241
242        let toolchain_version = ToolchainVersion::read(&package_root, env)?;
243        match toolchain_version {
244            // No ToolchainVersion and new Move.lock version implies current compiler.
245            None => {
246                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
247                package_version_map.insert(
248                    Symbol::from(package.as_str()),
249                    (current_toolchain(), vec![local_unit.clone()]),
250                );
251            }
252            // This dependency uses the current compiler.
253            Some(ToolchainVersion {
254                compiler_version, ..
255            }) if compiler_version == CURRENT_COMPILER_VERSION => {
256                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
257                package_version_map.insert(
258                    Symbol::from(package.as_str()),
259                    (current_toolchain(), vec![local_unit.clone()]),
260                );
261            }
262            // This dependency needs a prior compiler. Mark it and compile.
263            Some(toolchain_version) => {
264                println!(
265                    "{} {package} compiler @ {}",
266                    "REQUIRE".bold().green(),
267                    toolchain_version.compiler_version.yellow(),
268                );
269                package_version_map.insert(
270                    Symbol::from(package.as_str()),
271                    (toolchain_version, vec![local_unit.clone()]),
272                );
273            }
274        }
275    }
276
277    let mut units = vec![];
278    // Iterate over compiled units, and check if they need to be recompiled and replaced by a prior compiler's output.
279    for (package, (toolchain_version, local_units)) in package_version_map {
280        if toolchain_version.compiler_version == CURRENT_COMPILER_VERSION {
281            let local_units: Vec<_> = local_units.iter().map(|u| (package, u.clone())).collect();
282            units.extend(local_units);
283            continue;
284        }
285
286        if local_units.is_empty() {
287            bail!("Expected one or more modules, but none found");
288        }
289        let package_root = SourcePackageLayout::try_find_root(&local_units[0].source_path)?;
290        let install_dir = tempfile::tempdir()?; // place compiled packages in this temp dir, don't pollute this packages build dir
291        download_and_compile(
292            package_root.clone(),
293            &install_dir,
294            &toolchain_version,
295            &package,
296        )?;
297
298        let compiled_unit_paths = vec![package_root.clone()];
299        let compiled_units = find_filenames(&compiled_unit_paths, |path| {
300            extension_equals(path, MOVE_COMPILED_EXTENSION)
301        })?;
302        let build_path = install_dir
303            .path()
304            .join(CompiledPackageLayout::path(&CompiledPackageLayout::Root))
305            .join(package.as_str());
306        debug!("build path is {}", build_path.display());
307
308        // Add all units compiled with the previous compiler.
309        for bytecode_path in compiled_units {
310            info!("bytecode path {bytecode_path}, {package}");
311            let local_unit = decode_bytecode_file(build_path.clone(), &package, &bytecode_path)?;
312            units.push((package, local_unit))
313        }
314    }
315    Ok(units)
316}
317
318fn download_and_compile(
319    root: PathBuf,
320    install_dir: &TempDir,
321    ToolchainVersion {
322        compiler_version,
323        edition,
324        flavor,
325    }: &ToolchainVersion,
326    dep_name: &Symbol,
327) -> anyhow::Result<()> {
328    let dest_dir = PathBuf::from_iter([&*MOVE_HOME, "binaries"]); // E.g., ~/.move/binaries
329    let dest_version = dest_dir.join(compiler_version);
330    let mut dest_canonical_path = dest_version.clone();
331    dest_canonical_path.extend(["target", "release"]);
332    let mut dest_canonical_binary = dest_canonical_path.clone();
333
334    let platform = detect_platform(&root, compiler_version, &dest_canonical_path)?;
335    if platform == "windows-x86_64" {
336        dest_canonical_binary.push(CANONICAL_WIN_BINARY_NAME);
337    } else {
338        dest_canonical_binary.push(CANONICAL_UNIX_BINARY_NAME);
339    }
340
341    if !dest_canonical_binary.exists() {
342        // Check the platform and proceed if we can download a binary. If not, the user should follow error instructions to sideload the binary.
343        // Download if binary does not exist.
344        let mainnet_url = format!(
345            "https://github.com/MystenLabs/sui/releases/download/mainnet-v{compiler_version}/sui-mainnet-v{compiler_version}-{platform}.tgz",
346        );
347
348        println!(
349            "{} mainnet compiler @ {} (this may take a while)",
350            "DOWNLOADING".bold().green(),
351            compiler_version.yellow()
352        );
353
354        let mut response = match ureq::get(&mainnet_url).call() {
355            Ok(response) => response,
356            Err(ureq::Error::Status(404, _)) => {
357                println!(
358                    "{} sui mainnet compiler {} not available, attempting to download testnet compiler release...",
359                    "WARNING".bold().yellow(),
360                    compiler_version.yellow()
361                );
362                println!(
363                    "{} testnet compiler @ {} (this may take a while)",
364                    "DOWNLOADING".bold().green(),
365                    compiler_version.yellow()
366                );
367                let testnet_url = format!("https://github.com/MystenLabs/sui/releases/download/testnet-v{compiler_version}/sui-testnet-v{compiler_version}-{platform}.tgz");
368                ureq::get(&testnet_url).call()?
369            }
370            Err(e) => return Err(e.into()),
371        }.into_reader();
372
373        let dest_tarball = dest_version.join(format!("{}.tgz", compiler_version));
374        debug!("tarball destination: {} ", dest_tarball.display());
375        if let Some(parent) = dest_tarball.parent() {
376            std::fs::create_dir_all(parent)
377                .map_err(|e| anyhow!("failed to create directory for tarball: {e}"))?;
378        }
379        let mut dest_file = File::create(&dest_tarball)?;
380        io::copy(&mut response, &mut dest_file)?;
381
382        // Extract the tarball using the tar crate
383        let tar_gz = File::open(&dest_tarball)?;
384        let tar = flate2::read::GzDecoder::new(tar_gz);
385        let mut archive = Archive::new(tar);
386        archive
387            .unpack(&dest_version)
388            .map_err(|e| anyhow!("failed to untar compiler binary: {e}"))?;
389
390        let mut dest_binary = dest_version.clone();
391        dest_binary.extend(["target", "release"]);
392        if platform == "windows-x86_64" {
393            dest_binary.push(format!("sui-{platform}.exe"));
394        } else {
395            dest_binary.push(format!("sui-{platform}"));
396        }
397        let dest_binary_os = OsStr::new(dest_binary.as_path());
398        set_executable_permission(dest_binary_os)?;
399        std::fs::rename(dest_binary_os, dest_canonical_binary.clone())?;
400    }
401
402    debug!(
403        "{} move build --default-move-edition {} --default-move-flavor {} -p {} --install-dir {}",
404        dest_canonical_binary.display(),
405        edition.to_string().as_str(),
406        flavor.to_string().as_str(),
407        root.display(),
408        install_dir.path().display(),
409    );
410    info!(
411        "{} {} (compiler @ {})",
412        "BUILDING".bold().green(),
413        dep_name.as_str(),
414        compiler_version.yellow()
415    );
416    Command::new(dest_canonical_binary)
417        .args([
418            OsStr::new("move"),
419            OsStr::new("build"),
420            OsStr::new("--default-move-edition"),
421            OsStr::new(edition.to_string().as_str()),
422            OsStr::new("--default-move-flavor"),
423            OsStr::new(flavor.to_string().as_str()),
424            OsStr::new("-p"),
425            OsStr::new(root.as_path()),
426            OsStr::new("--install-dir"),
427            OsStr::new(install_dir.path()),
428        ])
429        .output()
430        .map_err(|e| {
431            anyhow!("failed to build package from compiler binary {compiler_version}: {e}",)
432        })?;
433    Ok(())
434}
435
436fn detect_platform(
437    package_path: &Path,
438    compiler_version: &String,
439    dest_dir: &Path,
440) -> anyhow::Result<String> {
441    let s = match (std::env::consts::OS, std::env::consts::ARCH) {
442        ("macos", "aarch64") => "macos-arm64",
443        ("macos", "x86_64") => "macos-x86_64",
444        ("linux", "x86_64") => "ubuntu-x86_64",
445        ("windows", "x86_64") => "windows-x86_64",
446        (os, arch) => {
447            let mut binary_name = CANONICAL_UNIX_BINARY_NAME;
448            if os == "windows" {
449                binary_name = CANONICAL_WIN_BINARY_NAME;
450            };
451            bail!(
452                "The package {} needs to be built with sui compiler version {compiler_version} but there \
453                 is no binary release available to download for your platform:\n\
454                 Operating System: {os}\n\
455                 Architecture: {arch}\n\
456                 You can manually put a {binary_name} binary for your platform in {} and rerun your command to continue.",
457                package_path.display(),
458                dest_dir.display(),
459            )
460        }
461    };
462    Ok(s.into())
463}
464
465#[cfg(unix)]
466fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
467    use std::fs;
468    use std::os::unix::prelude::PermissionsExt;
469    let mut perms = fs::metadata(path)?.permissions();
470    perms.set_mode(0o755);
471    fs::set_permissions(path, perms)?;
472    Ok(())
473}
474
475#[cfg(not(unix))]
476fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
477    Command::new("icacls")
478        .args([path, OsStr::new("/grant"), OsStr::new("Everyone:(RX)")])
479        .status()?;
480    Ok(())
481}
482
483fn decode_bytecode_file(
484    root_path: PathBuf,
485    package_name: &Symbol,
486    bytecode_path_str: &str,
487) -> anyhow::Result<CompiledUnitWithSource> {
488    let package_name_opt = Some(*package_name);
489    let bytecode_path = Path::new(bytecode_path_str);
490    let path_to_file = CompiledPackageLayout::path_to_file_after_category(bytecode_path);
491    let bytecode_bytes = std::fs::read(bytecode_path)?;
492    let source_map = source_map_from_file(
493        &root_path
494            .join(CompiledPackageLayout::DebugInfo.path())
495            .join(&path_to_file)
496            .with_extension(DEBUG_INFO_EXTENSION),
497    )?;
498    let source_path = &root_path
499        .join(CompiledPackageLayout::Sources.path())
500        .join(path_to_file)
501        .with_extension(MOVE_EXTENSION);
502    ensure!(
503        source_path.is_file(),
504        "Error decoding package: Unable to find corresponding source file for '{bytecode_path_str}' in package {package_name}"
505    );
506    let module = CompiledModule::deserialize_with_defaults(&bytecode_bytes)?;
507    let (address_bytes, module_name) = {
508        let id = module.self_id();
509        let parsed_addr = NumericalAddress::new(
510            id.address().into_bytes(),
511            move_compiler::shared::NumberFormat::Hex,
512        );
513        let module_name = FileName::from(id.name().as_str());
514        (parsed_addr, module_name)
515    };
516    let unit = NamedCompiledModule {
517        package_name: package_name_opt,
518        address: address_bytes,
519        name: module_name,
520        module,
521        source_map,
522        address_name: None,
523    };
524    Ok(CompiledUnitWithSource {
525        unit,
526        source_path: source_path.clone(),
527    })
528}
sui_source_validation/toolchain.rs

sui_source_validation/
toolchain.rs
