sui_source_validation/
toolchain.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{
5    collections::HashMap,
6    ffi::OsStr,
7    fs::File,
8    io::{self, Seek},
9    path::{Path, PathBuf},
10    process::Command,
11};
12
13use anyhow::{anyhow, bail, ensure};
14use colored::Colorize;
15use move_binary_format::CompiledModule;
16use move_bytecode_source_map::utils::source_map_from_file;
17use move_command_line_common::{
18    env::MOVE_HOME,
19    files::{
20        DEBUG_INFO_EXTENSION, MOVE_COMPILED_EXTENSION, MOVE_EXTENSION, extension_equals,
21        find_filenames,
22    },
23};
24use move_compiler::{
25    compiled_unit::NamedCompiledModule,
26    editions::{Edition, Flavor},
27    shared::{NumericalAddress, files::FileName},
28};
29use move_package::{
30    compilation::{
31        compiled_package::CompiledUnitWithSource, package_layout::CompiledPackageLayout,
32    },
33    lock_file::schema::{Header, ToolchainVersion},
34    source_package::{layout::SourcePackageLayout, parsed_manifest::PackageName},
35};
36use move_symbol_pool::Symbol;
37use tar::Archive;
38use tempfile::TempDir;
39use tracing::{debug, info};
40
41pub(crate) const CURRENT_COMPILER_VERSION: &str = env!("CARGO_PKG_VERSION");
42const LEGACY_COMPILER_VERSION: &str = CURRENT_COMPILER_VERSION; // TODO: update this when Move 2024 is released
43const PRE_TOOLCHAIN_MOVE_LOCK_VERSION: u16 = 0; // Used to detect lockfiles pre-toolchain versioning support
44const CANONICAL_UNIX_BINARY_NAME: &str = "sui";
45const CANONICAL_WIN_BINARY_NAME: &str = "sui.exe";
46
47pub(crate) fn current_toolchain() -> ToolchainVersion {
48    ToolchainVersion {
49        compiler_version: CURRENT_COMPILER_VERSION.into(),
50        edition: Edition::LEGACY, /* does not matter, unused for current_toolchain */
51        flavor: Flavor::Sui,      /* does not matter, unused for current_toolchain */
52    }
53}
54
55pub(crate) fn legacy_toolchain() -> ToolchainVersion {
56    ToolchainVersion {
57        compiler_version: LEGACY_COMPILER_VERSION.into(),
58        edition: Edition::LEGACY,
59        flavor: Flavor::Sui,
60    }
61}
62
63/// Ensures `compiled_units` are compiled with the right compiler version, based on
64/// Move.lock contents. This works by detecting if a compiled unit requires a prior compiler version:
65/// - If so, download the compiler, recompile the unit, and return that unit in the result.
66/// - If not, simply keep the current compiled unit.
67pub(crate) fn units_for_toolchain(
68    compiled_units: &Vec<(PackageName, CompiledUnitWithSource)>,
69) -> anyhow::Result<Vec<(PackageName, CompiledUnitWithSource)>> {
70    if std::env::var("SUI_RUN_TOOLCHAIN_BUILD").is_err() {
71        return Ok(compiled_units.clone());
72    }
73    let mut package_version_map: HashMap<Symbol, (ToolchainVersion, Vec<CompiledUnitWithSource>)> =
74        HashMap::new();
75    // First iterate over packages, mapping the required version for each package in `package_version_map`.
76    for (package, local_unit) in compiled_units {
77        if let Some((_, units)) = package_version_map.get_mut(package) {
78            // We've processed this package's required version.
79            units.push(local_unit.clone());
80            continue;
81        }
82
83        if sui_types::is_system_package(local_unit.unit.address.into_inner()) {
84            // System packages are always compiled with the current compiler.
85            package_version_map.insert(*package, (current_toolchain(), vec![local_unit.clone()]));
86            continue;
87        }
88
89        let package_root = SourcePackageLayout::try_find_root(&local_unit.source_path)?;
90        let lock_file = package_root.join(SourcePackageLayout::Lock.path());
91        if !lock_file.exists() {
92            // No lock file implies current compiler for this package.
93            package_version_map.insert(*package, (current_toolchain(), vec![local_unit.clone()]));
94            continue;
95        }
96
97        let mut lock_file = File::open(lock_file)?;
98        let lock_version = Header::read(&mut lock_file)?.version;
99        if lock_version == PRE_TOOLCHAIN_MOVE_LOCK_VERSION {
100            // No need to attempt reading lock file toolchain
101            debug!("{package} on legacy compiler",);
102            package_version_map.insert(*package, (legacy_toolchain(), vec![local_unit.clone()]));
103            continue;
104        }
105
106        // Read lock file toolchain info
107        lock_file.rewind()?;
108        let toolchain_version = ToolchainVersion::read(&mut lock_file)?;
109        match toolchain_version {
110            // No ToolchainVersion and new Move.lock version implies current compiler.
111            None => {
112                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
113                package_version_map
114                    .insert(*package, (current_toolchain(), vec![local_unit.clone()]));
115            }
116            // This dependency uses the current compiler.
117            Some(ToolchainVersion {
118                compiler_version, ..
119            }) if compiler_version == CURRENT_COMPILER_VERSION => {
120                debug!("{package} on current compiler @ {CURRENT_COMPILER_VERSION}",);
121                package_version_map
122                    .insert(*package, (current_toolchain(), vec![local_unit.clone()]));
123            }
124            // This dependency needs a prior compiler. Mark it and compile.
125            Some(toolchain_version) => {
126                println!(
127                    "{} {package} compiler @ {}",
128                    "REQUIRE".bold().green(),
129                    toolchain_version.compiler_version.yellow(),
130                );
131                package_version_map.insert(*package, (toolchain_version, vec![local_unit.clone()]));
132            }
133        }
134    }
135
136    let mut units = vec![];
137    // Iterate over compiled units, and check if they need to be recompiled and replaced by a prior compiler's output.
138    for (package, (toolchain_version, local_units)) in package_version_map {
139        if toolchain_version.compiler_version == CURRENT_COMPILER_VERSION {
140            let local_units: Vec<_> = local_units.iter().map(|u| (package, u.clone())).collect();
141            units.extend(local_units);
142            continue;
143        }
144
145        if local_units.is_empty() {
146            bail!("Expected one or more modules, but none found");
147        }
148        let package_root = SourcePackageLayout::try_find_root(&local_units[0].source_path)?;
149        let install_dir = tempfile::tempdir()?; // place compiled packages in this temp dir, don't pollute this packages build dir
150        download_and_compile(
151            package_root.clone(),
152            &install_dir,
153            &toolchain_version,
154            &package,
155        )?;
156
157        let compiled_unit_paths = vec![package_root.clone()];
158        let compiled_units = find_filenames(&compiled_unit_paths, |path| {
159            extension_equals(path, MOVE_COMPILED_EXTENSION)
160        })?;
161        let build_path = install_dir
162            .path()
163            .join(CompiledPackageLayout::path(&CompiledPackageLayout::Root))
164            .join(package.as_str());
165        debug!("build path is {}", build_path.display());
166
167        // Add all units compiled with the previous compiler.
168        for bytecode_path in compiled_units {
169            info!("bytecode path {bytecode_path}, {package}");
170            let local_unit = decode_bytecode_file(build_path.clone(), &package, &bytecode_path)?;
171            units.push((package, local_unit))
172        }
173    }
174    Ok(units)
175}
176
177fn download_and_compile(
178    root: PathBuf,
179    install_dir: &TempDir,
180    ToolchainVersion {
181        compiler_version,
182        edition,
183        flavor,
184    }: &ToolchainVersion,
185    dep_name: &Symbol,
186) -> anyhow::Result<()> {
187    let dest_dir = PathBuf::from_iter([&*MOVE_HOME, "binaries"]); // E.g., ~/.move/binaries
188    let dest_version = dest_dir.join(compiler_version);
189    let mut dest_canonical_path = dest_version.clone();
190    dest_canonical_path.extend(["target", "release"]);
191    let mut dest_canonical_binary = dest_canonical_path.clone();
192
193    let platform = detect_platform(&root, compiler_version, &dest_canonical_path)?;
194    if platform == "windows-x86_64" {
195        dest_canonical_binary.push(CANONICAL_WIN_BINARY_NAME);
196    } else {
197        dest_canonical_binary.push(CANONICAL_UNIX_BINARY_NAME);
198    }
199
200    if !dest_canonical_binary.exists() {
201        // Check the platform and proceed if we can download a binary. If not, the user should follow error instructions to sideload the binary.
202        // Download if binary does not exist.
203        let mainnet_url = format!(
204            "https://github.com/MystenLabs/sui/releases/download/mainnet-v{compiler_version}/sui-mainnet-v{compiler_version}-{platform}.tgz",
205        );
206
207        println!(
208            "{} mainnet compiler @ {} (this may take a while)",
209            "DOWNLOADING".bold().green(),
210            compiler_version.yellow()
211        );
212
213        let mut response = match ureq::get(&mainnet_url).call() {
214            Ok(response) => response,
215            Err(ureq::Error::Status(404, _)) => {
216                println!(
217                    "{} sui mainnet compiler {} not available, attempting to download testnet compiler release...",
218                    "WARNING".bold().yellow(),
219                    compiler_version.yellow()
220                );
221                println!(
222                    "{} testnet compiler @ {} (this may take a while)",
223                    "DOWNLOADING".bold().green(),
224                    compiler_version.yellow()
225                );
226                let testnet_url = format!("https://github.com/MystenLabs/sui/releases/download/testnet-v{compiler_version}/sui-testnet-v{compiler_version}-{platform}.tgz");
227                ureq::get(&testnet_url).call()?
228            }
229            Err(e) => return Err(e.into()),
230        }.into_reader();
231
232        let dest_tarball = dest_version.join(format!("{}.tgz", compiler_version));
233        debug!("tarball destination: {} ", dest_tarball.display());
234        if let Some(parent) = dest_tarball.parent() {
235            std::fs::create_dir_all(parent)
236                .map_err(|e| anyhow!("failed to create directory for tarball: {e}"))?;
237        }
238        let mut dest_file = File::create(&dest_tarball)?;
239        io::copy(&mut response, &mut dest_file)?;
240
241        // Extract the tarball using the tar crate
242        let tar_gz = File::open(&dest_tarball)?;
243        let tar = flate2::read::GzDecoder::new(tar_gz);
244        let mut archive = Archive::new(tar);
245        archive
246            .unpack(&dest_version)
247            .map_err(|e| anyhow!("failed to untar compiler binary: {e}"))?;
248
249        let mut dest_binary = dest_version.clone();
250        dest_binary.extend(["target", "release"]);
251        if platform == "windows-x86_64" {
252            dest_binary.push(format!("sui-{platform}.exe"));
253        } else {
254            dest_binary.push(format!("sui-{platform}"));
255        }
256        let dest_binary_os = OsStr::new(dest_binary.as_path());
257        set_executable_permission(dest_binary_os)?;
258        std::fs::rename(dest_binary_os, dest_canonical_binary.clone())?;
259    }
260
261    debug!(
262        "{} move build --default-move-edition {} --default-move-flavor {} -p {} --install-dir {}",
263        dest_canonical_binary.display(),
264        edition.to_string().as_str(),
265        flavor.to_string().as_str(),
266        root.display(),
267        install_dir.path().display(),
268    );
269    info!(
270        "{} {} (compiler @ {})",
271        "BUILDING".bold().green(),
272        dep_name.as_str(),
273        compiler_version.yellow()
274    );
275    Command::new(dest_canonical_binary)
276        .args([
277            OsStr::new("move"),
278            OsStr::new("build"),
279            OsStr::new("--default-move-edition"),
280            OsStr::new(edition.to_string().as_str()),
281            OsStr::new("--default-move-flavor"),
282            OsStr::new(flavor.to_string().as_str()),
283            OsStr::new("-p"),
284            OsStr::new(root.as_path()),
285            OsStr::new("--install-dir"),
286            OsStr::new(install_dir.path()),
287        ])
288        .output()
289        .map_err(|e| {
290            anyhow!("failed to build package from compiler binary {compiler_version}: {e}",)
291        })?;
292    Ok(())
293}
294
295fn detect_platform(
296    package_path: &Path,
297    compiler_version: &String,
298    dest_dir: &Path,
299) -> anyhow::Result<String> {
300    let s = match (std::env::consts::OS, std::env::consts::ARCH) {
301        ("macos", "aarch64") => "macos-arm64",
302        ("macos", "x86_64") => "macos-x86_64",
303        ("linux", "x86_64") => "ubuntu-x86_64",
304        ("windows", "x86_64") => "windows-x86_64",
305        (os, arch) => {
306            let mut binary_name = CANONICAL_UNIX_BINARY_NAME;
307            if os == "windows" {
308                binary_name = CANONICAL_WIN_BINARY_NAME;
309            };
310            bail!(
311                "The package {} needs to be built with sui compiler version {compiler_version} but there \
312                 is no binary release available to download for your platform:\n\
313                 Operating System: {os}\n\
314                 Architecture: {arch}\n\
315                 You can manually put a {binary_name} binary for your platform in {} and rerun your command to continue.",
316                package_path.display(),
317                dest_dir.display(),
318            )
319        }
320    };
321    Ok(s.into())
322}
323
324#[cfg(unix)]
325fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
326    use std::fs;
327    use std::os::unix::prelude::PermissionsExt;
328    let mut perms = fs::metadata(path)?.permissions();
329    perms.set_mode(0o755);
330    fs::set_permissions(path, perms)?;
331    Ok(())
332}
333
334#[cfg(not(unix))]
335fn set_executable_permission(path: &OsStr) -> anyhow::Result<()> {
336    Command::new("icacls")
337        .args([path, OsStr::new("/grant"), OsStr::new("Everyone:(RX)")])
338        .status()?;
339    Ok(())
340}
341
342fn decode_bytecode_file(
343    root_path: PathBuf,
344    package_name: &Symbol,
345    bytecode_path_str: &str,
346) -> anyhow::Result<CompiledUnitWithSource> {
347    let package_name_opt = Some(*package_name);
348    let bytecode_path = Path::new(bytecode_path_str);
349    let path_to_file = CompiledPackageLayout::path_to_file_after_category(bytecode_path);
350    let bytecode_bytes = std::fs::read(bytecode_path)?;
351    let source_map = source_map_from_file(
352        &root_path
353            .join(CompiledPackageLayout::DebugInfo.path())
354            .join(&path_to_file)
355            .with_extension(DEBUG_INFO_EXTENSION),
356    )?;
357    let source_path = &root_path
358        .join(CompiledPackageLayout::Sources.path())
359        .join(path_to_file)
360        .with_extension(MOVE_EXTENSION);
361    ensure!(
362        source_path.is_file(),
363        "Error decoding package: Unable to find corresponding source file for '{bytecode_path_str}' in package {package_name}"
364    );
365    let module = CompiledModule::deserialize_with_defaults(&bytecode_bytes)?;
366    let (address_bytes, module_name) = {
367        let id = module.self_id();
368        let parsed_addr = NumericalAddress::new(
369            id.address().into_bytes(),
370            move_compiler::shared::NumberFormat::Hex,
371        );
372        let module_name = FileName::from(id.name().as_str());
373        (parsed_addr, module_name)
374    };
375    let unit = NamedCompiledModule {
376        package_name: package_name_opt,
377        address: address_bytes,
378        name: module_name,
379        module,
380        source_map,
381        address_name: None,
382    };
383    Ok(CompiledUnitWithSource {
384        unit,
385        source_path: source_path.clone(),
386    })
387}
sui_source_validation/toolchain.rs

sui_source_validation/
toolchain.rs
