Skip to main content

sui_rpc_api/grpc/v2/move_package_service/
get_package.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use crate::{
5    ErrorReason, Result, RpcError, RpcService,
6    grpc::v2::move_package_service::{
7        conversions::{convert_error, convert_module},
8        load_package, load_package_by_id, load_package_by_id_and_version, parse_package_id,
9    },
10    reader::{NotReadyCause, index_empty, indexes_disabled, not_ready},
11};
12use sui_rpc::proto::google::rpc::bad_request::FieldViolation;
13use sui_rpc::proto::sui::rpc::v2::get_package_request::Selector;
14use sui_rpc::proto::sui::rpc::v2::{GetPackageRequest, GetPackageResponse, Package};
15use sui_types::base_types::ObjectID;
16use sui_types::move_package::MovePackage;
17use sui_types::object::OBJECT_START_VERSION;
18
19#[tracing::instrument(skip(service))]
20pub fn get_package(service: &RpcService, request: GetPackageRequest) -> Result<GetPackageResponse> {
21    let package_id_str = request.package_id.as_ref().ok_or_else(|| {
22        FieldViolation::new("package_id")
23            .with_description("missing package_id")
24            .with_reason(ErrorReason::FieldMissing)
25    })?;
26
27    let package = match request.selector {
28        Some(Selector::Version(version)) => {
29            load_package_at_version(service, package_id_str, version)?
30        }
31        Some(Selector::AtCheckpoint(at_checkpoint)) => {
32            load_package_at_checkpoint(service, package_id_str, at_checkpoint)?
33        }
34        None => load_package(service, package_id_str)?,
35        _ => {
36            return Err(FieldViolation::new("selector")
37                .with_description("unknown selector variant")
38                .with_reason(ErrorReason::FieldInvalid)
39                .into());
40        }
41    };
42
43    get_package_response(&package)
44}
45
46/// Load the package in `package_id_str`'s upgrade lineage with exactly
47/// `version`.
48///
49/// System packages are upgraded in place: all their versions share one ID, so
50/// an earlier version is read by that ID and the version, without the
51/// package-version index. Nodes without object version history refuse that
52/// read (`OutOfRange`, `VERSION_NOT_RETAINED`) even while the version is still
53/// stored. Other packages keep each version at its own storage ID, found
54/// through the package-version index, whose rows are keyed by the lineage's
55/// original package id.
56///
57/// A version this node does not store is refused (see
58/// [`refuse_unstored_version`]). A version below the loaded package's that the
59/// package-version index has no row for is refused as not ready while the
60/// index has no row for the loaded package either: the index trails the
61/// node's live state, and the requested version may have been published after
62/// the index's tip. Any other version the index has no row for is `NotFound`,
63/// unless the index is empty or still backfilling history after a restore and
64/// the row may yet be written.
65fn load_package_at_version(
66    service: &RpcService,
67    package_id_str: &str,
68    version: u64,
69) -> Result<MovePackage> {
70    let package = load_package(service, package_id_str)?;
71    let original_id = package.original_package_id();
72    if package.version().value() == version {
73        return Ok(package);
74    }
75    // A lineage's versions run from `OBJECT_START_VERSION` up without gaps, so a version
76    // below the loaded package's existed.
77    let existed = (OBJECT_START_VERSION.value()..package.version().value()).contains(&version);
78
79    // Only a package upgraded in place keeps its original ID past its first version, so the
80    // loaded package is its lineage's latest version.
81    if package.id() == original_id && package.version() > OBJECT_START_VERSION {
82        if !existed {
83            return Err(RpcError::not_found());
84        }
85        service.reader.require_object_version_history()?;
86        return match load_package_by_id_and_version(service, original_id, version)? {
87            Some(package) => Ok(package),
88            None => refuse_unstored_version(service, original_id, version),
89        };
90    }
91
92    let indexes = service
93        .reader
94        .inner()
95        .indexes()
96        .ok_or_else(indexes_disabled)?;
97
98    let storage_id = indexes
99        .get_package_version_storage_id(original_id, version)
100        .map_err(|e| RpcError::new(tonic::Code::Internal, e.to_string()))?;
101    let stored = match storage_id {
102        Some(storage_id) => load_package_by_id_and_version(service, storage_id, version)?,
103        None if existed => {
104            let latest_storage_id = indexes
105                .get_package_version_storage_id(original_id, package.version().value())
106                .map_err(|e| RpcError::new(tonic::Code::Internal, e.to_string()))?;
107            if latest_storage_id.is_none() {
108                check_restore_backfill_done(service)?;
109                return Err(not_ready(
110                    NotReadyCause::HistoryCatchingUp,
111                    format!(
112                        "the rpc indexes have not yet reached version {} of package {original_id}",
113                        package.version().value()
114                    ),
115                ));
116            }
117            // The index writes a lineage's rows in publish order, so with the loaded version's
118            // row present the requested version's row is too, unless it was missing because the
119            // index committed both rows between the two reads: read it again.
120            match indexes
121                .get_package_version_storage_id(original_id, version)
122                .map_err(|e| RpcError::new(tonic::Code::Internal, e.to_string()))?
123            {
124                Some(storage_id) => load_package_by_id_and_version(service, storage_id, version)?,
125                None => None,
126            }
127        }
128        // No lineage has a version below `OBJECT_START_VERSION`.
129        None if version < OBJECT_START_VERSION.value() => return Err(RpcError::not_found()),
130        None => {
131            check_restore_backfill_done(service)?;
132            return Err(RpcError::not_found());
133        }
134    };
135
136    match stored {
137        Some(package) => Ok(package),
138        None => refuse_unstored_version(service, original_id, version),
139    }
140}
141
142/// Refuse `version` of the package lineage `original_id`, which existed but this node does not
143/// store. The refusal is not ready while the rpc indexes are empty or backfill history after a
144/// restore, pruned when this node's object history has a floor (a node restored from a snapshot
145/// has none of the versions replaced before it), and an internal error on a node that has pruned
146/// nothing, which stores every version.
147fn refuse_unstored_version(
148    service: &RpcService,
149    original_id: ObjectID,
150    version: u64,
151) -> Result<MovePackage> {
152    check_restore_backfill_done(service)?;
153    let lower_bound = service.reader.at_checkpoint_lower_bound()?;
154    if lower_bound.pruned_floor().is_none() {
155        return Err(RpcError::new(
156            tonic::Code::Internal,
157            format!(
158                "version {version} of package {original_id} is not stored, although this node \
159                 has pruned no object history"
160            ),
161        ));
162    }
163    Err(lower_bound.pruned())
164}
165
166/// `UNAVAILABLE` (`NOT_READY`) while the rpc indexes backfill history after a restore (cause
167/// `restore_backfill`) or have not committed a checkpoint yet (cause `index_empty`), when an
168/// answer that a package version is missing may not hold once the indexes catch up.
169fn check_restore_backfill_done(service: &RpcService) -> Result<()> {
170    let Some(indexes) = service.reader.inner().indexes() else {
171        return Ok(());
172    };
173    if indexes
174        .is_history_backfill_pending()
175        .map_err(|e| RpcError::new(tonic::Code::Internal, e.to_string()))?
176    {
177        return Err(not_ready(
178            NotReadyCause::RestoreBackfill,
179            "rpc index history is still backfilling after a restore",
180        ));
181    }
182    if indexes
183        .get_highest_indexed_checkpoint_seq_number()
184        .map_err(|e| RpcError::new(tonic::Code::Internal, e.to_string()))?
185        .is_none()
186    {
187        return Err(index_empty());
188    }
189    Ok(())
190}
191
192/// Load the latest package in `package_id_str`'s upgrade lineage that
193/// existed at or before `at_checkpoint`.
194///
195/// `at_checkpoint` is checked against the shared checkpoint-pinned read
196/// bounds, the upper bound before the package read and the lower bound after
197/// it (see `StateReader::at_checkpoint_upper_bound`), so a successful answer
198/// is exact as of `at_checkpoint`. Both bounds apply to every outcome of the
199/// read, `NotFound` included, so an out-of-range checkpoint is `OutOfRange`
200/// whether or not the package exists, as on kv-rpc.
201fn load_package_at_checkpoint(
202    service: &RpcService,
203    package_id_str: &str,
204    at_checkpoint: u64,
205) -> Result<MovePackage> {
206    let package_id = parse_package_id(package_id_str)?;
207
208    service
209        .reader
210        .at_checkpoint_upper_bound()?
211        .check(at_checkpoint)?;
212
213    let indexes = service
214        .reader
215        .inner()
216        .indexes()
217        .ok_or_else(indexes_disabled)?;
218
219    let package = load_package_by_id(service, package_id).and_then(|package| {
220        indexes
221            .get_package_at_checkpoint(package.original_package_id(), at_checkpoint)
222            .map_err(|e| RpcError::new(tonic::Code::Internal, e.to_string()))?
223            .ok_or_else(RpcError::not_found)
224            .and_then(|(version, storage_id)| {
225                // The lower bound below replaces any answer at a checkpoint it refuses; at the
226                // others, pruning has not reached the version current at `at_checkpoint`.
227                load_package_by_id_and_version(service, storage_id, version)?.ok_or_else(|| {
228                    RpcError::new(
229                        tonic::Code::Internal,
230                        format!(
231                            "package {storage_id} version {version} has a package-version index \
232                             row but no stored object"
233                        ),
234                    )
235                })
236            })
237    });
238
239    service
240        .reader
241        .at_checkpoint_lower_bound()?
242        .check(at_checkpoint)?;
243    package
244}
245
246/// Build a `GetPackage` response from an already-loaded package. Shared with other implementors
247/// of `MovePackageService` (e.g. sui-kv-rpc) that load packages from a different backend.
248pub fn get_package_response(package: &MovePackage) -> Result<GetPackageResponse> {
249    let package_id = package.id();
250
251    let resolved_package =
252        sui_package_resolver::Package::read_from_package(package).map_err(convert_error)?;
253
254    let modules: Vec<_> = resolved_package
255        .modules()
256        .iter()
257        .map(|(module_name, resolver_module)| {
258            convert_module(module_name, resolver_module, &package_id)
259        })
260        .collect::<Result<Vec<_>>>()?;
261
262    let mut message = Package::default();
263    message.storage_id = Some(package_id.to_canonical_string(true));
264    message.original_id = Some(package.original_package_id().to_canonical_string(true));
265    message.version = Some(package.version().value());
266    message.modules = modules;
267
268    Ok(GetPackageResponse::new(message))
269}
270
271#[cfg(test)]
272mod tests {
273    use std::sync::Arc;
274
275    use move_binary_format::file_format::{CompiledModule, empty_module};
276    use sui_consistent_store::{ChainId, Db, FrameworkSchema, PipelineTaskKey, restore_state};
277    use sui_protocol_config::ProtocolConfig;
278    use sui_rpc_store::RpcStoreSchema;
279    use sui_rpc_store::schema::{objects, package_versions, pruning_watermark};
280    use sui_types::base_types::SequenceNumber;
281    use sui_types::digests::TransactionDigest;
282    use sui_types::object::{Data, Object};
283
284    use super::*;
285    use crate::ledger_history::query_options::assert_below_serving_floor_error;
286    use crate::reader::tests::{assert_refusal, commit_every_pipeline, record_restore_state};
287
288    /// An `RpcService` over an rpc-store holding `packages` and the package-version index rows
289    /// `(original ID, version, storage ID)` in `index_rows`.
290    struct Fixture {
291        _dir: tempfile::TempDir,
292        db: Db,
293        schema: Arc<RpcStoreSchema>,
294        service: RpcService,
295    }
296
297    impl Fixture {
298        fn new(packages: &[Object], index_rows: &[(ObjectID, u64, ObjectID)]) -> Self {
299            let dir = tempfile::tempdir().unwrap();
300            let (db, schema) =
301                Db::open::<RpcStoreSchema>(dir.path(), sui_consistent_store::DbOptions::default())
302                    .unwrap();
303            let schema = Arc::new(schema);
304
305            let mut batch = db.batch();
306            for package in packages {
307                let key = objects::Key {
308                    id: package.id(),
309                    version: package.version(),
310                };
311                batch
312                    .put(&schema.objects, &key, &objects::store(package))
313                    .unwrap();
314            }
315            for &(original_id, version, storage_id) in index_rows {
316                let (key, value) = package_versions::store(original_id, version, storage_id, 0);
317                batch.put(&schema.package_versions, &key, &value).unwrap();
318            }
319            batch
320                .put(
321                    &FrameworkSchema::new(db.clone()).chain_ids,
322                    &PipelineTaskKey::new("test"),
323                    &ChainId([1; 32]),
324                )
325                .unwrap();
326            batch.commit().unwrap();
327
328            let service = RpcService::new(Arc::new(sui_rpc_store::RpcStoreReader::new(
329                db.clone(),
330                schema.clone(),
331            )));
332            Self {
333                _dir: dir,
334                db,
335                schema,
336                service,
337            }
338        }
339
340        fn get_version(&self, package_id: ObjectID, version: u64) -> Result<GetPackageResponse> {
341            let mut request = GetPackageRequest::default();
342            request.package_id = Some(package_id.to_string());
343            request.selector = Some(Selector::Version(version));
344            get_package(&self.service, request)
345        }
346
347        fn assert_served(&self, package_id: ObjectID, version: u64) {
348            let package = self
349                .get_version(package_id, version)
350                .unwrap()
351                .package
352                .unwrap();
353            assert_eq!(package.version, Some(version));
354        }
355
356        fn assert_code(&self, package_id: ObjectID, version: u64, code: tonic::Code) {
357            let status: tonic::Status = self.get_version(package_id, version).unwrap_err().into();
358            assert_eq!(status.code(), code, "{package_id} v{version}: {status:?}");
359        }
360
361        fn assert_restore_backfill(&self, package_id: ObjectID, version: u64) {
362            self.assert_not_ready(package_id, version, "restore_backfill");
363        }
364
365        fn assert_not_ready(&self, package_id: ObjectID, version: u64, cause: &str) {
366            assert_refusal(
367                self.get_version(package_id, version).unwrap_err(),
368                tonic::Code::Unavailable,
369                ErrorReason::NotReady,
370                cause,
371            );
372        }
373
374        fn start_restore_backfill(&self) {
375            record_restore_state(
376                &self.db,
377                &["object_version_by_checkpoint"],
378                restore_state::State::InProgress(restore_state::InProgress::default()),
379            );
380        }
381
382        fn finish_restore_backfill(&self) {
383            record_restore_state(
384                &self.db,
385                &["object_version_by_checkpoint"],
386                restore_state::State::Complete(restore_state::Complete { restored_at: 20 }),
387            );
388            commit_every_pipeline(&self.db, 20);
389        }
390
391        /// Prune object history below `lowest_available`.
392        fn prune(&self, lowest_available: u64) {
393            let (key, value) = pruning_watermark::store(&pruning_watermark::Watermarks {
394                tx_seq_lo: 100,
395                checkpoint_lo: lowest_available,
396            });
397            let mut batch = self.db.batch();
398            batch
399                .put(&self.schema.pruning_watermark, &key, &value)
400                .unwrap();
401            batch.commit().unwrap();
402        }
403    }
404
405    /// A module whose self-address, and so its package's original ID, is `original_id`.
406    fn module(original_id: ObjectID) -> CompiledModule {
407        let mut module = empty_module();
408        module.address_identifiers[0] = original_id.into();
409        module
410    }
411
412    /// A user package lineage with one version per ID in `storage_ids`, from version 1 at the
413    /// original ID `storage_ids[0]` up.
414    fn user_package_lineage(storage_ids: &[ObjectID]) -> Vec<Object> {
415        let original_id = storage_ids[0];
416        let config = ProtocolConfig::get_for_max_version_UNSAFE();
417        let mut lineage = vec![
418            Object::new_package(
419                &[module(original_id)],
420                TransactionDigest::genesis_marker(),
421                &config,
422                [],
423            )
424            .unwrap(),
425        ];
426        for &storage_id in &storage_ids[1..] {
427            let Data::Package(previous) = &lineage.last().unwrap().data else {
428                unreachable!("packages only");
429            };
430            let upgraded = previous
431                .new_upgraded(storage_id, &[module(original_id)], &config, [])
432                .unwrap();
433            lineage.push(Object::new_package_from_data(
434                Data::Package(upgraded),
435                TransactionDigest::genesis_marker(),
436            ));
437        }
438        lineage
439    }
440
441    /// A system package upgraded in place to version 3 whose version 2 is not stored, while a
442    /// restore's history backfill is pending, after it completes, and once the node has pruned.
443    /// A stored replaced version is served throughout; the missing one is refused as not ready,
444    /// then as an internal error on a node that has pruned nothing, then as pruned.
445    #[test]
446    fn replaced_system_package_version_is_read_before_it_is_refused() {
447        let package_id = ObjectID::from_single_byte(0xAB);
448        let packages: Vec<_> = [1, 3]
449            .into_iter()
450            .map(|version| {
451                Object::new_package_from_data(
452                    Data::Package(MovePackage::new_system(
453                        SequenceNumber::from_u64(version),
454                        &[module(package_id)],
455                        [],
456                    )),
457                    TransactionDigest::genesis_marker(),
458                )
459            })
460            .collect();
461        let fixture = Fixture::new(&packages, &[]);
462
463        fixture.start_restore_backfill();
464        fixture.assert_served(package_id, 1);
465        fixture.assert_served(package_id, 3);
466        fixture.assert_restore_backfill(package_id, 2);
467
468        fixture.finish_restore_backfill();
469        fixture.assert_served(package_id, 1);
470        fixture.assert_code(package_id, 2, tonic::Code::Internal);
471
472        fixture.prune(10);
473        fixture.assert_served(package_id, 1);
474        assert_below_serving_floor_error(fixture.get_version(package_id, 2).unwrap_err(), 9);
475
476        // Versions this lineage never had.
477        fixture.assert_code(package_id, 0, tonic::Code::NotFound);
478        fixture.assert_code(package_id, 4, tonic::Code::NotFound);
479    }
480
481    /// A user package lineage `v1` at `v1_id`, `v2` at `v2_id` and `v3` at `v3_id`, read through
482    /// the package-version index while it lacks `v3`'s row and the node lacks `v2`'s object. A
483    /// version without an index row is refused as not ready during a restore's history backfill
484    /// and is `NotFound` after it, except version 0, which no lineage has and is always
485    /// `NotFound`; an indexed version without its object is refused as not ready, then as an
486    /// internal error on a node that has pruned nothing, then as pruned.
487    #[test]
488    fn user_package_version_missing_from_index_or_store_is_refused() {
489        let v1_id = ObjectID::from_single_byte(0xA1);
490        let v2_id = ObjectID::from_single_byte(0xB2);
491        let v3_id = ObjectID::from_single_byte(0xC3);
492        let [v1, _, v3]: [Object; 3] = user_package_lineage(&[v1_id, v2_id, v3_id])
493            .try_into()
494            .unwrap();
495        let fixture = Fixture::new(&[v1, v3], &[(v1_id, 1, v1_id), (v1_id, 2, v2_id)]);
496
497        fixture.start_restore_backfill();
498        fixture.assert_served(v3_id, 1);
499        fixture.assert_restore_backfill(v3_id, 2);
500        fixture.assert_restore_backfill(v1_id, 3);
501        fixture.assert_code(v3_id, 0, tonic::Code::NotFound);
502
503        fixture.finish_restore_backfill();
504        fixture.assert_served(v3_id, 1);
505        fixture.assert_code(v3_id, 2, tonic::Code::Internal);
506        fixture.assert_code(v1_id, 3, tonic::Code::NotFound);
507
508        fixture.prune(10);
509        assert_below_serving_floor_error(fixture.get_version(v3_id, 2).unwrap_err(), 9);
510    }
511
512    /// A user package lineage `v1` at `v1_id` and `v2` at `v2_id`, both stored, while the
513    /// package-version index trails the node's live state and has a row for neither: both were
514    /// published after the index's tip. The older version is refused as not ready (the index
515    /// is catching up), on a node that has pruned nothing and on one that has pruned, rather
516    /// than as an unstored version.
517    #[test]
518    fn user_package_version_published_after_index_tip_is_not_ready() {
519        let v1_id = ObjectID::from_single_byte(0xA1);
520        let v2_id = ObjectID::from_single_byte(0xB2);
521        let fixture = Fixture::new(&user_package_lineage(&[v1_id, v2_id]), &[]);
522        commit_every_pipeline(&fixture.db, 20);
523
524        fixture.assert_served(v2_id, 2);
525        fixture.assert_not_ready(v2_id, 1, "history_catching_up");
526
527        fixture.prune(10);
528        fixture.assert_not_ready(v2_id, 1, "history_catching_up");
529    }
530
531    /// A user package lineage `v1` at `v1_id` and `v2` at `v2_id` on a node whose rpc indexes
532    /// have not committed a checkpoint yet. A version the index has no row for is refused as not
533    /// ready (the index is empty), whether or not it existed, and once the index has committed a
534    /// checkpoint a version the lineage never had is `NotFound`.
535    #[test]
536    fn user_package_version_read_on_empty_index_is_not_ready() {
537        let v1_id = ObjectID::from_single_byte(0xA1);
538        let v2_id = ObjectID::from_single_byte(0xB2);
539        let fixture = Fixture::new(&user_package_lineage(&[v1_id, v2_id]), &[]);
540
541        fixture.assert_not_ready(v2_id, 1, "index_empty");
542        fixture.assert_not_ready(v1_id, 3, "index_empty");
543
544        commit_every_pipeline(&fixture.db, 20);
545        fixture.assert_code(v1_id, 3, tonic::Code::NotFound);
546    }
547}