Skip to main content

sui_core/transaction_driver/
mod.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4mod effects_certifier;
5mod error;
6mod metrics;
7mod reconfig_observer;
8mod request_retrier;
9mod transaction_submitter;
10
11/// Exports
12pub use error::TransactionDriverError;
13pub use metrics::*;
14pub use reconfig_observer::{OnsiteReconfigObserver, ReconfigObserver};
15
16use std::{
17    net::SocketAddr,
18    sync::Arc,
19    time::{Duration, Instant},
20};
21
22use arc_swap::ArcSwap;
23use effects_certifier::*;
24use mysten_common::backoff::ExponentialBackoff;
25use mysten_metrics::{monitored_future, spawn_logged_monitored_task};
26use nonempty::NonEmpty;
27use parking_lot::Mutex;
28use rand::Rng;
29use request_retrier::SELECT_LATENCY_DELTA;
30use sui_config::NodeConfig;
31use sui_types::{
32    base_types::AuthorityName,
33    committee::EpochId,
34    error::{ErrorCategory, SuiErrorKind, UserInputError},
35    messages_grpc::{SubmitTxRequest, SubmitTxResult, TxType},
36    transaction::{AllowedProposers, TransactionDataAPI as _},
37    transaction_executor::ProposerSelector,
38};
39use tokio::{
40    task::JoinSet,
41    time::{interval, sleep},
42};
43use tracing::instrument;
44use transaction_submitter::*;
45
46use crate::{
47    authority::{AuthorityState, authority_per_epoch_store::check_timestamp_window},
48    authority_aggregator::AuthorityAggregator,
49    authority_client::AuthorityAPI,
50    validator_client_monitor::{
51        OperationFeedback, OperationType, TransactionClass, ValidatorClientMetrics,
52        ValidatorClientMonitor,
53    },
54};
55
56#[cfg(test)]
57#[path = "unit_tests/proposer_selector_tests.rs"]
58mod proposer_selector_tests;
59
60/// Trait for components that can update their AuthorityAggregator during reconfiguration.
61/// Used by ReconfigObserver to notify components of epoch changes.
62pub trait AuthorityAggregatorUpdatable<A: Clone>: Send + Sync + 'static {
63    fn epoch(&self) -> EpochId;
64    fn authority_aggregator(&self) -> Arc<AuthorityAggregator<A>>;
65    fn update_authority_aggregator(&self, new_authorities: Arc<AuthorityAggregator<A>>);
66}
67
68/// Options for submitting a transaction.
69#[derive(Clone, Default, Debug)]
70pub struct SubmitTransactionOptions {
71    /// When forwarding transactions on behalf of a client, this is the client's address
72    /// specified for ddos protection.
73    pub forwarded_client_addr: Option<SocketAddr>,
74
75    /// When submitting a transaction, only the validators in the allowed validator list can be used to submit the transaction to.
76    /// When the allowed validator list is empty, any validator can be used.
77    pub allowed_validators: Vec<String>,
78
79    /// When submitting a transaction, the validators in the blocked validator list cannot be used to submit the transaction to.
80    /// When the blocked validator list is empty, no restrictions are applied.
81    pub blocked_validators: Vec<String>,
82}
83
84#[derive(Clone, Debug)]
85pub struct QuorumTransactionResponse {
86    pub effects: sui_types::transaction_driver_types::FinalizedEffects,
87
88    pub events: Option<sui_types::effects::TransactionEvents>,
89    // Input objects will only be populated in the happy path
90    pub input_objects: Option<Vec<sui_types::object::Object>>,
91    // Output objects will only be populated in the happy path
92    pub output_objects: Option<Vec<sui_types::object::Object>>,
93    pub auxiliary_data: Option<Vec<u8>>,
94}
95
96pub struct TransactionDriver<A: Clone> {
97    authority_aggregator: Arc<ArcSwap<AuthorityAggregator<A>>>,
98    state: Mutex<State>,
99    metrics: Arc<TransactionDriverMetrics>,
100    submitter: TransactionSubmitter,
101    certifier: EffectsCertifier,
102    client_monitor: Arc<ValidatorClientMonitor<A>>,
103}
104
105impl<A> TransactionDriver<A>
106where
107    A: AuthorityAPI + Send + Sync + 'static + Clone,
108{
109    // TODO: accept a TransactionDriverConfig to set default allowed & blocked validators.
110    pub fn new(
111        authority_aggregator: Arc<AuthorityAggregator<A>>,
112        reconfig_observer: Arc<dyn ReconfigObserver<A> + Sync + Send>,
113        metrics: Arc<TransactionDriverMetrics>,
114        node_config: Option<&NodeConfig>,
115        client_metrics: Arc<ValidatorClientMetrics>,
116    ) -> Arc<Self> {
117        if std::env::var("TRANSACTION_DRIVER").is_ok() {
118            tracing::warn!(
119                "Transaction Driver is the only supported driver for transaction submission. Setting TRANSACTION_DRIVER is a no-op."
120            );
121        }
122
123        let shared_swap = Arc::new(ArcSwap::new(authority_aggregator));
124
125        // Extract validator client monitor config from NodeConfig or use default
126        let monitor_config = node_config
127            .and_then(|nc| nc.validator_client_monitor_config.clone())
128            .unwrap_or_default();
129        let client_monitor =
130            ValidatorClientMonitor::new(monitor_config, client_metrics, shared_swap.clone());
131
132        let driver = Arc::new(Self {
133            authority_aggregator: shared_swap,
134            state: Mutex::new(State::new()),
135            metrics: metrics.clone(),
136            submitter: TransactionSubmitter::new(metrics.clone()),
137            certifier: EffectsCertifier::new(metrics),
138            client_monitor,
139        });
140
141        let driver_clone = driver.clone();
142
143        spawn_logged_monitored_task!(Self::run_latency_checks(driver_clone));
144
145        driver.enable_reconfig(reconfig_observer);
146        driver
147    }
148
149    /// Returns the authority aggregator wrapper which upgrades on epoch changes.
150    pub fn authority_aggregator(&self) -> &Arc<ArcSwap<AuthorityAggregator<A>>> {
151        &self.authority_aggregator
152    }
153
154    pub fn select_preferred_validators(
155        &self,
156        delta: f64,
157        tx_class: TransactionClass,
158    ) -> Vec<AuthorityName> {
159        let authority_aggregator = self.authority_aggregator.load();
160        self.client_monitor.select_shuffled_preferred_validators(
161            &authority_aggregator.committee,
162            delta,
163            tx_class,
164        )
165    }
166
167    /// The validators this node would prefer to submit to, as committee indices.
168    ///
169    /// These are the same targets `RequestRetrier` would pick, so a transaction restricted to them
170    /// names the validators it was going to be sent to anyway.
171    fn preferred_proposers_impl(&self, max: usize) -> Option<AllowedProposers> {
172        // Before any latency has been observed the ranking is an arbitrary shuffle, so pinning to
173        // it would be worse than leaving the transaction unrestricted.
174        if !self.client_monitor.has_observed_latencies() {
175            return None;
176        }
177
178        let authority_aggregator = self.authority_aggregator.load();
179        let committee = &authority_aggregator.committee;
180        // The set restricts the transaction, so it is chosen from restricted-class
181        // stats — the view staggering holds cannot pollute.
182        let mut proposers: Vec<u32> = self
183            .client_monitor
184            .select_shuffled_preferred_validators(
185                committee,
186                SELECT_LATENCY_DELTA,
187                TransactionClass::Restricted,
188            )
189            .into_iter()
190            .filter_map(|name| committee.authority_index(&name))
191            .take(max)
192            .collect();
193        // The set is unordered preference; `Validity` requires it strictly increasing.
194        proposers.sort_unstable();
195        proposers.dedup();
196
197        Some(AllowedProposers {
198            epoch: committee.epoch(),
199            proposers: NonEmpty::from_vec(proposers)?,
200        })
201    }
202
203    /// Drives transaction to finalization.
204    ///
205    /// Internally, retries the attempt to finalize a transaction until:
206    /// - The transaction is finalized.
207    /// - The transaction observes a non-retriable error.
208    /// - Timeout is reached.
209    #[instrument(level = "error", skip_all, fields(tx_digest = ?request.transaction.as_ref().map(|t| t.digest()), ping = %request.ping_type.is_some()))]
210    pub async fn drive_transaction(
211        &self,
212        request: SubmitTxRequest,
213        options: SubmitTransactionOptions,
214        timeout_duration: Option<Duration>,
215    ) -> Result<QuorumTransactionResponse, TransactionDriverError> {
216        const MAX_DRIVE_TRANSACTION_RETRY_DELAY: Duration = Duration::from_secs(10);
217
218        let tx_data = request.transaction.as_ref().map(|t| t.transaction_data());
219        // gas_price=0 for gasless; use 1 for baseline (RGP-equivalent) priority
220        let amplification_factor =
221            if request.ping_type.is_some() || tx_data.is_some_and(|d| d.is_gasless_transaction()) {
222                1
223            } else {
224                let tx_data = tx_data.unwrap();
225                let gas_price = tx_data.gas_price();
226                let reference_gas_price = self.authority_aggregator.load().reference_gas_price;
227                let amplification_factor = gas_price / reference_gas_price.max(1);
228                if amplification_factor == 0 {
229                    return Err(TransactionDriverError::ValidationFailed {
230                        error: UserInputError::GasPriceUnderRGP {
231                            gas_price,
232                            reference_gas_price,
233                        }
234                        .to_string(),
235                    });
236                }
237                amplification_factor
238            };
239
240        // Half of transactions skip this check so that validators see transactions outside
241        // their windows.
242        let skip_timestamp_check = mysten_common::in_test_configuration()
243            && mysten_common::random::get_rng().gen_bool(0.5);
244        let mut not_yet_valid_wait = None;
245        if let Some(tx_data) = tx_data
246            && !skip_timestamp_check
247        {
248            let expiration = tx_data.expiration();
249            let now_ms = AuthorityState::unixtime_now_ms();
250            match check_timestamp_window(expiration, now_ms) {
251                Ok(()) => {}
252                // Validators would reject the transaction until `min_timestamp`, so the driver
253                // waits until then. The wait counts against the finality timeout. A wait that
254                // would outlast it fails at once with a non-retriable error, so that no
255                // background retry submits the transaction after the client sees the failure.
256                Err(error)
257                    if matches!(
258                        error.as_inner(),
259                        SuiErrorKind::TransactionNotYetValid { .. }
260                    ) =>
261                {
262                    let valid_from_ms = expiration.first_valid_timestamp_ms().unwrap_or(now_ms);
263                    let wait = Duration::from_millis(valid_from_ms.saturating_sub(now_ms));
264                    if timeout_duration.is_some_and(|timeout| wait >= timeout) {
265                        return Err(TransactionDriverError::ValidationFailed {
266                            error: error.to_string(),
267                        });
268                    }
269                    not_yet_valid_wait = Some(wait);
270                }
271                Err(error) => {
272                    return Err(TransactionDriverError::ValidationFailed {
273                        error: error.to_string(),
274                    });
275                }
276            }
277        }
278
279        let tx_type = request.tx_type();
280        let ping_label = if request.ping_type.is_some() {
281            "true"
282        } else {
283            "false"
284        };
285        let mut backoff = ExponentialBackoff::new(
286            Duration::from_millis(100),
287            MAX_DRIVE_TRANSACTION_RETRY_DELAY,
288        );
289        let mut attempts = 0;
290        let mut latest_retriable_error = None;
291
292        let retry_loop = async {
293            if let Some(wait) = not_yet_valid_wait {
294                sleep(wait).await;
295            }
296            // Started after the wait, so that finality latency excludes the delay the client
297            // chose with `min_timestamp`.
298            let timer = Instant::now();
299            self.metrics
300                .total_transactions_submitted
301                .with_label_values(&[tx_type.as_str(), ping_label])
302                .inc();
303            loop {
304                // TODO(fastpath): Check local state before submitting transaction
305                match self
306                    .drive_transaction_once(amplification_factor, request.clone(), &options)
307                    .await
308                {
309                    Ok(resp) => {
310                        let settlement_finality_latency = timer.elapsed().as_secs_f64();
311                        self.metrics
312                            .settlement_finality_latency
313                            .with_label_values(&[tx_type.as_str(), ping_label])
314                            .observe(settlement_finality_latency);
315                        let is_out_of_expected_range = settlement_finality_latency >= 8.0
316                            || settlement_finality_latency <= 0.1;
317                        tracing::debug!(
318                            ?tx_type,
319                            ?is_out_of_expected_range,
320                            "Settlement finality latency: {:.3} seconds",
321                            settlement_finality_latency
322                        );
323                        // Record the number of retries for successful transaction
324                        self.metrics
325                            .transaction_retries
326                            .with_label_values(&["success", tx_type.as_str(), ping_label])
327                            .observe(attempts as f64);
328                        return Ok(resp);
329                    }
330                    Err(e) => {
331                        self.metrics
332                            .drive_transaction_errors
333                            .with_label_values(&[
334                                e.categorize().into(),
335                                tx_type.as_str(),
336                                ping_label,
337                            ])
338                            .inc();
339                        if !e.is_submission_retriable() {
340                            // Record the number of retries for failed transaction
341                            self.metrics
342                                .transaction_retries
343                                .with_label_values(&["failure", tx_type.as_str(), ping_label])
344                                .observe(attempts as f64);
345                            if request.transaction.is_some() {
346                                tracing::info!(
347                                    "User transaction failed to finalize (attempt {}), with non-retriable error: {} ({})",
348                                    attempts,
349                                    e,
350                                    Into::<&str>::into(e.categorize())
351                                );
352                            }
353                            return Err(e);
354                        }
355                        if request.transaction.is_some() {
356                            tracing::info!(
357                                "User transaction failed to finalize (attempt {}): {} ({}). Retrying ...",
358                                attempts,
359                                e,
360                                Into::<&str>::into(e.categorize())
361                            );
362                        }
363                        // Buffer the latest retriable error to be returned in case of timeout
364                        latest_retriable_error = Some(e);
365                    }
366                }
367
368                let overload = if let Some(e) = &latest_retriable_error {
369                    e.categorize() == ErrorCategory::ValidatorOverloaded
370                } else {
371                    false
372                };
373                let delay = if overload {
374                    // Increase delay during overload.
375                    const OVERLOAD_ADDITIONAL_DELAY: Duration = Duration::from_secs(10);
376                    backoff.next().unwrap() + OVERLOAD_ADDITIONAL_DELAY
377                } else {
378                    backoff.next().unwrap()
379                };
380
381                tracing::debug!("Retrying after {:.3}s", delay.as_secs_f32());
382                sleep(delay).await;
383
384                attempts += 1;
385            }
386        };
387
388        match timeout_duration {
389            Some(duration) => {
390                tokio::time::timeout(duration, retry_loop)
391                    .await
392                    .unwrap_or_else(|_| {
393                        // Timeout occurred, return with latest retriable error if available
394                        let e = TransactionDriverError::TimeoutWithLastRetriableError {
395                            last_error: latest_retriable_error.map(Box::new),
396                            attempts,
397                            timeout: duration,
398                        };
399                        if request.transaction.is_some() {
400                            tracing::info!(
401                                "User transaction timed out after {} attempts. Last error: {}",
402                                attempts,
403                                e
404                            );
405                        }
406                        Err(e)
407                    })
408            }
409            None => retry_loop.await,
410        }
411    }
412
413    #[instrument(level = "error", skip_all, err(level = "debug"))]
414    async fn drive_transaction_once(
415        &self,
416        amplification_factor: u64,
417        request: SubmitTxRequest,
418        options: &SubmitTransactionOptions,
419    ) -> Result<QuorumTransactionResponse, TransactionDriverError> {
420        let auth_agg = self.authority_aggregator.load();
421        let start_time = Instant::now();
422        let tx_type = request.tx_type();
423        let tx_digest = request.tx_digest();
424        let ping_type = request.ping_type;
425        // Pings carry no transaction, so they are neither restricted nor unrestricted.
426        let tx_class = request.transaction.as_ref().map(|tx| {
427            if tx
428                .transaction_data()
429                .expiration()
430                .restricts_proposers(auth_agg.committee.epoch())
431            {
432                TransactionClass::Restricted
433            } else {
434                TransactionClass::Unrestricted
435            }
436        });
437
438        let (name, submit_txn_result) = self
439            .submitter
440            .submit_transaction(
441                &auth_agg,
442                &self.client_monitor,
443                tx_type,
444                amplification_factor,
445                request,
446                options,
447            )
448            .await?;
449        if let SubmitTxResult::Rejected { error } = &submit_txn_result {
450            return Err(TransactionDriverError::ClientInternal {
451                error: format!(
452                    "SubmitTxResult::Rejected should have been returned as an error in submit_transaction(): {}",
453                    error
454                ),
455            });
456        }
457
458        // Wait for quorum effects using EffectsCertifier
459        let result = self
460            .certifier
461            .get_certified_finalized_effects(
462                &auth_agg,
463                &self.client_monitor,
464                tx_digest,
465                tx_type,
466                name,
467                submit_txn_result,
468                options,
469            )
470            .await;
471
472        if result.is_ok() {
473            self.client_monitor
474                .record_interaction_result(OperationFeedback {
475                    authority_name: name,
476                    display_name: auth_agg.get_display_name(&name),
477                    operation: if tx_type == TxType::SingleWriter {
478                        OperationType::SingleWriterFinality
479                    } else {
480                        OperationType::SharedObjectFinality
481                    },
482                    tx_class,
483                    ping_type,
484                    result: Ok(start_time.elapsed()),
485                });
486        }
487        result
488    }
489
490    // Runs a background task to send ping transactions to all validators to perform latency checks for the consensus path.
491    async fn run_latency_checks(self: Arc<Self>) {
492        const INTERVAL_BETWEEN_RUNS: Duration = Duration::from_secs(15);
493        const MAX_JITTER: Duration = Duration::from_secs(10);
494        const PING_REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
495
496        let mut interval = interval(INTERVAL_BETWEEN_RUNS);
497        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
498
499        loop {
500            interval.tick().await;
501
502            // Only run latency checks for shared object transactions since single writer
503            // transactions no longer use a separate fast path and go through consensus.
504            let auth_agg = self.authority_aggregator.load().clone();
505            let validators = auth_agg.committee.names().cloned().collect::<Vec<_>>();
506
507            self.metrics.latency_check_runs.inc();
508
509            let mut tasks = JoinSet::new();
510
511            for name in validators {
512                let display_name = auth_agg.get_display_name(&name);
513                let delay_ms = rand::thread_rng().gen_range(0..MAX_JITTER.as_millis()) as u64;
514                let self_clone = self.clone();
515
516                let task = async move {
517                    // Add some random delay to the task to avoid all tasks running at the same time
518                    if delay_ms > 0 {
519                        tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
520                    }
521                    let start_time = Instant::now();
522
523                    // Send a consensus ping transaction to the validator
524                    match self_clone
525                        .drive_transaction(
526                            SubmitTxRequest::new_ping(),
527                            SubmitTransactionOptions {
528                                allowed_validators: vec![display_name.clone()],
529                                ..Default::default()
530                            },
531                            Some(PING_REQUEST_TIMEOUT),
532                        )
533                        .await
534                    {
535                        Ok(_) => {
536                            tracing::debug!(
537                                "Ping transaction to validator {} completed end to end in {} seconds",
538                                display_name,
539                                start_time.elapsed().as_secs_f64()
540                            );
541                        }
542                        Err(err) => {
543                            tracing::debug!(
544                                "Failed to get certified finalized effects for ping transaction to validator {}: {}",
545                                display_name,
546                                err
547                            );
548                        }
549                    }
550                };
551
552                tasks.spawn(task);
553            }
554
555            while let Some(result) = tasks.join_next().await {
556                if let Err(e) = result {
557                    tracing::debug!("Error while driving ping transaction: {}", e);
558                }
559            }
560        }
561    }
562
563    fn enable_reconfig(
564        self: &Arc<Self>,
565        reconfig_observer: Arc<dyn ReconfigObserver<A> + Sync + Send>,
566    ) {
567        let driver = self.clone();
568        self.state.lock().tasks.spawn(monitored_future!(async move {
569            let mut reconfig_observer = reconfig_observer.clone_boxed();
570            reconfig_observer.run(driver).await;
571        }));
572    }
573}
574
575impl<A> ProposerSelector for TransactionDriver<A>
576where
577    A: AuthorityAPI + Send + Sync + 'static + Clone,
578{
579    fn preferred_proposers(&self, max: usize) -> Option<AllowedProposers> {
580        self.preferred_proposers_impl(max)
581    }
582}
583
584impl<A> AuthorityAggregatorUpdatable<A> for TransactionDriver<A>
585where
586    A: AuthorityAPI + Send + Sync + 'static + Clone,
587{
588    fn epoch(&self) -> EpochId {
589        self.authority_aggregator.load().committee.epoch
590    }
591
592    fn authority_aggregator(&self) -> Arc<AuthorityAggregator<A>> {
593        self.authority_aggregator.load_full()
594    }
595
596    fn update_authority_aggregator(&self, new_authorities: Arc<AuthorityAggregator<A>>) {
597        tracing::info!(
598            "Transaction Driver updating AuthorityAggregator with committee {}",
599            new_authorities.committee
600        );
601
602        self.authority_aggregator.store(new_authorities);
603    }
604}
605
606// Inner state of TransactionDriver.
607struct State {
608    tasks: JoinSet<()>,
609}
610
611impl State {
612    fn new() -> Self {
613        Self {
614            tasks: JoinSet::new(),
615        }
616    }
617}