1mod effects_certifier;
5mod error;
6mod metrics;
7mod reconfig_observer;
8mod request_retrier;
9mod transaction_submitter;
10
11pub use error::TransactionDriverError;
13pub use metrics::*;
14pub use reconfig_observer::{OnsiteReconfigObserver, ReconfigObserver};
15
16use std::{
17 net::SocketAddr,
18 sync::Arc,
19 time::{Duration, Instant},
20};
21
22use arc_swap::ArcSwap;
23use effects_certifier::*;
24use mysten_common::backoff::ExponentialBackoff;
25use mysten_metrics::{monitored_future, spawn_logged_monitored_task};
26use nonempty::NonEmpty;
27use parking_lot::Mutex;
28use rand::Rng;
29use request_retrier::SELECT_LATENCY_DELTA;
30use sui_config::NodeConfig;
31use sui_types::{
32 base_types::AuthorityName,
33 committee::EpochId,
34 error::{ErrorCategory, SuiErrorKind, UserInputError},
35 messages_grpc::{SubmitTxRequest, SubmitTxResult, TxType},
36 transaction::{AllowedProposers, TransactionDataAPI as _},
37 transaction_executor::ProposerSelector,
38};
39use tokio::{
40 task::JoinSet,
41 time::{interval, sleep},
42};
43use tracing::instrument;
44use transaction_submitter::*;
45
46use crate::{
47 authority::{AuthorityState, authority_per_epoch_store::check_timestamp_window},
48 authority_aggregator::AuthorityAggregator,
49 authority_client::AuthorityAPI,
50 validator_client_monitor::{
51 OperationFeedback, OperationType, TransactionClass, ValidatorClientMetrics,
52 ValidatorClientMonitor,
53 },
54};
55
56#[cfg(test)]
57#[path = "unit_tests/proposer_selector_tests.rs"]
58mod proposer_selector_tests;
59
60pub trait AuthorityAggregatorUpdatable<A: Clone>: Send + Sync + 'static {
63 fn epoch(&self) -> EpochId;
64 fn authority_aggregator(&self) -> Arc<AuthorityAggregator<A>>;
65 fn update_authority_aggregator(&self, new_authorities: Arc<AuthorityAggregator<A>>);
66}
67
68#[derive(Clone, Default, Debug)]
70pub struct SubmitTransactionOptions {
71 pub forwarded_client_addr: Option<SocketAddr>,
74
75 pub allowed_validators: Vec<String>,
78
79 pub blocked_validators: Vec<String>,
82}
83
84#[derive(Clone, Debug)]
85pub struct QuorumTransactionResponse {
86 pub effects: sui_types::transaction_driver_types::FinalizedEffects,
87
88 pub events: Option<sui_types::effects::TransactionEvents>,
89 pub input_objects: Option<Vec<sui_types::object::Object>>,
91 pub output_objects: Option<Vec<sui_types::object::Object>>,
93 pub auxiliary_data: Option<Vec<u8>>,
94}
95
96pub struct TransactionDriver<A: Clone> {
97 authority_aggregator: Arc<ArcSwap<AuthorityAggregator<A>>>,
98 state: Mutex<State>,
99 metrics: Arc<TransactionDriverMetrics>,
100 submitter: TransactionSubmitter,
101 certifier: EffectsCertifier,
102 client_monitor: Arc<ValidatorClientMonitor<A>>,
103}
104
105impl<A> TransactionDriver<A>
106where
107 A: AuthorityAPI + Send + Sync + 'static + Clone,
108{
109 pub fn new(
111 authority_aggregator: Arc<AuthorityAggregator<A>>,
112 reconfig_observer: Arc<dyn ReconfigObserver<A> + Sync + Send>,
113 metrics: Arc<TransactionDriverMetrics>,
114 node_config: Option<&NodeConfig>,
115 client_metrics: Arc<ValidatorClientMetrics>,
116 ) -> Arc<Self> {
117 if std::env::var("TRANSACTION_DRIVER").is_ok() {
118 tracing::warn!(
119 "Transaction Driver is the only supported driver for transaction submission. Setting TRANSACTION_DRIVER is a no-op."
120 );
121 }
122
123 let shared_swap = Arc::new(ArcSwap::new(authority_aggregator));
124
125 let monitor_config = node_config
127 .and_then(|nc| nc.validator_client_monitor_config.clone())
128 .unwrap_or_default();
129 let client_monitor =
130 ValidatorClientMonitor::new(monitor_config, client_metrics, shared_swap.clone());
131
132 let driver = Arc::new(Self {
133 authority_aggregator: shared_swap,
134 state: Mutex::new(State::new()),
135 metrics: metrics.clone(),
136 submitter: TransactionSubmitter::new(metrics.clone()),
137 certifier: EffectsCertifier::new(metrics),
138 client_monitor,
139 });
140
141 let driver_clone = driver.clone();
142
143 spawn_logged_monitored_task!(Self::run_latency_checks(driver_clone));
144
145 driver.enable_reconfig(reconfig_observer);
146 driver
147 }
148
149 pub fn authority_aggregator(&self) -> &Arc<ArcSwap<AuthorityAggregator<A>>> {
151 &self.authority_aggregator
152 }
153
154 pub fn select_preferred_validators(
155 &self,
156 delta: f64,
157 tx_class: TransactionClass,
158 ) -> Vec<AuthorityName> {
159 let authority_aggregator = self.authority_aggregator.load();
160 self.client_monitor.select_shuffled_preferred_validators(
161 &authority_aggregator.committee,
162 delta,
163 tx_class,
164 )
165 }
166
167 fn preferred_proposers_impl(&self, max: usize) -> Option<AllowedProposers> {
172 if !self.client_monitor.has_observed_latencies() {
175 return None;
176 }
177
178 let authority_aggregator = self.authority_aggregator.load();
179 let committee = &authority_aggregator.committee;
180 let mut proposers: Vec<u32> = self
183 .client_monitor
184 .select_shuffled_preferred_validators(
185 committee,
186 SELECT_LATENCY_DELTA,
187 TransactionClass::Restricted,
188 )
189 .into_iter()
190 .filter_map(|name| committee.authority_index(&name))
191 .take(max)
192 .collect();
193 proposers.sort_unstable();
195 proposers.dedup();
196
197 Some(AllowedProposers {
198 epoch: committee.epoch(),
199 proposers: NonEmpty::from_vec(proposers)?,
200 })
201 }
202
203 #[instrument(level = "error", skip_all, fields(tx_digest = ?request.transaction.as_ref().map(|t| t.digest()), ping = %request.ping_type.is_some()))]
210 pub async fn drive_transaction(
211 &self,
212 request: SubmitTxRequest,
213 options: SubmitTransactionOptions,
214 timeout_duration: Option<Duration>,
215 ) -> Result<QuorumTransactionResponse, TransactionDriverError> {
216 const MAX_DRIVE_TRANSACTION_RETRY_DELAY: Duration = Duration::from_secs(10);
217
218 let tx_data = request.transaction.as_ref().map(|t| t.transaction_data());
219 let amplification_factor =
221 if request.ping_type.is_some() || tx_data.is_some_and(|d| d.is_gasless_transaction()) {
222 1
223 } else {
224 let tx_data = tx_data.unwrap();
225 let gas_price = tx_data.gas_price();
226 let reference_gas_price = self.authority_aggregator.load().reference_gas_price;
227 let amplification_factor = gas_price / reference_gas_price.max(1);
228 if amplification_factor == 0 {
229 return Err(TransactionDriverError::ValidationFailed {
230 error: UserInputError::GasPriceUnderRGP {
231 gas_price,
232 reference_gas_price,
233 }
234 .to_string(),
235 });
236 }
237 amplification_factor
238 };
239
240 let skip_timestamp_check = mysten_common::in_test_configuration()
243 && mysten_common::random::get_rng().gen_bool(0.5);
244 let mut not_yet_valid_wait = None;
245 if let Some(tx_data) = tx_data
246 && !skip_timestamp_check
247 {
248 let expiration = tx_data.expiration();
249 let now_ms = AuthorityState::unixtime_now_ms();
250 match check_timestamp_window(expiration, now_ms) {
251 Ok(()) => {}
252 Err(error)
257 if matches!(
258 error.as_inner(),
259 SuiErrorKind::TransactionNotYetValid { .. }
260 ) =>
261 {
262 let valid_from_ms = expiration.first_valid_timestamp_ms().unwrap_or(now_ms);
263 let wait = Duration::from_millis(valid_from_ms.saturating_sub(now_ms));
264 if timeout_duration.is_some_and(|timeout| wait >= timeout) {
265 return Err(TransactionDriverError::ValidationFailed {
266 error: error.to_string(),
267 });
268 }
269 not_yet_valid_wait = Some(wait);
270 }
271 Err(error) => {
272 return Err(TransactionDriverError::ValidationFailed {
273 error: error.to_string(),
274 });
275 }
276 }
277 }
278
279 let tx_type = request.tx_type();
280 let ping_label = if request.ping_type.is_some() {
281 "true"
282 } else {
283 "false"
284 };
285 let mut backoff = ExponentialBackoff::new(
286 Duration::from_millis(100),
287 MAX_DRIVE_TRANSACTION_RETRY_DELAY,
288 );
289 let mut attempts = 0;
290 let mut latest_retriable_error = None;
291
292 let retry_loop = async {
293 if let Some(wait) = not_yet_valid_wait {
294 sleep(wait).await;
295 }
296 let timer = Instant::now();
299 self.metrics
300 .total_transactions_submitted
301 .with_label_values(&[tx_type.as_str(), ping_label])
302 .inc();
303 loop {
304 match self
306 .drive_transaction_once(amplification_factor, request.clone(), &options)
307 .await
308 {
309 Ok(resp) => {
310 let settlement_finality_latency = timer.elapsed().as_secs_f64();
311 self.metrics
312 .settlement_finality_latency
313 .with_label_values(&[tx_type.as_str(), ping_label])
314 .observe(settlement_finality_latency);
315 let is_out_of_expected_range = settlement_finality_latency >= 8.0
316 || settlement_finality_latency <= 0.1;
317 tracing::debug!(
318 ?tx_type,
319 ?is_out_of_expected_range,
320 "Settlement finality latency: {:.3} seconds",
321 settlement_finality_latency
322 );
323 self.metrics
325 .transaction_retries
326 .with_label_values(&["success", tx_type.as_str(), ping_label])
327 .observe(attempts as f64);
328 return Ok(resp);
329 }
330 Err(e) => {
331 self.metrics
332 .drive_transaction_errors
333 .with_label_values(&[
334 e.categorize().into(),
335 tx_type.as_str(),
336 ping_label,
337 ])
338 .inc();
339 if !e.is_submission_retriable() {
340 self.metrics
342 .transaction_retries
343 .with_label_values(&["failure", tx_type.as_str(), ping_label])
344 .observe(attempts as f64);
345 if request.transaction.is_some() {
346 tracing::info!(
347 "User transaction failed to finalize (attempt {}), with non-retriable error: {} ({})",
348 attempts,
349 e,
350 Into::<&str>::into(e.categorize())
351 );
352 }
353 return Err(e);
354 }
355 if request.transaction.is_some() {
356 tracing::info!(
357 "User transaction failed to finalize (attempt {}): {} ({}). Retrying ...",
358 attempts,
359 e,
360 Into::<&str>::into(e.categorize())
361 );
362 }
363 latest_retriable_error = Some(e);
365 }
366 }
367
368 let overload = if let Some(e) = &latest_retriable_error {
369 e.categorize() == ErrorCategory::ValidatorOverloaded
370 } else {
371 false
372 };
373 let delay = if overload {
374 const OVERLOAD_ADDITIONAL_DELAY: Duration = Duration::from_secs(10);
376 backoff.next().unwrap() + OVERLOAD_ADDITIONAL_DELAY
377 } else {
378 backoff.next().unwrap()
379 };
380
381 tracing::debug!("Retrying after {:.3}s", delay.as_secs_f32());
382 sleep(delay).await;
383
384 attempts += 1;
385 }
386 };
387
388 match timeout_duration {
389 Some(duration) => {
390 tokio::time::timeout(duration, retry_loop)
391 .await
392 .unwrap_or_else(|_| {
393 let e = TransactionDriverError::TimeoutWithLastRetriableError {
395 last_error: latest_retriable_error.map(Box::new),
396 attempts,
397 timeout: duration,
398 };
399 if request.transaction.is_some() {
400 tracing::info!(
401 "User transaction timed out after {} attempts. Last error: {}",
402 attempts,
403 e
404 );
405 }
406 Err(e)
407 })
408 }
409 None => retry_loop.await,
410 }
411 }
412
413 #[instrument(level = "error", skip_all, err(level = "debug"))]
414 async fn drive_transaction_once(
415 &self,
416 amplification_factor: u64,
417 request: SubmitTxRequest,
418 options: &SubmitTransactionOptions,
419 ) -> Result<QuorumTransactionResponse, TransactionDriverError> {
420 let auth_agg = self.authority_aggregator.load();
421 let start_time = Instant::now();
422 let tx_type = request.tx_type();
423 let tx_digest = request.tx_digest();
424 let ping_type = request.ping_type;
425 let tx_class = request.transaction.as_ref().map(|tx| {
427 if tx
428 .transaction_data()
429 .expiration()
430 .restricts_proposers(auth_agg.committee.epoch())
431 {
432 TransactionClass::Restricted
433 } else {
434 TransactionClass::Unrestricted
435 }
436 });
437
438 let (name, submit_txn_result) = self
439 .submitter
440 .submit_transaction(
441 &auth_agg,
442 &self.client_monitor,
443 tx_type,
444 amplification_factor,
445 request,
446 options,
447 )
448 .await?;
449 if let SubmitTxResult::Rejected { error } = &submit_txn_result {
450 return Err(TransactionDriverError::ClientInternal {
451 error: format!(
452 "SubmitTxResult::Rejected should have been returned as an error in submit_transaction(): {}",
453 error
454 ),
455 });
456 }
457
458 let result = self
460 .certifier
461 .get_certified_finalized_effects(
462 &auth_agg,
463 &self.client_monitor,
464 tx_digest,
465 tx_type,
466 name,
467 submit_txn_result,
468 options,
469 )
470 .await;
471
472 if result.is_ok() {
473 self.client_monitor
474 .record_interaction_result(OperationFeedback {
475 authority_name: name,
476 display_name: auth_agg.get_display_name(&name),
477 operation: if tx_type == TxType::SingleWriter {
478 OperationType::SingleWriterFinality
479 } else {
480 OperationType::SharedObjectFinality
481 },
482 tx_class,
483 ping_type,
484 result: Ok(start_time.elapsed()),
485 });
486 }
487 result
488 }
489
490 async fn run_latency_checks(self: Arc<Self>) {
492 const INTERVAL_BETWEEN_RUNS: Duration = Duration::from_secs(15);
493 const MAX_JITTER: Duration = Duration::from_secs(10);
494 const PING_REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
495
496 let mut interval = interval(INTERVAL_BETWEEN_RUNS);
497 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
498
499 loop {
500 interval.tick().await;
501
502 let auth_agg = self.authority_aggregator.load().clone();
505 let validators = auth_agg.committee.names().cloned().collect::<Vec<_>>();
506
507 self.metrics.latency_check_runs.inc();
508
509 let mut tasks = JoinSet::new();
510
511 for name in validators {
512 let display_name = auth_agg.get_display_name(&name);
513 let delay_ms = rand::thread_rng().gen_range(0..MAX_JITTER.as_millis()) as u64;
514 let self_clone = self.clone();
515
516 let task = async move {
517 if delay_ms > 0 {
519 tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
520 }
521 let start_time = Instant::now();
522
523 match self_clone
525 .drive_transaction(
526 SubmitTxRequest::new_ping(),
527 SubmitTransactionOptions {
528 allowed_validators: vec![display_name.clone()],
529 ..Default::default()
530 },
531 Some(PING_REQUEST_TIMEOUT),
532 )
533 .await
534 {
535 Ok(_) => {
536 tracing::debug!(
537 "Ping transaction to validator {} completed end to end in {} seconds",
538 display_name,
539 start_time.elapsed().as_secs_f64()
540 );
541 }
542 Err(err) => {
543 tracing::debug!(
544 "Failed to get certified finalized effects for ping transaction to validator {}: {}",
545 display_name,
546 err
547 );
548 }
549 }
550 };
551
552 tasks.spawn(task);
553 }
554
555 while let Some(result) = tasks.join_next().await {
556 if let Err(e) = result {
557 tracing::debug!("Error while driving ping transaction: {}", e);
558 }
559 }
560 }
561 }
562
563 fn enable_reconfig(
564 self: &Arc<Self>,
565 reconfig_observer: Arc<dyn ReconfigObserver<A> + Sync + Send>,
566 ) {
567 let driver = self.clone();
568 self.state.lock().tasks.spawn(monitored_future!(async move {
569 let mut reconfig_observer = reconfig_observer.clone_boxed();
570 reconfig_observer.run(driver).await;
571 }));
572 }
573}
574
575impl<A> ProposerSelector for TransactionDriver<A>
576where
577 A: AuthorityAPI + Send + Sync + 'static + Clone,
578{
579 fn preferred_proposers(&self, max: usize) -> Option<AllowedProposers> {
580 self.preferred_proposers_impl(max)
581 }
582}
583
584impl<A> AuthorityAggregatorUpdatable<A> for TransactionDriver<A>
585where
586 A: AuthorityAPI + Send + Sync + 'static + Clone,
587{
588 fn epoch(&self) -> EpochId {
589 self.authority_aggregator.load().committee.epoch
590 }
591
592 fn authority_aggregator(&self) -> Arc<AuthorityAggregator<A>> {
593 self.authority_aggregator.load_full()
594 }
595
596 fn update_authority_aggregator(&self, new_authorities: Arc<AuthorityAggregator<A>>) {
597 tracing::info!(
598 "Transaction Driver updating AuthorityAggregator with committee {}",
599 new_authorities.committee
600 );
601
602 self.authority_aggregator.store(new_authorities);
603 }
604}
605
606struct State {
608 tasks: JoinSet<()>,
609}
610
611impl State {
612 fn new() -> Self {
613 Self {
614 tasks: JoinSet::new(),
615 }
616 }
617}