1use std::time::Instant;
25
26use mysten_common::ZipDebugEqIteratorExt;
27use sui_inverted_index::BitmapQuery;
28use sui_inverted_index::ScanDirection;
29use sui_rpc::field::FieldMaskTree;
30use sui_rpc::merge::Merge;
31use sui_rpc::proto::sui::rpc::v2::Checkpoint;
32use sui_rpc::proto::sui::rpc::v2::Event as ProtoEvent;
33use sui_rpc::proto::sui::rpc::v2::EventFilter;
34use sui_rpc::proto::sui::rpc::v2::ExecutedTransaction;
35use sui_rpc::proto::sui::rpc::v2::ObjectSet as ProtoObjectSet;
36use sui_rpc::proto::sui::rpc::v2::SubscribeCheckpointsRequest;
37use sui_rpc::proto::sui::rpc::v2::SubscribeCheckpointsResponse;
38use sui_rpc::proto::sui::rpc::v2::SubscribeEventsRequest;
39use sui_rpc::proto::sui::rpc::v2::SubscribeEventsResponse;
40use sui_rpc::proto::sui::rpc::v2::SubscribeTransactionsRequest;
41use sui_rpc::proto::sui::rpc::v2::SubscribeTransactionsResponse;
42use sui_rpc::proto::sui::rpc::v2::TransactionFilter;
43use sui_rpc::proto::sui::rpc::v2::Watermark;
44use sui_rpc::proto::sui::rpc::v2::subscription_service_server::SubscriptionService;
45use sui_types::balance_change::derive_balance_changes_2;
46use sui_types::effects::TransactionEffectsAPI;
47use tokio::sync::mpsc;
48use tonic::codegen::BoxStream;
49
50use crate::RpcError;
51use crate::RpcService;
52use crate::grpc::v2::render::RenderContext;
53use crate::ledger_history::cursor::LedgerPoint;
54use crate::ledger_history::cursor::Position;
55use crate::ledger_history::filter::event_filter_to_query;
56use crate::ledger_history::filter::transaction_filter_to_query;
57use crate::ledger_history::query_options::QueryOptions;
58use crate::ledger_history::watermark::advance_covered_bound_before_checkpoint;
59use crate::ledger_history::watermark::boundary_watermark;
60use crate::ledger_history::watermark::item_watermark;
61use crate::ledger_history::watermark::merge_covered_checkpoint_bound;
62use crate::metrics::SubscriptionFrameKind;
63use crate::metrics::SubscriptionStreamMetrics;
64use crate::read_mask_defaults;
65use crate::subscription::SubscriptionCheckpoint;
66use crate::subscription::SubscriptionKind;
67use crate::subscription::SubscriptionSpec;
68use crate::subscription::SubscriptionUpdate;
69
70#[tonic::async_trait]
71impl SubscriptionService for RpcService {
72 async fn subscribe_checkpoints(
73 &self,
74 request: tonic::Request<SubscribeCheckpointsRequest>,
75 ) -> Result<tonic::Response<BoxStream<SubscribeCheckpointsResponse>>, tonic::Status> {
76 let request = request.into_inner();
77 let read_mask = read_mask_defaults::validate_read_mask::<Checkpoint>(
78 request.read_mask,
79 read_mask_defaults::CHECKPOINT,
80 )?;
81 let query = compile_transaction_filter(self, request.filter.as_ref())?;
82 let (mut receiver, stream_metrics) = register(
83 self,
84 SubscriptionSpec {
85 kind: SubscriptionKind::Checkpoints,
86 query,
87 },
88 )
89 .await?;
90
91 let service = self.clone();
92 let response = Box::pin(async_stream::stream! {
93 while let Some(update) = receiver.recv().await {
94 let mut response = SubscribeCheckpointsResponse::default();
95 let frame_kind = match update {
96 SubscriptionUpdate::Matched(matched) => {
97 let cp = matched.checkpoint.summary().sequence_number;
98 response.cursor = Some(cp);
99 response.checkpoint = Some(render_checkpoint_message(
100 &service,
101 &matched.checkpoint,
102 &read_mask,
103 ));
104 SubscriptionFrameKind::Payload
105 }
106 SubscriptionUpdate::WatermarkTick { checkpoint: cp, .. } => {
107 response.cursor = Some(cp);
108 SubscriptionFrameKind::Watermark
109 }
110 };
111 stream_metrics.observe_frame(&response, frame_kind);
112 let yielded_at = Instant::now();
113 yield Ok(response);
114 stream_metrics.observe_yield_wait(yielded_at.elapsed());
115 }
116 });
117
118 Ok(tonic::Response::new(response))
119 }
120
121 async fn subscribe_transactions(
122 &self,
123 request: tonic::Request<SubscribeTransactionsRequest>,
124 ) -> Result<tonic::Response<BoxStream<SubscribeTransactionsResponse>>, tonic::Status> {
125 let request = request.into_inner();
126 let read_mask = read_mask_defaults::validate_read_mask::<ExecutedTransaction>(
127 request.read_mask,
128 read_mask_defaults::TRANSACTION,
129 )?;
130 let query = compile_transaction_filter(self, request.filter.as_ref())?;
131 let (mut receiver, stream_metrics) = register(
132 self,
133 SubscriptionSpec {
134 kind: SubscriptionKind::Transactions,
135 query,
136 },
137 )
138 .await?;
139
140 let service = self.clone();
141 let options = QueryOptions::subscription();
142 let response = Box::pin(async_stream::stream! {
143 let mut boundary: Option<u64> = None;
144 let mut entry_checkpoint = None;
145 while let Some(update) = receiver.recv().await {
146 match update {
147 SubscriptionUpdate::Matched(matched) => {
148 let checkpoint = &matched.checkpoint;
149 let Some(indices) = matched
150 .matches
151 .transaction_indices(checkpoint.transactions().len() as u32)
152 else {
153 continue;
154 };
155 let cp = checkpoint.summary().sequence_number;
156 let entry_checkpoint = *entry_checkpoint.get_or_insert(cp);
157 let tx_hi = checkpoint.summary().network_total_transactions;
158 let tx_lo = tx_hi - checkpoint.transactions().len() as u64;
159 for i in indices {
160 boundary = advance_covered_bound_before_checkpoint(
163 boundary,
164 cp,
165 entry_checkpoint,
166 &options,
167 );
168 let mut response = SubscribeTransactionsResponse::default();
169 response.transaction = Some(render_transaction_message(
170 &service,
171 checkpoint,
172 i as usize,
173 &read_mask,
174 ));
175 response.watermark = Some(transaction_item_watermark(
176 cp,
177 tx_lo,
178 i as u64,
179 boundary,
180 ));
181 stream_metrics.observe_frame(
182 &response,
183 SubscriptionFrameKind::Payload,
184 );
185 let yielded_at = Instant::now();
186 yield Ok(response);
187 stream_metrics.observe_yield_wait(yielded_at.elapsed());
188 }
189 }
190 SubscriptionUpdate::WatermarkTick { checkpoint: cp, tx_hi } => {
191 record_watermark_tick_coverage(
192 &mut boundary,
193 &mut entry_checkpoint,
194 cp,
195 &options,
196 );
197 let mut response = SubscribeTransactionsResponse::default();
198 response.watermark = Some(transaction_tick_watermark(cp, tx_hi, boundary));
199 stream_metrics.observe_frame(
200 &response,
201 SubscriptionFrameKind::Watermark,
202 );
203 let yielded_at = Instant::now();
204 yield Ok(response);
205 stream_metrics.observe_yield_wait(yielded_at.elapsed());
206 }
207 }
208 }
209 });
210
211 Ok(tonic::Response::new(response))
212 }
213
214 async fn subscribe_events(
215 &self,
216 request: tonic::Request<SubscribeEventsRequest>,
217 ) -> Result<tonic::Response<BoxStream<SubscribeEventsResponse>>, tonic::Status> {
218 let request = request.into_inner();
219 let read_mask = read_mask_defaults::validate_read_mask::<ProtoEvent>(
220 request.read_mask,
221 read_mask_defaults::EVENT,
222 )?;
223 let query = compile_event_filter(self, request.filter.as_ref())?;
224 let (mut receiver, stream_metrics) = register(
225 self,
226 SubscriptionSpec {
227 kind: SubscriptionKind::Events,
228 query,
229 },
230 )
231 .await?;
232
233 let service = self.clone();
234 let options = QueryOptions::subscription();
235 let response = Box::pin(async_stream::stream! {
236 let mut boundary: Option<u64> = None;
237 let mut entry_checkpoint = None;
238 while let Some(update) = receiver.recv().await {
239 match update {
240 SubscriptionUpdate::Matched(matched) => {
241 let checkpoint = &matched.checkpoint;
242 let Some(pairs) = matched.matches.event_indices(checkpoint) else {
243 continue;
244 };
245 let cp = checkpoint.summary().sequence_number;
246 let entry_checkpoint = *entry_checkpoint.get_or_insert(cp);
247 let tx_hi = checkpoint.summary().network_total_transactions;
248 let tx_lo = tx_hi - checkpoint.transactions().len() as u64;
249 for (tx_idx, ev) in pairs {
250 let tx = &checkpoint.transactions()[tx_idx as usize];
251 boundary = advance_covered_bound_before_checkpoint(
252 boundary,
253 cp,
254 entry_checkpoint,
255 &options,
256 );
257 let event = &tx
258 .events
259 .as_ref()
260 .expect("matched event implies events")
261 .data[ev as usize];
262 let render_context = RenderContext::new(
263 service.reader.inner(),
264 Some(checkpoint.object_set()),
265 );
266 let mut proto_event =
267 service.render_event_to_proto(event, &read_mask, &render_context);
268 if read_mask.contains(ProtoEvent::CHECKPOINT_FIELD.name) {
269 proto_event.checkpoint = Some(cp);
270 }
271 if read_mask.contains(ProtoEvent::TRANSACTION_DIGEST_FIELD.name) {
272 proto_event.transaction_digest =
273 Some(tx.effects.transaction_digest().base58_encode());
274 }
275 if read_mask.contains(ProtoEvent::TRANSACTION_INDEX_FIELD.name) {
276 proto_event.transaction_index = Some(tx_idx as u64);
277 }
278 if read_mask.contains(ProtoEvent::EVENT_INDEX_FIELD.name) {
279 proto_event.event_index = Some(ev);
280 }
281 let mut response = SubscribeEventsResponse::default();
282 response.event = Some(proto_event);
283 response.watermark = Some(event_item_watermark(
284 cp,
285 tx_lo,
286 tx_idx as u64,
287 ev,
288 boundary,
289 ));
290 stream_metrics.observe_frame(
291 &response,
292 SubscriptionFrameKind::Payload,
293 );
294 let yielded_at = Instant::now();
295 yield Ok(response);
296 stream_metrics.observe_yield_wait(yielded_at.elapsed());
297 }
298 }
299 SubscriptionUpdate::WatermarkTick { checkpoint: cp, tx_hi } => {
300 record_watermark_tick_coverage(
301 &mut boundary,
302 &mut entry_checkpoint,
303 cp,
304 &options,
305 );
306 let mut response = SubscribeEventsResponse::default();
307 response.watermark = Some(event_tick_watermark(cp, tx_hi, boundary));
308 stream_metrics.observe_frame(
309 &response,
310 SubscriptionFrameKind::Watermark,
311 );
312 let yielded_at = Instant::now();
313 yield Ok(response);
314 stream_metrics.observe_yield_wait(yielded_at.elapsed());
315 }
316 }
317 }
318 });
319
320 Ok(tonic::Response::new(response))
321 }
322}
323
324fn transaction_item_watermark(
328 cp: u64,
329 tx_lo: u64,
330 transaction_index: u64,
331 boundary: Option<u64>,
332) -> Watermark {
333 item_watermark(
334 Position::Transactions {
335 tx_seq: tx_lo + transaction_index,
336 public_position: LedgerPoint::transaction(cp, transaction_index),
337 },
338 ScanDirection::Ascending,
339 boundary,
340 )
341}
342
343fn event_item_watermark(
346 cp: u64,
347 tx_lo: u64,
348 transaction_index: u64,
349 event_index: u32,
350 boundary: Option<u64>,
351) -> Watermark {
352 item_watermark(
353 Position::Events {
354 tx_seq: tx_lo + transaction_index,
355 event_index,
356 public_position: LedgerPoint::event(cp, transaction_index, event_index),
357 },
358 ScanDirection::Ascending,
359 boundary,
360 )
361}
362
363fn transaction_tick_watermark(cp: u64, tx_hi: u64, boundary: Option<u64>) -> Watermark {
366 boundary_watermark(
367 Position::Transactions {
368 tx_seq: tx_hi,
369 public_position: LedgerPoint::checkpoint_start(cp + 1),
370 },
371 boundary,
372 )
373}
374
375fn event_tick_watermark(cp: u64, tx_hi: u64, boundary: Option<u64>) -> Watermark {
377 boundary_watermark(
378 Position::Events {
379 tx_seq: tx_hi,
380 event_index: 0,
381 public_position: LedgerPoint::checkpoint_start(cp + 1),
382 },
383 boundary,
384 )
385}
386
387fn render_checkpoint_message(
392 service: &RpcService,
393 checkpoint: &SubscriptionCheckpoint,
394 read_mask: &FieldMaskTree,
395) -> Checkpoint {
396 let mut checkpoint_message = match checkpoint {
397 SubscriptionCheckpoint::Certified(checkpoint) => {
398 Checkpoint::merge_from(checkpoint.as_ref(), read_mask)
399 }
400 SubscriptionCheckpoint::LocallyBuilt(checkpoint) => {
401 Checkpoint::merge_from(checkpoint.as_ref(), read_mask)
402 }
403 };
404 let object_set = checkpoint.object_set();
405
406 if let Some(tx_mask) = read_mask.subtree(Checkpoint::TRANSACTIONS_FIELD) {
407 let mut json_budget = service.config.max_json_move_value_response_size();
409 let render_context = RenderContext::new(service.reader.inner(), Some(object_set));
410 for (message, tx) in checkpoint_message
411 .transactions_mut()
412 .iter_mut()
413 .zip_debug_eq(checkpoint.transactions().iter())
414 {
415 service.annotate_checkpoint_transaction(
416 message,
417 tx,
418 object_set,
419 &tx_mask,
420 &mut json_budget,
421 &render_context,
422 );
423 }
424 }
425
426 if read_mask.contains("transactions.balance_changes") {
427 for (txn, effects) in checkpoint_message
428 .transactions_mut()
429 .iter_mut()
430 .zip_debug_eq(checkpoint.transactions().iter().map(|t| &t.effects))
431 {
432 *txn.balance_changes_mut() = derive_balance_changes_2(effects, object_set)
433 .into_iter()
434 .map(Into::into)
435 .collect();
436 }
437 }
438
439 checkpoint_message
440}
441
442fn transaction_objects<'a>(
443 transaction: &'a sui_types::full_checkpoint_content::ExecutedTransaction,
444 checkpoint_objects: &'a sui_types::full_checkpoint_content::ObjectSet,
445) -> impl Iterator<Item = &'a sui_types::object::Object> + 'a {
446 sui_types::storage::get_transaction_object_set(
447 &transaction.transaction,
448 &transaction.effects,
449 &transaction.unchanged_loaded_runtime_objects,
450 )
451 .into_iter()
452 .filter_map(move |key| checkpoint_objects.get(&key))
453}
454
455fn render_transaction_message(
460 service: &RpcService,
461 checkpoint: &SubscriptionCheckpoint,
462 index: usize,
463 read_mask: &FieldMaskTree,
464) -> ExecutedTransaction {
465 let object_set = checkpoint.object_set();
466 let tx = &checkpoint.transactions()[index];
467 let mut message = ExecutedTransaction::merge_from(tx, read_mask);
468 let mut json_budget = service.config.max_json_move_value_response_size();
476 let render_context = RenderContext::new(service.reader.inner(), Some(object_set));
477 service.annotate_checkpoint_transaction(
478 &mut message,
479 tx,
480 object_set,
481 read_mask,
482 &mut json_budget,
483 &render_context,
484 );
485
486 if let Some(object_mask) = read_mask
487 .subtree(ExecutedTransaction::OBJECTS_FIELD)
488 .and_then(|submask| submask.subtree(ProtoObjectSet::OBJECTS_FIELD))
489 {
490 message.objects = Some(
491 ProtoObjectSet::default().with_objects(
492 transaction_objects(tx, object_set)
493 .map(|object| {
494 service.render_object_to_proto(object, &object_mask, &render_context)
495 })
496 .collect(),
497 ),
498 );
499 }
500
501 if read_mask.contains(ExecutedTransaction::CHECKPOINT_FIELD) {
502 message.checkpoint = Some(checkpoint.summary().sequence_number);
503 }
504 if read_mask.contains(ExecutedTransaction::TIMESTAMP_FIELD) {
505 message.timestamp = Some(sui_rpc::proto::timestamp_ms_to_proto(
506 checkpoint.summary().timestamp_ms,
507 ));
508 }
509 if read_mask.contains(ExecutedTransaction::BALANCE_CHANGES_FIELD) {
510 message.balance_changes = derive_balance_changes_2(&tx.effects, object_set)
511 .into_iter()
512 .map(Into::into)
513 .collect();
514 }
515 if read_mask.contains(ExecutedTransaction::TRANSACTION_INDEX_FIELD) {
516 message.transaction_index = Some(index as u64);
517 }
518
519 message
520}
521
522fn record_watermark_tick_coverage(
527 covered_checkpoint_bound: &mut Option<u64>,
528 entry_checkpoint: &mut Option<u64>,
529 checkpoint: u64,
530 options: &QueryOptions,
531) {
532 if entry_checkpoint.is_none() {
533 *entry_checkpoint = Some(checkpoint.saturating_add(1));
534 }
535 *covered_checkpoint_bound =
536 merge_covered_checkpoint_bound(*covered_checkpoint_bound, checkpoint, options);
537}
538
539fn compile_transaction_filter(
540 service: &RpcService,
541 filter: Option<&TransactionFilter>,
542) -> Result<Option<BitmapQuery>, RpcError> {
543 let max_literals = service.config.ledger_history().max_bitmap_filter_literals();
544 filter
545 .map(|filter| transaction_filter_to_query(filter, max_literals))
546 .transpose()
547}
548
549fn compile_event_filter(
550 service: &RpcService,
551 filter: Option<&EventFilter>,
552) -> Result<Option<BitmapQuery>, RpcError> {
553 let max_literals = service.config.ledger_history().max_bitmap_filter_literals();
554 filter
555 .map(|filter| event_filter_to_query(filter, max_literals))
556 .transpose()
557}
558
559async fn register(
560 service: &RpcService,
561 spec: SubscriptionSpec,
562) -> Result<
563 (
564 mpsc::Receiver<SubscriptionUpdate>,
565 SubscriptionStreamMetrics,
566 ),
567 tonic::Status,
568> {
569 let handle = service
570 .subscription_service_handle
571 .as_ref()
572 .ok_or_else(|| tonic::Status::unimplemented("subscription service not enabled"))?;
573 let kind = spec.kind;
574 let receiver = handle
575 .register_subscription(spec)
576 .await
577 .ok_or_else(|| tonic::Status::unavailable("subscription service is unavailable"))?;
578 let stream_metrics = handle.stream_metrics(kind);
579 Ok((receiver, stream_metrics))
580}
581
582#[cfg(test)]
583mod tests {
584 use sui_types::test_checkpoint_data_builder::TestCheckpointBuilder;
585
586 use super::*;
587 use crate::ledger_history::cursor::CursorToken;
588
589 #[test]
590 fn initial_watermark_tick_covers_checkpoint_before_entry() {
591 let options = QueryOptions::subscription();
592 let mut covered_checkpoint_bound = None;
593 let mut entry_checkpoint = None;
594
595 record_watermark_tick_coverage(
596 &mut covered_checkpoint_bound,
597 &mut entry_checkpoint,
598 41,
599 &options,
600 );
601 let watermark = transaction_tick_watermark(41, 100, covered_checkpoint_bound);
602
603 assert_eq!(entry_checkpoint, Some(42));
604 assert_eq!(watermark.checkpoint, Some(41));
605 let point = LedgerPoint::checkpoint_start(42);
606 let cursor = Position::Transactions {
607 tx_seq: 100,
608 public_position: point,
609 };
610 assert_eq!(
611 watermark.cursor,
612 Some(CursorToken::boundary(cursor).encode())
613 );
614 assert_eq!(
615 watermark.cursor_position,
616 Some(point.to_ledger_position(&cursor))
617 );
618 }
619
620 #[test]
623 fn subscription_frames_carry_ledger_positions() {
624 let transaction_frames = [
626 transaction_item_watermark(7, 30, 0, None),
627 transaction_item_watermark(7, 30, 2, None),
628 transaction_tick_watermark(7, 33, Some(7)),
629 transaction_tick_watermark(8, 35, Some(8)),
630 ];
631 let event_frames = [
632 event_item_watermark(7, 30, 1, 0, None),
633 event_item_watermark(7, 30, 1, 1, None),
634 event_tick_watermark(7, 33, Some(7)),
635 event_item_watermark(9, 35, 0, 3, Some(8)),
636 ];
637 let positions = |frames: &[Watermark]| -> Vec<(u64, u64, u32)> {
638 frames
639 .iter()
640 .map(|watermark| {
641 let position = watermark.cursor_position.expect("every frame is placed");
642 (
643 position.checkpoint(),
644 position.transaction_index(),
645 position.event_index(),
646 )
647 })
648 .collect()
649 };
650 assert_eq!(
651 positions(&transaction_frames),
652 vec![(7, 1, 0), (7, 3, 0), (8, 0, 0), (9, 0, 0)]
653 );
654 assert_eq!(
655 positions(&event_frames),
656 vec![(7, 1, 1), (7, 1, 2), (8, 0, 0), (9, 0, 4)]
657 );
658 let cursor = |watermark: &Watermark| {
661 CursorToken::decode(watermark.cursor.as_ref().unwrap()).unwrap()
662 };
663 assert_eq!(
664 cursor(&transaction_frames[0]),
665 CursorToken::item(Position::Transactions {
666 tx_seq: 30,
667 public_position: LedgerPoint::transaction(7, 0),
668 })
669 );
670 assert_eq!(
671 cursor(&event_frames[1]),
672 CursorToken::item(Position::Events {
673 tx_seq: 31,
674 event_index: 1,
675 public_position: LedgerPoint::event(7, 1, 1),
676 })
677 );
678 assert_eq!(
679 cursor(&event_frames[2]),
680 CursorToken::boundary(Position::Events {
681 tx_seq: 33,
682 event_index: 0,
683 public_position: LedgerPoint::checkpoint_start(8),
684 })
685 );
686 }
687
688 #[test]
689 fn transaction_objects_exclude_checkpoint_siblings() {
690 let checkpoint = TestCheckpointBuilder::new(1)
691 .start_transaction(0)
692 .create_owned_object(10)
693 .finish_transaction()
694 .start_transaction(1)
695 .create_owned_object(20)
696 .finish_transaction()
697 .build_checkpoint();
698 let first_object = TestCheckpointBuilder::derive_object_id(10);
699 let second_object = TestCheckpointBuilder::derive_object_id(20);
700
701 let first_ids = transaction_objects(&checkpoint.transactions[0], &checkpoint.object_set)
702 .map(|object| object.id())
703 .collect::<Vec<_>>();
704 let second_ids = transaction_objects(&checkpoint.transactions[1], &checkpoint.object_set)
705 .map(|object| object.id())
706 .collect::<Vec<_>>();
707
708 assert!(first_ids.contains(&first_object));
709 assert!(!first_ids.contains(&second_object));
710 assert!(second_ids.contains(&second_object));
711 assert!(!second_ids.contains(&first_object));
712 }
713}