1use crate::metrics::SubscriptionMetrics;
5use futures::{StreamExt, stream::FuturesUnordered};
6use std::sync::Arc;
7use std::sync::atomic::AtomicUsize;
8use std::sync::atomic::Ordering;
9use std::time::Duration;
10use sui_inverted_index::BitmapQuery;
11use sui_types::full_checkpoint_content::Checkpoint;
12use sui_types::full_checkpoint_content::ExecutedTransaction;
13use sui_types::full_checkpoint_content::LocallyBuiltCheckpoint;
14use sui_types::full_checkpoint_content::ObjectSet;
15use sui_types::messages_checkpoint::CheckpointSummary;
16use tokio::sync::broadcast;
17use tokio::sync::mpsc;
18use tokio::sync::oneshot;
19use tokio::time::Instant;
20use tokio::time::sleep;
21use tracing::info;
22use tracing::trace;
23use tracing::warn;
24
25mod matcher;
26
27const CHECKPOINT_MAILBOX_SIZE: usize = 1024;
28const ADMISSION_TURN_LIMIT: usize = 128;
31const SUBSCRIPTION_CHANNEL_SIZE: usize = 256;
32const DEFAULT_MAX_SUBSCRIBERS: usize = 1024;
33const SHARD_MAILBOX_SIZE: usize = 64;
36
37const DEFAULT_WATERMARK_INTERVAL: u32 = 25;
40
41fn default_shards() -> u32 {
44 std::thread::available_parallelism()
45 .map(|n| n.get() as u32)
46 .unwrap_or(1)
47}
48
49const INDEX_WAIT_POLL_INTERVAL: Duration = Duration::from_millis(10);
52const INDEX_WAIT_TIMEOUT: Duration = Duration::from_secs(10);
56
57pub type IndexedCheckpointFn = Arc<dyn Fn() -> Option<u64> + Send + Sync>;
64
65#[derive(Clone, Copy, Debug, PartialEq, Eq)]
67pub enum SubscriptionKind {
68 Checkpoints,
69 Transactions,
70 Events,
71}
72
73impl SubscriptionKind {
74 pub(crate) fn metric_label(self) -> &'static str {
75 match self {
76 Self::Checkpoints => "checkpoint",
77 Self::Transactions => "transaction",
78 Self::Events => "event",
79 }
80 }
81}
82
83#[derive(Clone, Copy)]
84pub(crate) enum SubscriptionTerminationReason {
85 ClientClosed,
86 SlowConsumer,
87 SourceLag,
88 SourceGap,
92 ServiceShutdown,
93}
94
95impl SubscriptionTerminationReason {
96 fn metric_label(self) -> &'static str {
97 match self {
98 Self::ClientClosed => "client_closed",
99 Self::SlowConsumer => "slow_consumer",
100 Self::SourceLag => "source_lag",
101 Self::SourceGap => "source_gap",
102 Self::ServiceShutdown => "service_shutdown",
103 }
104 }
105}
106
107pub(crate) struct SubscriptionLifecycleGuard {
108 kind: SubscriptionKind,
109 filtered: bool,
110 reservation: SubscriberReservation,
111 inflight_subscribers: prometheus::IntGauge,
112 terminations_total: prometheus::IntCounterVec,
113 termination_reason: SubscriptionTerminationReason,
114}
115
116impl SubscriptionLifecycleGuard {
117 pub(crate) fn new(
118 kind: SubscriptionKind,
119 filtered: bool,
120 reservation: SubscriberReservation,
121 metrics: &SubscriptionMetrics,
122 ) -> Self {
123 reservation.increment_resident_counts(kind, filtered);
124 let inflight_subscribers = metrics
125 .inflight_subscribers
126 .with_label_values(&[kind.metric_label(), if filtered { "true" } else { "false" }]);
127 inflight_subscribers.inc();
128
129 Self {
130 kind,
131 filtered,
132 reservation,
133 inflight_subscribers,
134 terminations_total: metrics.terminations_total.clone(),
135 termination_reason: SubscriptionTerminationReason::ServiceShutdown,
136 }
137 }
138
139 pub(crate) fn terminate(mut self, reason: SubscriptionTerminationReason) {
143 self.termination_reason = reason;
144 }
145}
146
147impl Drop for SubscriptionLifecycleGuard {
148 fn drop(&mut self) {
149 self.reservation
150 .decrement_resident_counts(self.kind, self.filtered);
151 self.inflight_subscribers.dec();
152 self.terminations_total
153 .with_label_values(&[
154 self.kind.metric_label(),
155 self.termination_reason.metric_label(),
156 ])
157 .inc();
158 }
159}
160
161pub struct SubscriptionSpec {
164 pub kind: SubscriptionKind,
165 pub query: Option<BitmapQuery>,
166}
167
168pub enum SubscriptionUpdate {
171 Matched(MatchedCheckpoint),
172 WatermarkTick {
178 checkpoint: u64,
179 tx_hi: u64,
180 },
181}
182
183pub struct MatchedCheckpoint {
184 pub checkpoint: SubscriptionCheckpoint,
185 pub matches: SubscriptionMatches,
186}
187
188#[derive(Clone)]
190pub enum SubscriptionCheckpoint {
191 Certified(Arc<Checkpoint>),
193 LocallyBuilt(Arc<LocallyBuiltCheckpoint>),
196}
197
198impl SubscriptionCheckpoint {
199 pub fn summary(&self) -> &CheckpointSummary {
200 match self {
201 Self::Certified(checkpoint) => checkpoint.summary.data(),
202 Self::LocallyBuilt(checkpoint) => &checkpoint.summary,
203 }
204 }
205
206 pub fn transactions(&self) -> &[ExecutedTransaction] {
207 match self {
208 Self::Certified(checkpoint) => &checkpoint.transactions,
209 Self::LocallyBuilt(checkpoint) => &checkpoint.transactions,
210 }
211 }
212
213 pub fn object_set(&self) -> &ObjectSet {
214 match self {
215 Self::Certified(checkpoint) => &checkpoint.object_set,
216 Self::LocallyBuilt(checkpoint) => &checkpoint.object_set,
217 }
218 }
219}
220
221impl From<Arc<Checkpoint>> for SubscriptionCheckpoint {
222 fn from(checkpoint: Arc<Checkpoint>) -> Self {
223 Self::Certified(checkpoint)
224 }
225}
226
227impl From<Arc<LocallyBuiltCheckpoint>> for SubscriptionCheckpoint {
228 fn from(checkpoint: Arc<LocallyBuiltCheckpoint>) -> Self {
229 Self::LocallyBuilt(checkpoint)
230 }
231}
232
233enum CheckpointMailbox {
235 Certified(broadcast::Receiver<Arc<Checkpoint>>),
236 LocallyBuilt(broadcast::Receiver<Arc<LocallyBuiltCheckpoint>>),
237}
238
239impl CheckpointMailbox {
240 async fn recv(&mut self) -> Result<SubscriptionCheckpoint, broadcast::error::RecvError> {
242 match self {
243 Self::Certified(receiver) => receiver.recv().await.map(SubscriptionCheckpoint::from),
244 Self::LocallyBuilt(receiver) => receiver.recv().await.map(SubscriptionCheckpoint::from),
245 }
246 }
247}
248
249pub enum SubscriptionMatches {
251 Checkpoint,
254 Transactions(Vec<std::ops::Range<u32>>),
259 AllTransactions,
263 Events(Vec<(u32, Vec<u32>)>),
266 AllEvents,
270}
271
272impl SubscriptionMatches {
273 pub fn transaction_indices(
278 &self,
279 tx_count: u32,
280 ) -> Option<Box<dyn Iterator<Item = u32> + Send + '_>> {
281 match self {
282 Self::Transactions(ranges) => Some(Box::new(ranges.iter().flat_map(Clone::clone))),
283 Self::AllTransactions => Some(Box::new(0..tx_count)),
284 _ => None,
285 }
286 }
287
288 pub fn event_indices<'a>(
292 &'a self,
293 checkpoint: &'a SubscriptionCheckpoint,
294 ) -> Option<Box<dyn Iterator<Item = (u32, u32)> + Send + 'a>> {
295 match self {
296 Self::Events(txs) => Some(Box::new(
297 txs.iter()
298 .flat_map(|(tx, evs)| evs.iter().map(move |&ev| (*tx, ev))),
299 )),
300 Self::AllEvents => Some(Box::new(
301 checkpoint
302 .transactions()
303 .iter()
304 .enumerate()
305 .flat_map(|(tx_idx, tx)| {
306 let event_count =
307 tx.events.as_ref().map(|e| e.data.len()).unwrap_or(0) as u32;
308 (0..event_count).map(move |ev| (tx_idx as u32, ev))
309 }),
310 )),
311 _ => None,
312 }
313 }
314}
315
316struct SubscriptionRequest {
317 spec: SubscriptionSpec,
318 response_sender: oneshot::Sender<mpsc::Receiver<SubscriptionUpdate>>,
319 reservation: SubscriberReservation,
320}
321
322#[allow(clippy::disallowed_methods)]
325fn subscription_admission_channel() -> (
326 mpsc::UnboundedSender<SubscriptionRequest>,
327 mpsc::UnboundedReceiver<SubscriptionRequest>,
328) {
329 mpsc::unbounded_channel()
330}
331
332enum AdmissionState {
333 Accepting,
334 WaitingForShard(SubscriptionRequest),
335 Closed,
336}
337
338enum SubscriptionServiceEvent {
339 Checkpoint(Result<SubscriptionCheckpoint, broadcast::error::RecvError>),
340 AdmissionRequest(SubscriptionRequest),
341 AdmissionClosed,
342 ShardCapacity {
343 shard: usize,
344 permit: mpsc::OwnedPermit<ShardMsg>,
345 },
346 AdmissionCanceled,
347}
348
349#[derive(Clone)]
350pub struct SubscriptionServiceHandle {
351 admission_sender: mpsc::UnboundedSender<SubscriptionRequest>,
352 counters: Arc<SubscriberCounts>,
353 metrics: SubscriptionMetrics,
354}
355
356impl SubscriptionServiceHandle {
357 pub async fn register_subscription(
358 &self,
359 spec: SubscriptionSpec,
360 ) -> Option<mpsc::Receiver<SubscriptionUpdate>> {
361 let reservation = match self.counters.try_reserve() {
362 Some(reservation) => reservation,
363 None => {
364 trace!(
365 "failed to register new subscriber: hit maximum number of subscribers {}",
366 self.counters.limit
367 );
368 return None;
369 }
370 };
371
372 let (response_sender, response_receiver) = oneshot::channel();
373 let request = SubscriptionRequest {
374 spec,
375 response_sender,
376 reservation,
377 };
378 self.admission_sender.send(request).ok()?;
379
380 response_receiver.await.ok()
381 }
382
383 pub(crate) fn stream_metrics(
384 &self,
385 kind: SubscriptionKind,
386 ) -> crate::metrics::SubscriptionStreamMetrics {
387 self.metrics.stream_metrics(kind.metric_label())
388 }
389}
390
391pub(crate) struct SubscriberCounts {
397 limit: usize,
398 reserved: AtomicUsize,
399 total: AtomicUsize,
400 filtered_tx: AtomicUsize,
401 filtered_event: AtomicUsize,
402}
403
404impl SubscriberCounts {
405 fn new(limit: usize) -> Self {
406 Self {
407 limit,
408 reserved: AtomicUsize::new(0),
409 total: AtomicUsize::new(0),
410 filtered_tx: AtomicUsize::new(0),
411 filtered_event: AtomicUsize::new(0),
412 }
413 }
414
415 fn try_reserve(self: &Arc<Self>) -> Option<SubscriberReservation> {
416 self.reserved
417 .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |reserved| {
418 (reserved < self.limit).then_some(reserved + 1)
419 })
420 .ok()?;
421 Some(SubscriberReservation {
422 counters: Arc::clone(self),
423 })
424 }
425
426 #[cfg(test)]
427 fn reserved(&self) -> usize {
428 self.reserved.load(Ordering::Relaxed)
429 }
430}
431
432pub(crate) struct SubscriberReservation {
433 counters: Arc<SubscriberCounts>,
434}
435
436impl SubscriberReservation {
437 fn increment_resident_counts(&self, kind: SubscriptionKind, filtered: bool) {
438 self.counters.total.fetch_add(1, Ordering::Relaxed);
439 if filtered {
440 match kind {
441 SubscriptionKind::Checkpoints | SubscriptionKind::Transactions => {
442 self.counters.filtered_tx.fetch_add(1, Ordering::Relaxed);
443 }
444 SubscriptionKind::Events => {
445 self.counters.filtered_event.fetch_add(1, Ordering::Relaxed);
446 }
447 }
448 }
449 }
450
451 fn decrement_resident_counts(&self, kind: SubscriptionKind, filtered: bool) {
452 self.counters.total.fetch_sub(1, Ordering::Relaxed);
453 if filtered {
454 match kind {
455 SubscriptionKind::Checkpoints | SubscriptionKind::Transactions => {
456 self.counters.filtered_tx.fetch_sub(1, Ordering::Relaxed);
457 }
458 SubscriptionKind::Events => {
459 self.counters.filtered_event.fetch_sub(1, Ordering::Relaxed);
460 }
461 }
462 }
463 }
464}
465
466impl Drop for SubscriberReservation {
467 fn drop(&mut self) {
468 let previous = self.counters.reserved.fetch_sub(1, Ordering::Relaxed);
469 debug_assert!(previous > 0, "subscriber reservation count underflow");
470 }
471}
472
473enum ShardMsg {
474 Checkpoint(SubscriptionCheckpoint, Arc<matcher::CheckpointKeys>),
476 Register {
478 spec: SubscriptionSpec,
479 sender: mpsc::Sender<SubscriptionUpdate>,
480 guard: SubscriptionLifecycleGuard,
481 },
482 Clear(SubscriptionTerminationReason),
484}
485
486struct SubscriptionShard {
489 mailbox: mpsc::Receiver<ShardMsg>,
490 matcher: matcher::SubscriptionMatcher,
491 watermark_interval: u32,
494}
495
496impl SubscriptionShard {
497 async fn run(mut self) {
498 while let Some(msg) = self.mailbox.recv().await {
499 self.handle_msg(msg);
500 }
501 self.matcher
502 .clear(SubscriptionTerminationReason::ServiceShutdown);
503 }
504
505 fn handle_msg(&mut self, msg: ShardMsg) {
506 match msg {
507 ShardMsg::Register {
508 spec,
509 sender,
510 guard,
511 } => {
512 self.matcher.insert(spec, sender, guard);
513 }
514 ShardMsg::Checkpoint(checkpoint, keys) => {
515 self.matcher
516 .dispatch_with_keys(&checkpoint, &keys, self.watermark_interval);
517 }
518 ShardMsg::Clear(reason) => {
519 self.matcher.clear(reason);
520 }
521 }
522 }
523
524 #[cfg(test)]
528 fn drain(&mut self) {
529 while let Ok(msg) = self.mailbox.try_recv() {
530 self.handle_msg(msg);
531 }
532 }
533}
534
535pub struct SubscriptionService {
536 checkpoint_mailbox: CheckpointMailbox,
544 admission_mailbox: mpsc::UnboundedReceiver<SubscriptionRequest>,
545 shards: Vec<mpsc::Sender<ShardMsg>>,
548 next_shard: usize,
550 counters: Arc<SubscriberCounts>,
553
554 indexed_checkpoint: Option<IndexedCheckpointFn>,
558
559 metrics: SubscriptionMetrics,
560}
561
562impl SubscriptionService {
563 pub fn build(
570 registry: &prometheus::Registry,
571 indexed_checkpoint: Option<IndexedCheckpointFn>,
572 watermark_interval: Option<u32>,
573 max_subscribers: Option<usize>,
574 shards: Option<u32>,
575 ) -> (
576 broadcast::Sender<Arc<Checkpoint>>,
577 SubscriptionServiceHandle,
578 ) {
579 let (checkpoint_sender, checkpoint_mailbox) = broadcast::channel(CHECKPOINT_MAILBOX_SIZE);
580 let handle = Self::spawn(
581 registry,
582 CheckpointMailbox::Certified(checkpoint_mailbox),
583 indexed_checkpoint,
584 watermark_interval,
585 max_subscribers,
586 shards,
587 );
588 (checkpoint_sender, handle)
589 }
590
591 pub fn build_for_locally_built_checkpoints(
600 registry: &prometheus::Registry,
601 watermark_interval: Option<u32>,
602 max_subscribers: Option<usize>,
603 shards: Option<u32>,
604 ) -> (
605 broadcast::Sender<Arc<LocallyBuiltCheckpoint>>,
606 SubscriptionServiceHandle,
607 ) {
608 let (checkpoint_sender, checkpoint_mailbox) = broadcast::channel(CHECKPOINT_MAILBOX_SIZE);
609 let handle = Self::spawn(
610 registry,
611 CheckpointMailbox::LocallyBuilt(checkpoint_mailbox),
612 None,
613 watermark_interval,
614 max_subscribers,
615 shards,
616 );
617 (checkpoint_sender, handle)
618 }
619
620 fn spawn(
621 registry: &prometheus::Registry,
622 checkpoint_mailbox: CheckpointMailbox,
623 indexed_checkpoint: Option<IndexedCheckpointFn>,
624 watermark_interval: Option<u32>,
625 max_subscribers: Option<usize>,
626 shards: Option<u32>,
627 ) -> SubscriptionServiceHandle {
628 let metrics = SubscriptionMetrics::new(registry);
629 let max_subscribers = max_subscribers.unwrap_or(DEFAULT_MAX_SUBSCRIBERS);
630 let (admission_sender, admission_mailbox) = subscription_admission_channel();
631 let counters = Arc::new(SubscriberCounts::new(max_subscribers));
632 let handle = SubscriptionServiceHandle {
633 admission_sender,
634 counters: Arc::clone(&counters),
635 metrics: metrics.clone(),
636 };
637
638 let watermark_interval = watermark_interval
639 .unwrap_or(DEFAULT_WATERMARK_INTERVAL)
640 .max(1);
641 let shards = shards.unwrap_or_else(default_shards).max(1);
642 let mut shard_senders = Vec::with_capacity(shards as usize);
643 for _ in 0..shards {
644 let (sender, shard_mailbox) = mpsc::channel(SHARD_MAILBOX_SIZE);
645 tokio::spawn(
646 SubscriptionShard {
647 mailbox: shard_mailbox,
648 matcher: matcher::SubscriptionMatcher::default(),
649 watermark_interval,
650 }
651 .run(),
652 );
653 shard_senders.push(sender);
654 }
655
656 tokio::spawn(
657 Self {
658 checkpoint_mailbox,
659 admission_mailbox,
660 shards: shard_senders,
661 next_shard: 0,
662 counters,
663 indexed_checkpoint,
664 metrics,
665 }
666 .start(),
667 );
668
669 handle
670 }
671
672 async fn start(mut self) {
673 let mut admission_state = AdmissionState::Accepting;
674 loop {
675 let event = match &mut admission_state {
676 AdmissionState::Accepting => {
677 tokio::select! {
678 biased;
679
680 result = self.checkpoint_mailbox.recv() => {
681 SubscriptionServiceEvent::Checkpoint(result)
682 },
683 request = self.admission_mailbox.recv() => {
684 match request {
685 Some(request) => {
686 SubscriptionServiceEvent::AdmissionRequest(request)
687 }
688 None => SubscriptionServiceEvent::AdmissionClosed,
689 }
690 },
691 }
692 }
693 AdmissionState::WaitingForShard(request) => {
694 let shards: Vec<_> = (0..self.shards.len())
695 .map(|offset| {
696 let shard = (self.next_shard + offset) % self.shards.len();
697 (shard, self.shards[shard].clone())
698 })
699 .collect();
700
701 tokio::select! {
702 biased;
703
704 result = self.checkpoint_mailbox.recv() => {
705 SubscriptionServiceEvent::Checkpoint(result)
706 },
707 _ = request.response_sender.closed() => {
708 SubscriptionServiceEvent::AdmissionCanceled
709 },
710 (shard, permit) = Self::reserve_first_available_shard(shards) => {
711 SubscriptionServiceEvent::ShardCapacity { shard, permit }
712 },
713 }
714 }
715 AdmissionState::Closed => {
716 SubscriptionServiceEvent::Checkpoint(self.checkpoint_mailbox.recv().await)
717 }
718 };
719
720 match event {
721 SubscriptionServiceEvent::Checkpoint(Ok(checkpoint)) => {
722 self.handle_checkpoint(checkpoint).await;
723 }
724 SubscriptionServiceEvent::Checkpoint(Err(broadcast::error::RecvError::Lagged(
725 skipped,
726 ))) => {
727 self.handle_lag(skipped).await;
728 }
729 SubscriptionServiceEvent::Checkpoint(Err(broadcast::error::RecvError::Closed)) => {
732 break;
733 }
734 SubscriptionServiceEvent::AdmissionRequest(request) => {
735 admission_state = self.admit_ready_requests(request);
736 }
737 SubscriptionServiceEvent::AdmissionClosed => {
738 admission_state = AdmissionState::Closed;
741 }
742 SubscriptionServiceEvent::ShardCapacity { shard, permit } => {
743 let AdmissionState::WaitingForShard(request) = admission_state else {
744 unreachable!("shard capacity requires a waiting admission");
745 };
746 self.complete_admission(shard, permit, request);
747 admission_state = AdmissionState::Accepting;
748 }
749 SubscriptionServiceEvent::AdmissionCanceled => {
750 let AdmissionState::WaitingForShard(_) = admission_state else {
751 unreachable!("admission cancellation requires a waiting admission");
752 };
753 admission_state = AdmissionState::Accepting;
754 }
755 }
756 }
757
758 for shard in &self.shards {
759 shard
760 .send(ShardMsg::Clear(
761 SubscriptionTerminationReason::ServiceShutdown,
762 ))
763 .await
764 .expect("subscription shard terminated unexpectedly");
765 }
766
767 info!("RPC Subscription Services ended");
768 }
769
770 async fn handle_checkpoint(&mut self, checkpoint: SubscriptionCheckpoint) {
771 let sequence_number = checkpoint.summary().sequence_number;
772
773 {
781 let last_sequence_number = self.metrics.last_recieved_checkpoint.get();
782 let expected_sequence_number = last_sequence_number + 1;
783
784 if last_sequence_number != 0 && expected_sequence_number != sequence_number as i64 {
785 match checkpoint {
786 SubscriptionCheckpoint::LocallyBuilt(_) => {
787 warn!(
788 expected = expected_sequence_number,
789 received = sequence_number,
790 "locally built checkpoint stream broke sequence; \
791 dropping all in-progress subscriptions"
792 );
793 self.clear_subscribers(SubscriptionTerminationReason::SourceGap)
794 .await;
795 }
796 SubscriptionCheckpoint::Certified(_) => panic!(
797 "recieved checkpoint out-of-order. expected checkpoint {}, recieved {}",
798 expected_sequence_number, sequence_number
799 ),
800 }
801 }
802
803 self.metrics
805 .last_recieved_checkpoint
806 .set(sequence_number as i64);
807 }
808
809 self.wait_until_indexed(sequence_number).await;
813
814 if self.counters.total.load(Ordering::Relaxed) == 0 {
818 return;
819 }
820
821 let keys = Arc::new(matcher::extract_checkpoint_keys(
827 &checkpoint,
828 self.counters.filtered_tx.load(Ordering::Relaxed) > 0,
829 self.counters.filtered_event.load(Ordering::Relaxed) > 0,
830 ));
831 for shard in &self.shards {
832 shard
833 .send(ShardMsg::Checkpoint(checkpoint.clone(), Arc::clone(&keys)))
834 .await
835 .expect("subscription shard terminated unexpectedly");
836 }
837 }
838
839 async fn wait_until_indexed(&self, sequence_number: u64) {
846 let Some(indexed) = &self.indexed_checkpoint else {
847 return;
848 };
849
850 if indexed().is_some_and(|hi| hi >= sequence_number) {
851 return;
852 }
853
854 let wait_started = Instant::now();
855 let deadline = wait_started + INDEX_WAIT_TIMEOUT;
856 loop {
857 sleep(INDEX_WAIT_POLL_INTERVAL).await;
858 if indexed().is_some_and(|hi| hi >= sequence_number) {
859 self.metrics
860 .index_wait_seconds
861 .observe(wait_started.elapsed().as_secs_f64());
862 return;
863 }
864 if Instant::now() >= deadline {
865 self.metrics.index_wait_timeouts_total.inc();
866 self.metrics
867 .index_wait_seconds
868 .observe(wait_started.elapsed().as_secs_f64());
869 warn!(
870 checkpoint = sequence_number,
871 "index did not catch up within {INDEX_WAIT_TIMEOUT:?}; \
872 delivering checkpoint anyway"
873 );
874 return;
875 }
876 }
877 }
878
879 async fn handle_lag(&mut self, skipped: u64) {
885 warn!(
886 skipped,
887 "subscription service lagged behind the checkpoint stream; \
888 dropping all in-progress subscriptions"
889 );
890 self.clear_subscribers(SubscriptionTerminationReason::SourceLag)
891 .await;
892 self.metrics.last_recieved_checkpoint.set(0);
895 }
896
897 async fn clear_subscribers(&mut self, reason: SubscriptionTerminationReason) {
900 for shard in &self.shards {
904 shard
905 .send(ShardMsg::Clear(reason))
906 .await
907 .expect("subscription shard terminated unexpectedly");
908 }
909 }
910
911 fn try_reserve_least_backlogged_shard(&self) -> Option<(usize, mpsc::OwnedPermit<ShardMsg>)> {
912 let mut selected_shard = None;
913 let mut greatest_capacity = 0;
914
915 for offset in 0..self.shards.len() {
916 let shard = (self.next_shard + offset) % self.shards.len();
917 let sender = &self.shards[shard];
918 if sender.is_closed() {
919 panic!("subscription shard terminated unexpectedly");
920 }
921
922 let capacity = sender.capacity();
923 if selected_shard.is_none() || capacity > greatest_capacity {
924 selected_shard = Some(shard);
925 greatest_capacity = capacity;
926 }
927 }
928
929 let shard = selected_shard.expect("subscription service requires at least one shard");
930 match self.shards[shard].clone().try_reserve_owned() {
931 Ok(permit) => Some((shard, permit)),
932 Err(mpsc::error::TrySendError::Full(_)) => None,
933 Err(mpsc::error::TrySendError::Closed(_)) => {
934 panic!("subscription shard terminated unexpectedly")
935 }
936 }
937 }
938
939 async fn reserve_first_available_shard(
940 shards: Vec<(usize, mpsc::Sender<ShardMsg>)>,
941 ) -> (usize, mpsc::OwnedPermit<ShardMsg>) {
942 assert!(
943 !shards.is_empty(),
944 "subscription service requires at least one shard"
945 );
946 for (_, sender) in &shards {
947 if sender.is_closed() {
948 panic!("subscription shard terminated unexpectedly");
949 }
950 }
951
952 let mut reservations = FuturesUnordered::new();
953 for (shard, sender) in shards {
954 reservations.push(async move { (shard, sender.reserve_owned().await) });
955 }
956
957 match reservations
958 .next()
959 .await
960 .expect("subscription service requires at least one shard")
961 {
962 (shard, Ok(permit)) => (shard, permit),
963 (_, Err(_)) => panic!("subscription shard terminated unexpectedly"),
964 }
965 }
966
967 fn complete_admission(
968 &mut self,
969 shard: usize,
970 permit: mpsc::OwnedPermit<ShardMsg>,
971 request: SubscriptionRequest,
972 ) {
973 if request.response_sender.is_closed() {
974 trace!("failed to register new subscriber: request was cancelled");
975 return;
976 }
977
978 let (sender, receiver) = mpsc::channel(SUBSCRIPTION_CHANNEL_SIZE);
979 if request.response_sender.send(receiver).is_err() {
980 trace!("failed to register new subscriber: request was cancelled");
981 return;
982 }
983
984 trace!("successfully registered new subscriber");
985 let kind = request.spec.kind;
986 let filtered = request.spec.query.is_some();
987 let guard =
988 SubscriptionLifecycleGuard::new(kind, filtered, request.reservation, &self.metrics);
989 permit.send(ShardMsg::Register {
990 spec: request.spec,
991 sender,
992 guard,
993 });
994 self.next_shard = (shard + 1) % self.shards.len();
995 }
996
997 fn try_admit(&mut self, request: SubscriptionRequest) -> AdmissionState {
998 if request.response_sender.is_closed() {
999 trace!("failed to register new subscriber: request was cancelled");
1000 return AdmissionState::Accepting;
1001 }
1002
1003 let Some((shard, permit)) = self.try_reserve_least_backlogged_shard() else {
1004 trace!("waiting for a subscription shard to have capacity");
1005 return AdmissionState::WaitingForShard(request);
1006 };
1007
1008 self.complete_admission(shard, permit, request);
1009 AdmissionState::Accepting
1010 }
1011
1012 fn admit_ready_requests(&mut self, first_request: SubscriptionRequest) -> AdmissionState {
1013 let mut request = first_request;
1014 let mut remaining_attempts = ADMISSION_TURN_LIMIT;
1015
1016 loop {
1017 match self.try_admit(request) {
1018 AdmissionState::Accepting => {}
1019 state => return state,
1020 }
1021
1022 remaining_attempts -= 1;
1023 if remaining_attempts == 0 {
1024 return AdmissionState::Accepting;
1025 }
1026
1027 request = match self.admission_mailbox.try_recv() {
1028 Ok(request) => request,
1029 Err(mpsc::error::TryRecvError::Empty) => {
1030 return AdmissionState::Accepting;
1031 }
1032 Err(mpsc::error::TryRecvError::Disconnected) => {
1033 return AdmissionState::Closed;
1034 }
1035 };
1036 }
1037 }
1038}
1039
1040#[cfg(test)]
1041mod tests {
1042 use std::sync::atomic::AtomicU64;
1043
1044 use move_core_types::account_address::AccountAddress;
1045 use move_core_types::identifier::Identifier;
1046 use move_core_types::language_storage::StructTag;
1047 use sui_rpc::proto::sui::rpc::v2 as proto;
1048 use sui_types::base_types::ObjectID;
1049 use sui_types::base_types::SuiAddress;
1050 use sui_types::event::Event;
1051 use sui_types::test_checkpoint_data_builder::TestCheckpointBuilder;
1052
1053 use crate::ledger_history::filter::event_filter_to_query;
1054 use crate::ledger_history::filter::transaction_filter_to_query;
1055
1056 use super::*;
1057
1058 fn test_service(shard_count: usize) -> (SubscriptionService, Vec<SubscriptionShard>) {
1061 test_service_with(shard_count, 25, None)
1062 }
1063
1064 fn test_service_with(
1065 shard_count: usize,
1066 watermark_interval: u32,
1067 indexed_checkpoint: Option<IndexedCheckpointFn>,
1068 ) -> (SubscriptionService, Vec<SubscriptionShard>) {
1069 let (service, _checkpoint_sender, _request_sender, shards) = actor_service_with(
1070 shard_count,
1071 watermark_interval,
1072 indexed_checkpoint,
1073 16,
1074 DEFAULT_MAX_SUBSCRIBERS,
1075 );
1076 (service, shards)
1077 }
1078
1079 fn actor_service_with(
1080 shard_count: usize,
1081 watermark_interval: u32,
1082 indexed_checkpoint: Option<IndexedCheckpointFn>,
1083 checkpoint_capacity: usize,
1084 max_subscribers: usize,
1085 ) -> (
1086 SubscriptionService,
1087 broadcast::Sender<Arc<Checkpoint>>,
1088 mpsc::UnboundedSender<SubscriptionRequest>,
1089 Vec<SubscriptionShard>,
1090 ) {
1091 let (checkpoint_sender, checkpoint_mailbox) = broadcast::channel(checkpoint_capacity);
1092 let (request_sender, admission_mailbox) = subscription_admission_channel();
1093 let metrics = SubscriptionMetrics::new(&prometheus::Registry::new());
1094 let counters = Arc::new(SubscriberCounts::new(max_subscribers));
1095
1096 let mut shard_senders = Vec::with_capacity(shard_count);
1097 let mut shards = Vec::with_capacity(shard_count);
1098 for _ in 0..shard_count {
1099 let (sender, shard_mailbox) = mpsc::channel(SHARD_MAILBOX_SIZE);
1100 shard_senders.push(sender);
1101 shards.push(SubscriptionShard {
1102 mailbox: shard_mailbox,
1103 matcher: matcher::SubscriptionMatcher::default(),
1104 watermark_interval,
1105 });
1106 }
1107
1108 let service = SubscriptionService {
1109 checkpoint_mailbox: CheckpointMailbox::Certified(checkpoint_mailbox),
1110 admission_mailbox,
1111 shards: shard_senders,
1112 next_shard: 0,
1113 counters,
1114 indexed_checkpoint,
1115 metrics,
1116 };
1117 (service, checkpoint_sender, request_sender, shards)
1118 }
1119
1120 fn checkpoint(sequence_number: u64) -> Arc<Checkpoint> {
1121 Arc::new(TestCheckpointBuilder::new(sequence_number).build_checkpoint())
1122 }
1123
1124 fn locally_built_checkpoint(sequence_number: u64) -> SubscriptionCheckpoint {
1125 let checkpoint = TestCheckpointBuilder::new(sequence_number).build_checkpoint();
1126 Arc::new(LocallyBuiltCheckpoint {
1127 summary: checkpoint.summary.into_data(),
1128 contents: checkpoint.contents,
1129 transactions: checkpoint.transactions,
1130 object_set: checkpoint.object_set,
1131 })
1132 .into()
1133 }
1134
1135 fn checkpoint_with_senders(seq: u64, senders: &[u8]) -> Arc<Checkpoint> {
1137 let mut builder = TestCheckpointBuilder::new(seq);
1138 for &sender in senders {
1139 builder = builder.start_transaction(sender).finish_transaction();
1140 }
1141 Arc::new(builder.build_checkpoint())
1142 }
1143
1144 fn checkpoint_with_events(seq: u64) -> Arc<Checkpoint> {
1146 let package = AccountAddress::random();
1147 let event = |name: &str| Event {
1148 package_id: ObjectID::from(package),
1149 transaction_module: Identifier::new("emitter").unwrap(),
1150 sender: addr(1),
1151 type_: StructTag {
1152 address: package,
1153 module: Identifier::new("mod_t").unwrap(),
1154 name: Identifier::new(name).unwrap(),
1155 type_params: vec![],
1156 },
1157 contents: vec![],
1158 };
1159 let mut builder = TestCheckpointBuilder::new(seq);
1160 builder = builder.start_transaction(0).finish_transaction();
1161 builder = builder
1162 .start_transaction(1)
1163 .with_events(vec![event("EventA"), event("EventB")])
1164 .finish_transaction();
1165 Arc::new(builder.build_checkpoint())
1166 }
1167
1168 fn addr(idx: u8) -> SuiAddress {
1169 TestCheckpointBuilder::derive_address(idx)
1170 }
1171
1172 fn sender_query(address: SuiAddress, negated: bool) -> BitmapQuery {
1173 let mut sender = proto::SenderFilter::default();
1174 sender.address = Some(address.to_string());
1175 let mut literal = proto::TransactionLiteral::default();
1176 literal.predicate = Some(proto::transaction_literal::Predicate::Sender(sender));
1177 literal.negated = negated;
1178 let mut term = proto::TransactionTerm::default();
1179 term.literals = vec![literal];
1180 let mut filter = proto::TransactionFilter::default();
1181 filter.terms = vec![term];
1182 transaction_filter_to_query(&filter, 16).unwrap()
1183 }
1184
1185 fn event_type_query(type_str: &str) -> BitmapQuery {
1186 let mut event_type = proto::EventTypeFilter::default();
1187 event_type.event_type = Some(type_str.to_owned());
1188 let mut literal = proto::EventLiteral::default();
1189 literal.predicate = Some(proto::event_literal::Predicate::EventType(event_type));
1190 let mut term = proto::EventTerm::default();
1191 term.literals = vec![literal];
1192 let mut filter = proto::EventFilter::default();
1193 filter.terms = vec![term];
1194 event_filter_to_query(&filter, 16).unwrap()
1195 }
1196
1197 fn unfiltered() -> SubscriptionSpec {
1198 SubscriptionSpec {
1199 kind: SubscriptionKind::Checkpoints,
1200 query: None,
1201 }
1202 }
1203
1204 fn subscription_request(
1205 counters: &Arc<SubscriberCounts>,
1206 spec: SubscriptionSpec,
1207 response_sender: oneshot::Sender<mpsc::Receiver<SubscriptionUpdate>>,
1208 ) -> SubscriptionRequest {
1209 let reservation = counters
1210 .try_reserve()
1211 .expect("test request requires subscriber capacity");
1212 SubscriptionRequest {
1213 spec,
1214 response_sender,
1215 reservation,
1216 }
1217 }
1218
1219 async fn register(
1224 service: &mut SubscriptionService,
1225 spec: SubscriptionSpec,
1226 ) -> Option<mpsc::Receiver<SubscriptionUpdate>> {
1227 let reservation = service.counters.try_reserve()?;
1228 let (response_sender, response_receiver) = oneshot::channel();
1229 assert!(matches!(
1230 service.try_admit(SubscriptionRequest {
1231 spec,
1232 response_sender,
1233 reservation,
1234 }),
1235 AdmissionState::Accepting
1236 ));
1237 response_receiver.await.ok()
1238 }
1239 fn inflight_subscribers(metrics: &SubscriptionMetrics) -> i64 {
1240 ["checkpoint", "transaction", "event"]
1241 .into_iter()
1242 .flat_map(|kind| {
1243 ["true", "false"].into_iter().map(move |filtered| {
1244 metrics
1245 .inflight_subscribers
1246 .with_label_values(&[kind, filtered])
1247 .get()
1248 })
1249 })
1250 .sum()
1251 }
1252
1253 fn terminations(
1254 metrics: &SubscriptionMetrics,
1255 kind: &'static str,
1256 reason: &'static str,
1257 ) -> u64 {
1258 metrics
1259 .terminations_total
1260 .with_label_values(&[kind, reason])
1261 .get()
1262 }
1263
1264 fn drain(shards: &mut [SubscriptionShard]) {
1266 for shard in shards {
1267 shard.drain();
1268 }
1269 }
1270
1271 fn matched_sequence_number(update: SubscriptionUpdate) -> u64 {
1272 match update {
1273 SubscriptionUpdate::Matched(matched) => {
1274 assert!(matches!(matched.matches, SubscriptionMatches::Checkpoint));
1275 matched.checkpoint.summary().sequence_number
1276 }
1277 SubscriptionUpdate::WatermarkTick { .. } => {
1278 panic!("expected a matched checkpoint, got a watermark tick")
1279 }
1280 }
1281 }
1282
1283 fn matched_transactions(update: SubscriptionUpdate) -> Vec<u32> {
1284 match update {
1285 SubscriptionUpdate::Matched(matched) => match matched.matches {
1286 SubscriptionMatches::Transactions(ranges) => ranges.into_iter().flatten().collect(),
1287 _ => panic!("expected transaction matches"),
1288 },
1289 SubscriptionUpdate::WatermarkTick { .. } => {
1290 panic!("expected a matched checkpoint, got a watermark tick")
1291 }
1292 }
1293 }
1294
1295 #[test]
1296 fn subscriber_reservations_are_bounded_and_released() {
1297 let counters = Arc::new(SubscriberCounts::new(8));
1298 let attempted = Arc::new(std::sync::Barrier::new(65));
1299 let release = Arc::new(std::sync::Barrier::new(65));
1300
1301 std::thread::scope(|scope| {
1302 let mut handles = Vec::with_capacity(64);
1303 for _ in 0..64 {
1304 let counters = Arc::clone(&counters);
1305 let attempted = Arc::clone(&attempted);
1306 let release = Arc::clone(&release);
1307 handles.push(scope.spawn(move || {
1308 let reservation = counters.try_reserve();
1309 attempted.wait();
1310 release.wait();
1311 reservation.is_some()
1312 }));
1313 }
1314
1315 attempted.wait();
1316 assert_eq!(counters.reserved(), 8);
1317 release.wait();
1318
1319 let successful_reservations = handles
1320 .into_iter()
1321 .map(|handle| handle.join().unwrap())
1322 .filter(|success| *success)
1323 .count();
1324 assert_eq!(successful_reservations, 8);
1325 });
1326
1327 assert_eq!(counters.reserved(), 0);
1328 }
1329
1330 #[tokio::test]
1331 async fn pending_request_reserves_final_subscriber_slot() {
1332 let counters = Arc::new(SubscriberCounts::new(3));
1333 let active_reservations = [
1334 counters.try_reserve().unwrap(),
1335 counters.try_reserve().unwrap(),
1336 ];
1337 let (admission_sender, mailbox) = subscription_admission_channel();
1338 let handle = SubscriptionServiceHandle {
1339 admission_sender,
1340 counters: Arc::clone(&counters),
1341 metrics: SubscriptionMetrics::new(&prometheus::Registry::new()),
1342 };
1343
1344 let pending_handle = handle.clone();
1345 let pending_registration =
1346 tokio::spawn(async move { pending_handle.register_subscription(unfiltered()).await });
1347 tokio::time::timeout(Duration::from_secs(1), async {
1348 while mailbox.len() != 1 {
1349 tokio::task::yield_now().await;
1350 }
1351 })
1352 .await
1353 .expect("pending registration did not enter admission");
1354
1355 assert_eq!(counters.reserved(), 3);
1356 let rejected = tokio::time::timeout(
1357 Duration::from_secs(1),
1358 handle.register_subscription(unfiltered()),
1359 )
1360 .await
1361 .expect("the final subscriber slot is already reserved");
1362 assert!(rejected.is_none());
1363 assert_eq!(mailbox.len(), 1);
1364
1365 drop(mailbox);
1366 assert!(pending_registration.await.unwrap().is_none());
1367 assert_eq!(counters.reserved(), 2);
1368
1369 drop(active_reservations);
1370 assert_eq!(counters.reserved(), 0);
1371 }
1372
1373 #[tokio::test]
1374 async fn zero_subscriber_limit_rejects_before_ingress() {
1375 let (_checkpoint_sender, handle) =
1376 SubscriptionService::build(&prometheus::Registry::new(), None, None, Some(0), Some(1));
1377 assert!(handle.counters.try_reserve().is_none());
1378 assert!(handle.register_subscription(unfiltered()).await.is_none());
1379 assert_eq!(handle.counters.reserved(), 0);
1380 }
1381
1382 #[tokio::test]
1383 async fn public_admission_rejects_before_queue_at_limit() {
1384 let counters = Arc::new(SubscriberCounts::new(1));
1385 let held_reservation = counters.try_reserve().unwrap();
1386 let (admission_sender, mut mailbox) = subscription_admission_channel();
1387 let handle = SubscriptionServiceHandle {
1388 admission_sender,
1389 counters: Arc::clone(&counters),
1390 metrics: SubscriptionMetrics::new(&prometheus::Registry::new()),
1391 };
1392
1393 let result = tokio::time::timeout(
1394 Duration::from_secs(1),
1395 handle.register_subscription(unfiltered()),
1396 )
1397 .await
1398 .expect("a saturated service must reject before queueing");
1399 assert!(result.is_none());
1400 assert!(matches!(
1401 mailbox.try_recv(),
1402 Err(mpsc::error::TryRecvError::Empty)
1403 ));
1404
1405 drop(held_reservation);
1406 assert_eq!(counters.reserved(), 0);
1407 }
1408
1409 #[tokio::test]
1410 async fn closed_public_admission_queue_releases_reservation() {
1411 let counters = Arc::new(SubscriberCounts::new(1));
1412 let (admission_sender, mailbox) = subscription_admission_channel();
1413 drop(mailbox);
1414 let handle = SubscriptionServiceHandle {
1415 admission_sender,
1416 counters: Arc::clone(&counters),
1417 metrics: SubscriptionMetrics::new(&prometheus::Registry::new()),
1418 };
1419
1420 let result = handle.register_subscription(unfiltered()).await;
1421 assert!(result.is_none());
1422 assert_eq!(counters.reserved(), 0);
1423 }
1424
1425 #[test]
1426 fn admission_turn_is_bounded_and_drains_ready_requests() {
1427 let (mut service, _checkpoint_sender, request_sender, _shards) =
1428 actor_service_with(3, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1429 let mut response_receivers = Vec::with_capacity(ADMISSION_TURN_LIMIT + 1);
1430
1431 for _ in 0..=ADMISSION_TURN_LIMIT {
1432 let (response_sender, response_receiver) = oneshot::channel();
1433 assert!(
1434 request_sender
1435 .send(subscription_request(
1436 &service.counters,
1437 unfiltered(),
1438 response_sender,
1439 ))
1440 .is_ok()
1441 );
1442 response_receivers.push(response_receiver);
1443 }
1444
1445 let first_request = service.admission_mailbox.try_recv().unwrap();
1446 assert!(matches!(
1447 service.admit_ready_requests(first_request),
1448 AdmissionState::Accepting
1449 ));
1450 assert_eq!(service.admission_mailbox.len(), 1);
1451
1452 for response_receiver in response_receivers.iter_mut().take(ADMISSION_TURN_LIMIT) {
1453 response_receiver
1454 .try_recv()
1455 .expect("request should be admitted");
1456 }
1457 assert!(matches!(
1458 response_receivers.last_mut().unwrap().try_recv(),
1459 Err(oneshot::error::TryRecvError::Empty)
1460 ));
1461
1462 let final_request = service.admission_mailbox.try_recv().unwrap();
1463 assert!(matches!(
1464 service.admit_ready_requests(final_request),
1465 AdmissionState::Accepting
1466 ));
1467 response_receivers
1468 .last_mut()
1469 .unwrap()
1470 .try_recv()
1471 .expect("final request should be admitted");
1472 }
1473
1474 #[test]
1475 fn canceled_ready_admission_releases_reservation() {
1476 let (mut service, _checkpoint_sender, _request_sender, _shards) =
1477 actor_service_with(1, 25, None, 4, 1);
1478 let (response_sender, response_receiver) = oneshot::channel();
1479 let request = subscription_request(&service.counters, unfiltered(), response_sender);
1480 assert_eq!(service.counters.reserved(), 1);
1481
1482 drop(response_receiver);
1483 assert!(matches!(
1484 service.try_admit(request),
1485 AdmissionState::Accepting
1486 ));
1487 assert_eq!(service.counters.reserved(), 0);
1488 }
1489
1490 #[tokio::test]
1491 async fn checkpoint_backlog_drains_before_registration() {
1492 let (service, checkpoint_sender, request_sender, mut shards) =
1495 actor_service_with(1, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1496 let metrics = service.metrics.clone();
1497 let (reply_sender, reply_receiver) = oneshot::channel();
1498 assert!(
1499 request_sender
1500 .send(subscription_request(
1501 &service.counters,
1502 unfiltered(),
1503 reply_sender,
1504 ))
1505 .is_ok()
1506 );
1507 assert_eq!(checkpoint_sender.send(checkpoint(1)).unwrap(), 1);
1508 assert_eq!(checkpoint_sender.send(checkpoint(2)).unwrap(), 1);
1509
1510 let actor = tokio::spawn(service.start());
1513 let mut receiver = tokio::time::timeout(Duration::from_secs(1), reply_receiver)
1514 .await
1515 .expect("admission remained blocked after the checkpoint backlog drained")
1516 .unwrap();
1517 assert_eq!(metrics.last_recieved_checkpoint.get(), 2);
1518
1519 assert_eq!(checkpoint_sender.send(checkpoint(3)).unwrap(), 1);
1522 drop(checkpoint_sender);
1523 actor.await.unwrap();
1524
1525 drain(&mut shards);
1526 assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 3);
1527 assert!(receiver.recv().await.is_none());
1528 }
1529
1530 #[tokio::test]
1531 async fn queued_admission_releases_reservation_when_actor_shuts_down() {
1532 let (service, checkpoint_sender, request_sender, _shards) =
1533 actor_service_with(1, 25, None, 4, 1);
1534 let counters = Arc::clone(&service.counters);
1535 let (response_sender, response_receiver) = oneshot::channel();
1536 assert!(
1537 request_sender
1538 .send(subscription_request(
1539 &counters,
1540 unfiltered(),
1541 response_sender,
1542 ))
1543 .is_ok()
1544 );
1545 assert_eq!(counters.reserved(), 1);
1546
1547 drop(checkpoint_sender);
1548 service.start().await;
1549
1550 assert!(response_receiver.await.is_err());
1551 assert_eq!(counters.reserved(), 0);
1552 }
1553
1554 #[tokio::test]
1555 async fn admission_prefers_least_backlogged_shard() {
1556 let (mut service, mut shards) = test_service(3);
1559 for _ in 0..2 {
1560 assert!(
1561 service.shards[0]
1562 .try_send(ShardMsg::Clear(
1563 SubscriptionTerminationReason::ServiceShutdown
1564 ))
1565 .is_ok()
1566 );
1567 }
1568 assert!(
1569 service.shards[1]
1570 .try_send(ShardMsg::Clear(
1571 SubscriptionTerminationReason::ServiceShutdown
1572 ))
1573 .is_ok()
1574 );
1575
1576 let receiver = register(&mut service, unfiltered()).await.unwrap();
1579 let registration = shards[2].mailbox.try_recv().unwrap();
1580 assert!(matches!(registration, ShardMsg::Register { .. }));
1581 shards[2].handle_msg(registration);
1582 assert_eq!(service.counters.total.load(Ordering::Relaxed), 1);
1583 assert_eq!(inflight_subscribers(&service.metrics), 1);
1584 drop(receiver);
1585 }
1586
1587 #[tokio::test]
1588 async fn saturated_admission_waits_for_any_shard() {
1589 let (service, _checkpoint_sender, request_sender, mut shards) =
1592 actor_service_with(2, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1593 for shard in &service.shards {
1594 for _ in 0..SHARD_MAILBOX_SIZE {
1595 assert!(
1596 shard
1597 .try_send(ShardMsg::Clear(
1598 SubscriptionTerminationReason::ServiceShutdown
1599 ))
1600 .is_ok()
1601 );
1602 }
1603 }
1604
1605 let (reply_sender, reply_receiver) = oneshot::channel();
1608 assert!(
1609 request_sender
1610 .send(subscription_request(
1611 &service.counters,
1612 unfiltered(),
1613 reply_sender,
1614 ))
1615 .is_ok()
1616 );
1617 let mut actor = Box::pin(service.start());
1618 assert!(futures::poll!(&mut actor).is_pending());
1619 let actor = tokio::spawn(actor);
1620
1621 assert!(matches!(
1624 shards[1].mailbox.try_recv(),
1625 Ok(ShardMsg::Clear(_))
1626 ));
1627 let _receiver = tokio::time::timeout(Duration::from_secs(1), reply_receiver)
1628 .await
1629 .expect("admission did not wake when shard capacity became available")
1630 .unwrap();
1631
1632 for _ in 1..SHARD_MAILBOX_SIZE {
1635 assert!(matches!(
1636 shards[1].mailbox.try_recv(),
1637 Ok(ShardMsg::Clear(_))
1638 ));
1639 }
1640 assert!(matches!(
1641 shards[1].mailbox.try_recv(),
1642 Ok(ShardMsg::Register { .. })
1643 ));
1644 assert_eq!(shards[0].mailbox.len(), SHARD_MAILBOX_SIZE);
1645
1646 actor.abort();
1647 let _ = actor.await;
1648 }
1649
1650 #[tokio::test]
1651 async fn saturated_head_blocks_newer_request_until_canceled() {
1652 let (service, _checkpoint_sender, request_sender, mut shards) =
1654 actor_service_with(2, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1655 let counters = Arc::clone(&service.counters);
1656 for shard in &service.shards {
1657 for _ in 0..SHARD_MAILBOX_SIZE {
1658 assert!(
1659 shard
1660 .try_send(ShardMsg::Clear(
1661 SubscriptionTerminationReason::ServiceShutdown
1662 ))
1663 .is_ok()
1664 );
1665 }
1666 }
1667
1668 let (first_sender, first_reply) = oneshot::channel();
1669 let (second_sender, second_reply) = oneshot::channel();
1670 assert!(
1671 request_sender
1672 .send(subscription_request(
1673 &service.counters,
1674 unfiltered(),
1675 first_sender,
1676 ))
1677 .is_ok()
1678 );
1679 assert!(
1680 request_sender
1681 .send(subscription_request(
1682 &service.counters,
1683 unfiltered(),
1684 second_sender,
1685 ))
1686 .is_ok()
1687 );
1688 assert_eq!(counters.reserved(), 2);
1689
1690 let mut actor = Box::pin(service.start());
1693 assert!(futures::poll!(&mut actor).is_pending());
1694
1695 drop(first_reply);
1696 assert!(futures::poll!(&mut actor).is_pending());
1699 assert_eq!(counters.reserved(), 1);
1700 let actor = tokio::spawn(actor);
1701
1702 assert!(matches!(
1704 shards[0].mailbox.try_recv(),
1705 Ok(ShardMsg::Clear(_))
1706 ));
1707 let _receiver = tokio::time::timeout(Duration::from_secs(1), second_reply)
1708 .await
1709 .expect("newer admission remained blocked after canceling the waiting request")
1710 .unwrap();
1711 assert_eq!(counters.reserved(), 1);
1712
1713 actor.abort();
1714 let _ = actor.await;
1715 drop(shards);
1716 assert_eq!(counters.reserved(), 0);
1717 }
1718
1719 #[tokio::test]
1720 async fn run_loop_lag_clears_subscribers_and_resets_sequence_tracker() {
1721 let (mut service, checkpoint_sender, _request_sender, mut shards) =
1724 actor_service_with(1, 25, None, 1, DEFAULT_MAX_SUBSCRIBERS);
1725 let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1726 let counters = Arc::clone(&service.counters);
1727 let metrics = service.metrics.clone();
1728 metrics.last_recieved_checkpoint.set(99);
1731
1732 assert_eq!(checkpoint_sender.send(checkpoint(1)).unwrap(), 1);
1733 assert_eq!(checkpoint_sender.send(checkpoint(2)).unwrap(), 1);
1734 let actor = tokio::spawn(service.start());
1735 drop(checkpoint_sender);
1736 actor.await.unwrap();
1737
1738 drain(&mut shards);
1741 assert!(receiver.recv().await.is_none());
1742 assert_eq!(counters.total.load(Ordering::Relaxed), 0);
1743 assert_eq!(inflight_subscribers(&metrics), 0);
1744 assert_eq!(terminations(&metrics, "checkpoint", "source_lag"), 1);
1745 assert_eq!(metrics.last_recieved_checkpoint.get(), 2);
1748 }
1749
1750 #[tokio::test]
1751 async fn labeled_inflight_gauge_covers_all_types_and_filter_states() {
1752 let (mut service, mut shards) = test_service(1);
1753 let kinds = [
1754 SubscriptionKind::Checkpoints,
1755 SubscriptionKind::Transactions,
1756 SubscriptionKind::Events,
1757 ];
1758 let mut receivers = Vec::new();
1759
1760 for kind in kinds {
1761 for filtered in [false, true] {
1762 let query = filtered.then(|| match kind {
1763 SubscriptionKind::Checkpoints | SubscriptionKind::Transactions => {
1764 sender_query(addr(0), false)
1765 }
1766 SubscriptionKind::Events => event_type_query(
1767 "0x0000000000000000000000000000000000000000000000000000000000000002::coin::CoinEvent",
1768 ),
1769 });
1770 receivers.push(
1771 register(&mut service, SubscriptionSpec { kind, query })
1772 .await
1773 .unwrap(),
1774 );
1775 }
1776 }
1777 drain(&mut shards);
1778
1779 for kind in kinds {
1780 for filtered in ["false", "true"] {
1781 assert_eq!(
1782 service
1783 .metrics
1784 .inflight_subscribers
1785 .with_label_values(&[kind.metric_label(), filtered])
1786 .get(),
1787 1
1788 );
1789 }
1790 }
1791
1792 drop(receivers);
1793 service.handle_checkpoint(checkpoint(1).into()).await;
1794 drain(&mut shards);
1795 assert_eq!(inflight_subscribers(&service.metrics), 0);
1796 assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1797
1798 shards[0].handle_msg(ShardMsg::Clear(
1799 SubscriptionTerminationReason::ServiceShutdown,
1800 ));
1801 assert_eq!(inflight_subscribers(&service.metrics), 0);
1802 assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1803 for kind in ["checkpoint", "transaction", "event"] {
1804 assert_eq!(terminations(&service.metrics, kind, "client_closed"), 2);
1805 }
1806 }
1807
1808 #[tokio::test]
1809 async fn handle_checkpoint_fans_out_in_order() {
1810 let (mut service, mut shards) = test_service(1);
1811 let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1812
1813 service.handle_checkpoint(checkpoint(1).into()).await;
1814 service.handle_checkpoint(checkpoint(2).into()).await;
1815 drain(&mut shards);
1816
1817 assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 1);
1818 assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 2);
1819 assert_eq!(shards[0].matcher.len(), 1);
1820 }
1821
1822 #[tokio::test]
1823 async fn handle_checkpoint_drops_departed_subscriber() {
1824 let (mut service, mut shards) = test_service(1);
1825 let receiver = register(&mut service, unfiltered()).await.unwrap();
1826 drop(receiver);
1827
1828 service.handle_checkpoint(checkpoint(1).into()).await;
1829 drain(&mut shards);
1830
1831 assert!(shards[0].matcher.is_empty());
1832 assert_eq!(inflight_subscribers(&service.metrics), 0);
1833 assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1834 assert_eq!(
1835 terminations(&service.metrics, "checkpoint", "client_closed"),
1836 1
1837 );
1838 }
1839
1840 #[tokio::test(start_paused = true)]
1841 async fn handle_checkpoint_waits_for_index_before_delivering() {
1842 let indexed = Arc::new(AtomicU64::new(4));
1844 let gate = indexed.clone();
1845 let (mut service, mut shards) = test_service_with(
1846 1,
1847 25,
1848 Some(Arc::new(move || Some(gate.load(Ordering::SeqCst)))),
1849 );
1850 let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1851
1852 service.handle_checkpoint(checkpoint(4).into()).await;
1853 drain(&mut shards);
1854 assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 4);
1855 assert_eq!(service.metrics.index_wait_seconds.get_sample_count(), 0);
1856 assert_eq!(service.metrics.index_wait_timeouts_total.get(), 0);
1857
1858 {
1860 let mut deliver = std::pin::pin!(service.handle_checkpoint(checkpoint(5).into()));
1861 assert!(
1862 futures::poll!(&mut deliver).is_pending(),
1863 "delivery should block while checkpoint 5 is unindexed"
1864 );
1865 assert!(receiver.try_recv().is_err());
1866
1867 indexed.store(5, Ordering::SeqCst);
1868 tokio::time::advance(INDEX_WAIT_POLL_INTERVAL).await;
1869 deliver.await;
1870 }
1871 drain(&mut shards);
1872
1873 assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 5);
1874 assert_eq!(service.metrics.index_wait_seconds.get_sample_count(), 1);
1875 assert!(
1876 service.metrics.index_wait_seconds.get_sample_sum()
1877 >= INDEX_WAIT_POLL_INTERVAL.as_secs_f64()
1878 );
1879 assert_eq!(service.metrics.index_wait_timeouts_total.get(), 0);
1880 }
1881
1882 #[tokio::test(start_paused = true)]
1883 async fn index_wait_timeout_records_and_delivers_with_paused_time() {
1884 let indexed = Arc::new(AtomicU64::new(4));
1885 let gate = indexed.clone();
1886 let (mut service, mut shards) = test_service_with(
1887 1,
1888 25,
1889 Some(Arc::new(move || Some(gate.load(Ordering::SeqCst)))),
1890 );
1891 let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1892
1893 {
1894 let mut deliver = std::pin::pin!(service.handle_checkpoint(checkpoint(5).into()));
1895 assert!(
1896 futures::poll!(&mut deliver).is_pending(),
1897 "delivery should block while checkpoint 5 is unindexed"
1898 );
1899 tokio::time::advance(INDEX_WAIT_TIMEOUT).await;
1900 deliver.await;
1901 }
1902 drain(&mut shards);
1903
1904 assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 5);
1905 assert_eq!(service.metrics.index_wait_seconds.get_sample_count(), 1);
1906 assert!(
1907 service.metrics.index_wait_seconds.get_sample_sum() >= INDEX_WAIT_TIMEOUT.as_secs_f64()
1908 );
1909 assert_eq!(service.metrics.index_wait_timeouts_total.get(), 1);
1910 }
1911
1912 #[tokio::test]
1913 async fn handle_lag_drops_all_subscribers_and_resets_tracker() {
1914 let (mut service, mut shards) = test_service(2);
1915 let mut receiver_1 = register(&mut service, unfiltered()).await.unwrap();
1916 let mut receiver_2 = register(&mut service, unfiltered()).await.unwrap();
1917
1918 service.handle_checkpoint(checkpoint(5).into()).await;
1919 drain(&mut shards);
1920 assert_eq!(service.metrics.last_recieved_checkpoint.get(), 5);
1921
1922 service.handle_lag(10).await;
1923 drain(&mut shards);
1924
1925 assert!(shards[0].matcher.is_empty());
1927 assert!(shards[1].matcher.is_empty());
1928 assert_eq!(inflight_subscribers(&service.metrics), 0);
1929 assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1930 assert!(receiver_1.recv().await.is_some()); assert!(receiver_1.recv().await.is_none());
1933 assert!(receiver_2.recv().await.is_some());
1934 assert!(receiver_2.recv().await.is_none());
1935 assert_eq!(service.metrics.last_recieved_checkpoint.get(), 0);
1938 assert_eq!(
1939 terminations(&service.metrics, "checkpoint", "source_lag"),
1940 2
1941 );
1942 service.handle_lag(1).await;
1943 drain(&mut shards);
1944 assert_eq!(
1945 terminations(&service.metrics, "checkpoint", "source_lag"),
1946 2
1947 );
1948
1949 let mut receiver_3 = register(&mut service, unfiltered()).await.unwrap();
1950 service.handle_checkpoint(checkpoint(100).into()).await;
1951 drain(&mut shards);
1952 assert_eq!(
1953 matched_sequence_number(receiver_3.recv().await.unwrap()),
1954 100
1955 );
1956 }
1957
1958 #[tokio::test]
1959 async fn locally_built_gap_drops_subscribers() {
1960 let (mut service, mut shards) = test_service(2);
1961 let mut before_gap = register(&mut service, unfiltered()).await.unwrap();
1962
1963 service.handle_checkpoint(locally_built_checkpoint(5)).await;
1964 service.handle_checkpoint(locally_built_checkpoint(9)).await;
1965 drain(&mut shards);
1966
1967 assert_eq!(matched_sequence_number(before_gap.recv().await.unwrap()), 5);
1970 assert!(before_gap.recv().await.is_none());
1971 assert_eq!(
1972 terminations(&service.metrics, "checkpoint", "source_gap"),
1973 1
1974 );
1975 assert_eq!(service.metrics.last_recieved_checkpoint.get(), 9);
1976
1977 let mut after_gap = register(&mut service, unfiltered()).await.unwrap();
1978 service
1979 .handle_checkpoint(locally_built_checkpoint(10))
1980 .await;
1981 drain(&mut shards);
1982 assert_eq!(matched_sequence_number(after_gap.recv().await.unwrap()), 10);
1983 }
1984
1985 #[tokio::test]
1986 #[should_panic(expected = "out-of-order")]
1987 async fn certified_gap_panics() {
1988 let (mut service, _shards) = test_service(1);
1989 service.handle_checkpoint(checkpoint(5).into()).await;
1990 service.handle_checkpoint(checkpoint(9).into()).await;
1991 }
1992
1993 #[tokio::test]
1994 async fn service_shutdown_records_each_subscription_once() {
1995 let (mut service, mut shards) = test_service(2);
1996 let receiver_explicit = register(&mut service, unfiltered()).await.unwrap();
1997 let receiver_fallback = register(&mut service, unfiltered()).await.unwrap();
1998 drain(&mut shards);
1999 let metrics = service.metrics.clone();
2000 let counters = Arc::clone(&service.counters);
2001 assert_eq!(counters.reserved(), 2);
2002
2003 service.shards[0]
2004 .send(ShardMsg::Clear(
2005 SubscriptionTerminationReason::ServiceShutdown,
2006 ))
2007 .await
2008 .unwrap();
2009 shards[0].drain();
2010 assert_eq!(terminations(&metrics, "checkpoint", "service_shutdown"), 1);
2011 assert_eq!(counters.reserved(), 1);
2012
2013 let fallback_shard = shards.pop().unwrap();
2014 drop(service);
2015 fallback_shard.run().await;
2016 assert!(receiver_explicit.is_closed());
2017 assert!(receiver_fallback.is_closed());
2018 assert_eq!(inflight_subscribers(&metrics), 0);
2019 assert_eq!(counters.total.load(Ordering::Relaxed), 0);
2020 assert_eq!(counters.reserved(), 0);
2021 assert_eq!(counters.filtered_tx.load(Ordering::Relaxed), 0);
2022 assert_eq!(counters.filtered_event.load(Ordering::Relaxed), 0);
2023 assert_eq!(terminations(&metrics, "checkpoint", "service_shutdown"), 2);
2024
2025 drop(shards);
2026 assert_eq!(terminations(&metrics, "checkpoint", "service_shutdown"), 2);
2027 }
2028
2029 #[tokio::test]
2030 async fn subscribers_on_every_shard_receive_a_matched_checkpoint() {
2031 let (mut service, mut shards) = test_service(2);
2032 let mut receivers = Vec::new();
2033 for _ in 0..4 {
2034 receivers.push(register(&mut service, unfiltered()).await.unwrap());
2035 }
2036 drain(&mut shards);
2037 assert_eq!(shards[0].matcher.len(), 2);
2039 assert_eq!(shards[1].matcher.len(), 2);
2040
2041 service.handle_checkpoint(checkpoint(1).into()).await;
2042 drain(&mut shards);
2043 for receiver in &mut receivers {
2044 assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 1);
2045 }
2046 }
2047
2048 #[tokio::test]
2049 async fn filtered_subscribers_match_via_dispatcher_extracted_keys() {
2050 let (mut service, mut shards) = test_service(2);
2051 let mut include_rx = register(
2053 &mut service,
2054 SubscriptionSpec {
2055 kind: SubscriptionKind::Transactions,
2056 query: Some(sender_query(addr(0), false)),
2057 },
2058 )
2059 .await
2060 .unwrap();
2061 let mut exclude_rx = register(
2065 &mut service,
2066 SubscriptionSpec {
2067 kind: SubscriptionKind::Transactions,
2068 query: Some(sender_query(addr(0), true)),
2069 },
2070 )
2071 .await
2072 .unwrap();
2073 assert_eq!(service.counters.filtered_tx.load(Ordering::Relaxed), 2);
2074
2075 let checkpoint = checkpoint_with_senders(1, &[0, 1]);
2076 let tx_lo = checkpoint.summary.data().network_total_transactions
2077 - checkpoint.transactions.len() as u64;
2078 service.handle_checkpoint(checkpoint.into()).await;
2079 drain(&mut shards);
2080
2081 assert!(matches!(
2082 include_rx.recv().await.unwrap(),
2083 SubscriptionUpdate::WatermarkTick {
2084 checkpoint: 0,
2085 tx_hi
2086 } if tx_hi == tx_lo
2087 ));
2088 assert!(matches!(
2089 exclude_rx.recv().await.unwrap(),
2090 SubscriptionUpdate::WatermarkTick {
2091 checkpoint: 0,
2092 tx_hi
2093 } if tx_hi == tx_lo
2094 ));
2095
2096 assert_eq!(
2097 matched_transactions(include_rx.recv().await.unwrap()),
2098 vec![0]
2099 );
2100 assert_eq!(
2101 matched_transactions(exclude_rx.recv().await.unwrap()),
2102 vec![1]
2103 );
2104 }
2105
2106 #[tokio::test]
2107 async fn configured_cap_is_enforced_globally_across_shards() {
2108 let max_subscribers = 3;
2109 let (mut service, _, _, mut shards) = actor_service_with(2, 25, None, 16, max_subscribers);
2110 let mut receivers = Vec::with_capacity(max_subscribers);
2111 for _ in 0..max_subscribers {
2112 receivers.push(register(&mut service, unfiltered()).await.unwrap());
2113 }
2114 drain(&mut shards);
2115 assert_eq!(
2116 service.counters.total.load(Ordering::Relaxed),
2117 max_subscribers
2118 );
2119 assert_eq!(service.counters.reserved(), max_subscribers);
2120 assert_eq!(
2121 inflight_subscribers(&service.metrics),
2122 max_subscribers as i64
2123 );
2124 assert!(!shards[0].matcher.is_empty());
2125 assert!(!shards[1].matcher.is_empty());
2126
2127 assert!(register(&mut service, unfiltered()).await.is_none());
2130 assert_eq!(
2131 service.counters.total.load(Ordering::Relaxed),
2132 max_subscribers
2133 );
2134 assert_eq!(service.counters.reserved(), max_subscribers);
2135 assert_eq!(
2136 inflight_subscribers(&service.metrics),
2137 max_subscribers as i64
2138 );
2139 assert_eq!(
2140 terminations(&service.metrics, "checkpoint", "service_shutdown"),
2141 0
2142 );
2143
2144 drop(receivers);
2145 service.handle_checkpoint(checkpoint(1).into()).await;
2146 drain(&mut shards);
2147 assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
2148 assert_eq!(service.counters.reserved(), 0);
2149 assert_eq!(inflight_subscribers(&service.metrics), 0);
2150
2151 let replacement = register(&mut service, unfiltered()).await.unwrap();
2152 drain(&mut shards);
2153 assert_eq!(service.counters.total.load(Ordering::Relaxed), 1);
2154 assert_eq!(service.counters.reserved(), 1);
2155
2156 drop(replacement);
2157 service.handle_checkpoint(checkpoint(2).into()).await;
2158 drain(&mut shards);
2159 assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
2160 assert_eq!(service.counters.reserved(), 0);
2161 }
2162
2163 #[tokio::test]
2164 async fn unprocessed_register_finalizes_guard_when_shard_drops() {
2165 let (mut service, shards) = test_service(1);
2166 let _receiver = register(&mut service, unfiltered()).await.unwrap();
2167 assert_eq!(service.counters.total.load(Ordering::Relaxed), 1);
2168 assert_eq!(inflight_subscribers(&service.metrics), 1);
2169 assert_eq!(service.counters.reserved(), 1);
2170
2171 drop(shards);
2176 assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
2177 assert_eq!(service.counters.reserved(), 0);
2178 assert_eq!(inflight_subscribers(&service.metrics), 0);
2179 assert_eq!(
2180 terminations(&service.metrics, "checkpoint", "service_shutdown"),
2181 1
2182 );
2183 }
2184
2185 #[tokio::test]
2186 async fn registration_racing_dispatch_lands_at_the_next_checkpoint() {
2187 let (mut service, mut shards) = test_service(1);
2188 let mut receiver_a = register(&mut service, unfiltered()).await.unwrap();
2189 service.handle_checkpoint(checkpoint(1).into()).await;
2190 let mut receiver_b = register(&mut service, unfiltered()).await.unwrap();
2191 service.handle_checkpoint(checkpoint(2).into()).await;
2192 drain(&mut shards);
2193
2194 assert_eq!(matched_sequence_number(receiver_a.recv().await.unwrap()), 1);
2197 assert_eq!(matched_sequence_number(receiver_a.recv().await.unwrap()), 2);
2198 assert_eq!(matched_sequence_number(receiver_b.recv().await.unwrap()), 2);
2199 assert!(receiver_b.try_recv().is_err());
2200 }
2201
2202 #[tokio::test]
2203 async fn space_counters_track_filtered_subscribers() {
2204 let (mut service, mut shards) = test_service_with(2, 1, None);
2205 let tx_rx = register(
2206 &mut service,
2207 SubscriptionSpec {
2208 kind: SubscriptionKind::Transactions,
2209 query: Some(sender_query(addr(0), false)),
2210 },
2211 )
2212 .await
2213 .unwrap();
2214 assert_eq!(service.counters.filtered_tx.load(Ordering::Relaxed), 1);
2215 let event_rx = register(
2216 &mut service,
2217 SubscriptionSpec {
2218 kind: SubscriptionKind::Events,
2219 query: Some(event_type_query(
2220 "0x0000000000000000000000000000000000000000000000000000000000000002::coin::CoinEvent",
2221 )),
2222 },
2223 )
2224 .await
2225 .unwrap();
2226 assert_eq!(service.counters.filtered_event.load(Ordering::Relaxed), 1);
2227
2228 drop(tx_rx);
2231 drop(event_rx);
2232 service.handle_checkpoint(checkpoint(1).into()).await;
2233 drain(&mut shards);
2234
2235 assert_eq!(service.counters.filtered_tx.load(Ordering::Relaxed), 0);
2236 assert_eq!(service.counters.filtered_event.load(Ordering::Relaxed), 0);
2237 assert_eq!(inflight_subscribers(&service.metrics), 0);
2238 }
2239
2240 #[tokio::test]
2241 async fn unfiltered_subscribers_receive_all_matches() {
2242 let (mut service, mut shards) = test_service(1);
2243 let mut tx_rx = register(
2244 &mut service,
2245 SubscriptionSpec {
2246 kind: SubscriptionKind::Transactions,
2247 query: None,
2248 },
2249 )
2250 .await
2251 .unwrap();
2252 let mut ev_rx = register(
2253 &mut service,
2254 SubscriptionSpec {
2255 kind: SubscriptionKind::Events,
2256 query: None,
2257 },
2258 )
2259 .await
2260 .unwrap();
2261
2262 let event_checkpoint = checkpoint_with_events(1);
2263 let tx_lo = event_checkpoint.summary.data().network_total_transactions
2264 - event_checkpoint.transactions.len() as u64;
2265 service.handle_checkpoint(event_checkpoint.into()).await;
2266 drain(&mut shards);
2267
2268 assert!(matches!(
2269 tx_rx.recv().await.unwrap(),
2270 SubscriptionUpdate::WatermarkTick {
2271 checkpoint: 0,
2272 tx_hi
2273 } if tx_hi == tx_lo
2274 ));
2275 assert!(matches!(
2276 ev_rx.recv().await.unwrap(),
2277 SubscriptionUpdate::WatermarkTick {
2278 checkpoint: 0,
2279 tx_hi
2280 } if tx_hi == tx_lo
2281 ));
2282
2283 let matched = match tx_rx.recv().await.unwrap() {
2286 SubscriptionUpdate::Matched(matched) => matched,
2287 SubscriptionUpdate::WatermarkTick { .. } => panic!("expected a match, got a tick"),
2288 };
2289 assert!(matches!(
2290 matched.matches,
2291 SubscriptionMatches::AllTransactions
2292 ));
2293 let tx_count = matched.checkpoint.transactions().len() as u32;
2294 assert_eq!(
2295 matched
2296 .matches
2297 .transaction_indices(tx_count)
2298 .unwrap()
2299 .collect::<Vec<_>>(),
2300 vec![0, 1]
2301 );
2302
2303 let matched = match ev_rx.recv().await.unwrap() {
2304 SubscriptionUpdate::Matched(matched) => matched,
2305 SubscriptionUpdate::WatermarkTick { .. } => panic!("expected a match, got a tick"),
2306 };
2307 assert!(matches!(matched.matches, SubscriptionMatches::AllEvents));
2308 assert_eq!(
2309 matched
2310 .matches
2311 .event_indices(&matched.checkpoint)
2312 .unwrap()
2313 .collect::<Vec<_>>(),
2314 vec![(1, 0), (1, 1)]
2315 );
2316
2317 service.handle_checkpoint(checkpoint(2).into()).await;
2320 drain(&mut shards);
2321 assert!(tx_rx.try_recv().is_err());
2322 assert!(ev_rx.try_recv().is_err());
2323 }
2324
2325 #[test]
2326 fn event_indices_flatten_sparse_matches_in_order() {
2327 let matches = SubscriptionMatches::Events(vec![(0, vec![1, 2]), (3, vec![0])]);
2328 let checkpoint = checkpoint(1).into();
2329 assert_eq!(
2330 matches
2331 .event_indices(&checkpoint)
2332 .unwrap()
2333 .collect::<Vec<_>>(),
2334 vec![(0, 1), (0, 2), (3, 0)]
2335 );
2336 }
2337}