Skip to main content

sui_rpc_api/subscription/
mod.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use crate::metrics::SubscriptionMetrics;
5use futures::{StreamExt, stream::FuturesUnordered};
6use std::sync::Arc;
7use std::sync::atomic::AtomicUsize;
8use std::sync::atomic::Ordering;
9use std::time::Duration;
10use sui_inverted_index::BitmapQuery;
11use sui_types::full_checkpoint_content::Checkpoint;
12use sui_types::full_checkpoint_content::ExecutedTransaction;
13use sui_types::full_checkpoint_content::LocallyBuiltCheckpoint;
14use sui_types::full_checkpoint_content::ObjectSet;
15use sui_types::messages_checkpoint::CheckpointSummary;
16use tokio::sync::broadcast;
17use tokio::sync::mpsc;
18use tokio::sync::oneshot;
19use tokio::time::Instant;
20use tokio::time::sleep;
21use tracing::info;
22use tracing::trace;
23use tracing::warn;
24
25mod matcher;
26
27const CHECKPOINT_MAILBOX_SIZE: usize = 1024;
28/// Amortizes admission select/receive overhead while bounding how many
29/// requests can delay the next checkpoint poll.
30const ADMISSION_TURN_LIMIT: usize = 128;
31const SUBSCRIPTION_CHANNEL_SIZE: usize = 256;
32const DEFAULT_MAX_SUBSCRIBERS: usize = 1024;
33/// Bound on each shard task's mailbox (registrations, checkpoint fan-out,
34/// and lag teardowns from the dispatcher).
35const SHARD_MAILBOX_SIZE: usize = 64;
36
37/// Default for [`SubscriptionService::build`]'s `watermark_interval`: ~5
38/// seconds at mainnet checkpoint cadence.
39const DEFAULT_WATERMARK_INTERVAL: u32 = 25;
40
41/// Default for [`SubscriptionService::build`]'s `shards`: the host's
42/// available parallelism, floor 1.
43fn default_shards() -> u32 {
44    std::thread::available_parallelism()
45        .map(|n| n.get() as u32)
46        .unwrap_or(1)
47}
48
49/// Poll interval while waiting for the index to catch up to a checkpoint
50/// before delivering it (see [`IndexedCheckpointFn`]).
51const INDEX_WAIT_POLL_INTERVAL: Duration = Duration::from_millis(10);
52/// Upper bound on how long delivery of a single checkpoint waits for the
53/// index. A healthy index catches up in milliseconds; the bound just keeps
54/// a stalled indexer from wedging the subscription stream forever.
55const INDEX_WAIT_TIMEOUT: Duration = Duration::from_secs(10);
56
57/// Reads the highest checkpoint the index has committed (the embedded
58/// rpc-store's live-cohort watermark), or `None` if it has indexed nothing
59/// yet. When supplied to [`SubscriptionService::build`], the service holds a
60/// checkpoint back until the index has committed it, so a client that
61/// observes a checkpoint (e.g. via `execute_transaction_and_wait_for_checkpoint`)
62/// can immediately read that checkpoint's indexed state.
63pub type IndexedCheckpointFn = Arc<dyn Fn() -> Option<u64> + Send + Sync>;
64
65/// Which item stream a subscriber asked for.
66#[derive(Clone, Copy, Debug, PartialEq, Eq)]
67pub enum SubscriptionKind {
68    Checkpoints,
69    Transactions,
70    Events,
71}
72
73impl SubscriptionKind {
74    pub(crate) fn metric_label(self) -> &'static str {
75        match self {
76            Self::Checkpoints => "checkpoint",
77            Self::Transactions => "transaction",
78            Self::Events => "event",
79        }
80    }
81}
82
83#[derive(Clone, Copy)]
84pub(crate) enum SubscriptionTerminationReason {
85    ClientClosed,
86    SlowConsumer,
87    SourceLag,
88    /// The locally built checkpoint stream did not continue from the last
89    /// checkpoint delivered (see
90    /// [`SubscriptionService::build_for_locally_built_checkpoints`]).
91    SourceGap,
92    ServiceShutdown,
93}
94
95impl SubscriptionTerminationReason {
96    fn metric_label(self) -> &'static str {
97        match self {
98            Self::ClientClosed => "client_closed",
99            Self::SlowConsumer => "slow_consumer",
100            Self::SourceLag => "source_lag",
101            Self::SourceGap => "source_gap",
102            Self::ServiceShutdown => "service_shutdown",
103        }
104    }
105}
106
107pub(crate) struct SubscriptionLifecycleGuard {
108    kind: SubscriptionKind,
109    filtered: bool,
110    reservation: SubscriberReservation,
111    inflight_subscribers: prometheus::IntGauge,
112    terminations_total: prometheus::IntCounterVec,
113    termination_reason: SubscriptionTerminationReason,
114}
115
116impl SubscriptionLifecycleGuard {
117    pub(crate) fn new(
118        kind: SubscriptionKind,
119        filtered: bool,
120        reservation: SubscriberReservation,
121        metrics: &SubscriptionMetrics,
122    ) -> Self {
123        reservation.increment_resident_counts(kind, filtered);
124        let inflight_subscribers = metrics
125            .inflight_subscribers
126            .with_label_values(&[kind.metric_label(), if filtered { "true" } else { "false" }]);
127        inflight_subscribers.inc();
128
129        Self {
130            kind,
131            filtered,
132            reservation,
133            inflight_subscribers,
134            terminations_total: metrics.terminations_total.clone(),
135            termination_reason: SubscriptionTerminationReason::ServiceShutdown,
136        }
137    }
138
139    /// Finalizes the subscription now, recording `reason` instead of the
140    /// default `service_shutdown`: consuming `self` runs `Drop`, which
141    /// decrements the counts/gauge and increments the termination counter.
142    pub(crate) fn terminate(mut self, reason: SubscriptionTerminationReason) {
143        self.termination_reason = reason;
144    }
145}
146
147impl Drop for SubscriptionLifecycleGuard {
148    fn drop(&mut self) {
149        self.reservation
150            .decrement_resident_counts(self.kind, self.filtered);
151        self.inflight_subscribers.dec();
152        self.terminations_total
153            .with_label_values(&[
154                self.kind.metric_label(),
155                self.termination_reason.metric_label(),
156            ])
157            .inc();
158    }
159}
160
161/// What a subscriber asked for. `query: None` = unfiltered (stream
162/// everything).
163pub struct SubscriptionSpec {
164    pub kind: SubscriptionKind,
165    pub query: Option<BitmapQuery>,
166}
167
168/// Updates delivered to a subscriber while processing checkpoints. An initial
169/// progress frame may precede a matched frame for the entry checkpoint.
170pub enum SubscriptionUpdate {
171    Matched(MatchedCheckpoint),
172    /// An initial tick identifies the safe position immediately before
173    /// subscription entry: `checkpoint` is the entry checkpoint minus one and
174    /// `tx_hi` is the entry checkpoint's transaction lower bound. A periodic
175    /// tick identifies a fully processed `checkpoint`; `tx_hi` is its
176    /// `network_total_transactions` (the exclusive transaction upper bound).
177    WatermarkTick {
178        checkpoint: u64,
179        tx_hi: u64,
180    },
181}
182
183pub struct MatchedCheckpoint {
184    pub checkpoint: SubscriptionCheckpoint,
185    pub matches: SubscriptionMatches,
186}
187
188/// A checkpoint the subscription service streams to its subscribers.
189#[derive(Clone)]
190pub enum SubscriptionCheckpoint {
191    /// Certified by a quorum of validators and executed by this node.
192    Certified(Arc<Checkpoint>),
193    /// Built by this validator before the network certified it, so it has no
194    /// quorum signature.
195    LocallyBuilt(Arc<LocallyBuiltCheckpoint>),
196}
197
198impl SubscriptionCheckpoint {
199    pub fn summary(&self) -> &CheckpointSummary {
200        match self {
201            Self::Certified(checkpoint) => checkpoint.summary.data(),
202            Self::LocallyBuilt(checkpoint) => &checkpoint.summary,
203        }
204    }
205
206    pub fn transactions(&self) -> &[ExecutedTransaction] {
207        match self {
208            Self::Certified(checkpoint) => &checkpoint.transactions,
209            Self::LocallyBuilt(checkpoint) => &checkpoint.transactions,
210        }
211    }
212
213    pub fn object_set(&self) -> &ObjectSet {
214        match self {
215            Self::Certified(checkpoint) => &checkpoint.object_set,
216            Self::LocallyBuilt(checkpoint) => &checkpoint.object_set,
217        }
218    }
219}
220
221impl From<Arc<Checkpoint>> for SubscriptionCheckpoint {
222    fn from(checkpoint: Arc<Checkpoint>) -> Self {
223        Self::Certified(checkpoint)
224    }
225}
226
227impl From<Arc<LocallyBuiltCheckpoint>> for SubscriptionCheckpoint {
228    fn from(checkpoint: Arc<LocallyBuiltCheckpoint>) -> Self {
229        Self::LocallyBuilt(checkpoint)
230    }
231}
232
233/// The broadcast stream a [`SubscriptionService`] reads checkpoints from.
234enum CheckpointMailbox {
235    Certified(broadcast::Receiver<Arc<Checkpoint>>),
236    LocallyBuilt(broadcast::Receiver<Arc<LocallyBuiltCheckpoint>>),
237}
238
239impl CheckpointMailbox {
240    /// Cancel-safe, like [`broadcast::Receiver::recv`].
241    async fn recv(&mut self) -> Result<SubscriptionCheckpoint, broadcast::error::RecvError> {
242        match self {
243            Self::Certified(receiver) => receiver.recv().await.map(SubscriptionCheckpoint::from),
244            Self::LocallyBuilt(receiver) => receiver.recv().await.map(SubscriptionCheckpoint::from),
245        }
246    }
247}
248
249/// Kind-specific match payload. Indices are within-checkpoint and ascending.
250pub enum SubscriptionMatches {
251    /// Checkpoint subscription: the checkpoint matched (some tx satisfied the
252    /// filter, or the subscription is unfiltered).
253    Checkpoint,
254    /// Transaction subscription: matched transaction indices, run-length
255    /// encoded as half-open ranges -- ascending, non-overlapping, and
256    /// maximally coalesced. Bounds the payload at O(runs) for
257    /// densely-matching filters (e.g. unanchored negation).
258    Transactions(Vec<std::ops::Range<u32>>),
259    /// Transaction subscription without a filter: every transaction in the
260    /// checkpoint matched. O(1) representation of "all"; never constructed
261    /// for a checkpoint with no transactions.
262    AllTransactions,
263    /// Event subscription: per matched transaction index, matched event
264    /// indices.
265    Events(Vec<(u32, Vec<u32>)>),
266    /// Event subscription without a filter: every event in the checkpoint
267    /// matched. O(1) representation of "all"; never constructed for a
268    /// checkpoint with no events.
269    AllEvents,
270}
271
272impl SubscriptionMatches {
273    /// Matched transaction indices of a transaction-subscription payload,
274    /// ascending; `None` for other payload kinds. `tx_count` is the
275    /// checkpoint's transaction count, used to expand
276    /// [`Self::AllTransactions`].
277    pub fn transaction_indices(
278        &self,
279        tx_count: u32,
280    ) -> Option<Box<dyn Iterator<Item = u32> + Send + '_>> {
281        match self {
282            Self::Transactions(ranges) => Some(Box::new(ranges.iter().flat_map(Clone::clone))),
283            Self::AllTransactions => Some(Box::new(0..tx_count)),
284            _ => None,
285        }
286    }
287
288    /// Matched `(transaction index, event index)` pairs of an
289    /// event-subscription payload, ascending; `None` for other payload
290    /// kinds. `checkpoint` is used to expand [`Self::AllEvents`].
291    pub fn event_indices<'a>(
292        &'a self,
293        checkpoint: &'a SubscriptionCheckpoint,
294    ) -> Option<Box<dyn Iterator<Item = (u32, u32)> + Send + 'a>> {
295        match self {
296            Self::Events(txs) => Some(Box::new(
297                txs.iter()
298                    .flat_map(|(tx, evs)| evs.iter().map(move |&ev| (*tx, ev))),
299            )),
300            Self::AllEvents => Some(Box::new(
301                checkpoint
302                    .transactions()
303                    .iter()
304                    .enumerate()
305                    .flat_map(|(tx_idx, tx)| {
306                        let event_count =
307                            tx.events.as_ref().map(|e| e.data.len()).unwrap_or(0) as u32;
308                        (0..event_count).map(move |ev| (tx_idx as u32, ev))
309                    }),
310            )),
311            _ => None,
312        }
313    }
314}
315
316struct SubscriptionRequest {
317    spec: SubscriptionSpec,
318    response_sender: oneshot::Sender<mpsc::Receiver<SubscriptionUpdate>>,
319    reservation: SubscriberReservation,
320}
321
322/// Every request owns a reservation, so pending plus resident subscriptions
323/// remain bounded by the configured subscriber limit without a transport bound.
324#[allow(clippy::disallowed_methods)]
325fn subscription_admission_channel() -> (
326    mpsc::UnboundedSender<SubscriptionRequest>,
327    mpsc::UnboundedReceiver<SubscriptionRequest>,
328) {
329    mpsc::unbounded_channel()
330}
331
332enum AdmissionState {
333    Accepting,
334    WaitingForShard(SubscriptionRequest),
335    Closed,
336}
337
338enum SubscriptionServiceEvent {
339    Checkpoint(Result<SubscriptionCheckpoint, broadcast::error::RecvError>),
340    AdmissionRequest(SubscriptionRequest),
341    AdmissionClosed,
342    ShardCapacity {
343        shard: usize,
344        permit: mpsc::OwnedPermit<ShardMsg>,
345    },
346    AdmissionCanceled,
347}
348
349#[derive(Clone)]
350pub struct SubscriptionServiceHandle {
351    admission_sender: mpsc::UnboundedSender<SubscriptionRequest>,
352    counters: Arc<SubscriberCounts>,
353    metrics: SubscriptionMetrics,
354}
355
356impl SubscriptionServiceHandle {
357    pub async fn register_subscription(
358        &self,
359        spec: SubscriptionSpec,
360    ) -> Option<mpsc::Receiver<SubscriptionUpdate>> {
361        let reservation = match self.counters.try_reserve() {
362            Some(reservation) => reservation,
363            None => {
364                trace!(
365                    "failed to register new subscriber: hit maximum number of subscribers {}",
366                    self.counters.limit
367                );
368                return None;
369            }
370        };
371
372        let (response_sender, response_receiver) = oneshot::channel();
373        let request = SubscriptionRequest {
374            spec,
375            response_sender,
376            reservation,
377        };
378        self.admission_sender.send(request).ok()?;
379
380        response_receiver.await.ok()
381    }
382
383    pub(crate) fn stream_metrics(
384        &self,
385        kind: SubscriptionKind,
386    ) -> crate::metrics::SubscriptionStreamMetrics {
387        self.metrics.stream_metrics(kind.metric_label())
388    }
389}
390
391/// Shared subscription admission and lifecycle accounting.
392///
393/// `reserved` is the admission authority and counts pending, in-flight, and
394/// resident subscriptions. `total` and the filtered counters track resident
395/// subscriptions through their lifecycle.
396pub(crate) struct SubscriberCounts {
397    limit: usize,
398    reserved: AtomicUsize,
399    total: AtomicUsize,
400    filtered_tx: AtomicUsize,
401    filtered_event: AtomicUsize,
402}
403
404impl SubscriberCounts {
405    fn new(limit: usize) -> Self {
406        Self {
407            limit,
408            reserved: AtomicUsize::new(0),
409            total: AtomicUsize::new(0),
410            filtered_tx: AtomicUsize::new(0),
411            filtered_event: AtomicUsize::new(0),
412        }
413    }
414
415    fn try_reserve(self: &Arc<Self>) -> Option<SubscriberReservation> {
416        self.reserved
417            .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |reserved| {
418                (reserved < self.limit).then_some(reserved + 1)
419            })
420            .ok()?;
421        Some(SubscriberReservation {
422            counters: Arc::clone(self),
423        })
424    }
425
426    #[cfg(test)]
427    fn reserved(&self) -> usize {
428        self.reserved.load(Ordering::Relaxed)
429    }
430}
431
432pub(crate) struct SubscriberReservation {
433    counters: Arc<SubscriberCounts>,
434}
435
436impl SubscriberReservation {
437    fn increment_resident_counts(&self, kind: SubscriptionKind, filtered: bool) {
438        self.counters.total.fetch_add(1, Ordering::Relaxed);
439        if filtered {
440            match kind {
441                SubscriptionKind::Checkpoints | SubscriptionKind::Transactions => {
442                    self.counters.filtered_tx.fetch_add(1, Ordering::Relaxed);
443                }
444                SubscriptionKind::Events => {
445                    self.counters.filtered_event.fetch_add(1, Ordering::Relaxed);
446                }
447            }
448        }
449    }
450
451    fn decrement_resident_counts(&self, kind: SubscriptionKind, filtered: bool) {
452        self.counters.total.fetch_sub(1, Ordering::Relaxed);
453        if filtered {
454            match kind {
455                SubscriptionKind::Checkpoints | SubscriptionKind::Transactions => {
456                    self.counters.filtered_tx.fetch_sub(1, Ordering::Relaxed);
457                }
458                SubscriptionKind::Events => {
459                    self.counters.filtered_event.fetch_sub(1, Ordering::Relaxed);
460                }
461            }
462        }
463    }
464}
465
466impl Drop for SubscriberReservation {
467    fn drop(&mut self) {
468        let previous = self.counters.reserved.fetch_sub(1, Ordering::Relaxed);
469        debug_assert!(previous > 0, "subscriber reservation count underflow");
470    }
471}
472
473enum ShardMsg {
474    /// A checkpoint plus its pre-extracted dimension keys.
475    Checkpoint(SubscriptionCheckpoint, Arc<matcher::CheckpointKeys>),
476    /// An admitted subscriber (handshake already completed by the dispatcher).
477    Register {
478        spec: SubscriptionSpec,
479        sender: mpsc::Sender<SubscriptionUpdate>,
480        guard: SubscriptionLifecycleGuard,
481    },
482    /// Drop every subscriber on this shard with the supplied bounded reason.
483    Clear(SubscriptionTerminationReason),
484}
485
486/// One worker task owning a partition of the subscribers: it evaluates their
487/// filters against pre-extracted checkpoint keys and delivers their updates.
488struct SubscriptionShard {
489    mailbox: mpsc::Receiver<ShardMsg>,
490    matcher: matcher::SubscriptionMatcher,
491    /// Checkpoints a subscriber may go without any frame before a standalone
492    /// watermark tick is delivered (see `RpcConfig::subscription_watermark_interval`).
493    watermark_interval: u32,
494}
495
496impl SubscriptionShard {
497    async fn run(mut self) {
498        while let Some(msg) = self.mailbox.recv().await {
499            self.handle_msg(msg);
500        }
501        self.matcher
502            .clear(SubscriptionTerminationReason::ServiceShutdown);
503    }
504
505    fn handle_msg(&mut self, msg: ShardMsg) {
506        match msg {
507            ShardMsg::Register {
508                spec,
509                sender,
510                guard,
511            } => {
512                self.matcher.insert(spec, sender, guard);
513            }
514            ShardMsg::Checkpoint(checkpoint, keys) => {
515                self.matcher
516                    .dispatch_with_keys(&checkpoint, &keys, self.watermark_interval);
517            }
518            ShardMsg::Clear(reason) => {
519                self.matcher.clear(reason);
520            }
521        }
522    }
523
524    /// Synchronously process every message already in the mailbox: a
525    /// test-only stand-in for the spawned `run` loop that keeps the actor
526    /// tests deterministic.
527    #[cfg(test)]
528    fn drain(&mut self) {
529        while let Ok(msg) = self.mailbox.try_recv() {
530            self.handle_msg(msg);
531        }
532    }
533}
534
535pub struct SubscriptionService {
536    // Broadcast receiver for the checkpoints this service streams: certified
537    // checkpoints published by the Checkpoint Executor, or locally built ones
538    // published by a validator's checkpoint builder.
539    //
540    // Publishers send non-blocking, so a slow service can fall behind and
541    // observe `RecvError::Lagged`; checkpoints delivered between lags arrive
542    // in-order.
543    checkpoint_mailbox: CheckpointMailbox,
544    admission_mailbox: mpsc::UnboundedReceiver<SubscriptionRequest>,
545    /// Registration targets: one mailbox per shard task, each owning a
546    /// partition of the subscribers.
547    shards: Vec<mpsc::Sender<ShardMsg>>,
548    /// Rotating tie-break cursor for shards with equal free mailbox capacity.
549    next_shard: usize,
550    /// Filtered-subscriber counts per key space, shared with the shards;
551    /// gates per-checkpoint key extraction.
552    counters: Arc<SubscriberCounts>,
553
554    // When set, delivery of a checkpoint waits until the index has committed
555    // it (see [`IndexedCheckpointFn`]). `None` preserves the immediate-delivery
556    // behavior used with the legacy synchronously-committed index.
557    indexed_checkpoint: Option<IndexedCheckpointFn>,
558
559    metrics: SubscriptionMetrics,
560}
561
562impl SubscriptionService {
563    /// Builds a service that streams certified checkpoints, which the
564    /// checkpoint executor publishes in sequence order without gaps.
565    ///
566    /// `None` defaults `watermark_interval` to 25 checkpoints,
567    /// `max_subscribers` to 1024, and `shards` to the host's available
568    /// parallelism, with a minimum of one shard.
569    pub fn build(
570        registry: &prometheus::Registry,
571        indexed_checkpoint: Option<IndexedCheckpointFn>,
572        watermark_interval: Option<u32>,
573        max_subscribers: Option<usize>,
574        shards: Option<u32>,
575    ) -> (
576        broadcast::Sender<Arc<Checkpoint>>,
577        SubscriptionServiceHandle,
578    ) {
579        let (checkpoint_sender, checkpoint_mailbox) = broadcast::channel(CHECKPOINT_MAILBOX_SIZE);
580        let handle = Self::spawn(
581            registry,
582            CheckpointMailbox::Certified(checkpoint_mailbox),
583            indexed_checkpoint,
584            watermark_interval,
585            max_subscribers,
586            shards,
587        );
588        (checkpoint_sender, handle)
589    }
590
591    /// Builds a service that streams the checkpoints a validator builds or
592    /// executes, usually before the network certifies them. Publishers send
593    /// these in increasing sequence order, but may skip sequence numbers: the
594    /// validator's publisher drops a checkpoint when it falls behind or fails
595    /// to load one. A skip drops every subscriber, as a lag does, since none of
596    /// them can receive a gap-free stream any more.
597    ///
598    /// The optional parameters default as in [`Self::build`].
599    pub fn build_for_locally_built_checkpoints(
600        registry: &prometheus::Registry,
601        watermark_interval: Option<u32>,
602        max_subscribers: Option<usize>,
603        shards: Option<u32>,
604    ) -> (
605        broadcast::Sender<Arc<LocallyBuiltCheckpoint>>,
606        SubscriptionServiceHandle,
607    ) {
608        let (checkpoint_sender, checkpoint_mailbox) = broadcast::channel(CHECKPOINT_MAILBOX_SIZE);
609        let handle = Self::spawn(
610            registry,
611            CheckpointMailbox::LocallyBuilt(checkpoint_mailbox),
612            None,
613            watermark_interval,
614            max_subscribers,
615            shards,
616        );
617        (checkpoint_sender, handle)
618    }
619
620    fn spawn(
621        registry: &prometheus::Registry,
622        checkpoint_mailbox: CheckpointMailbox,
623        indexed_checkpoint: Option<IndexedCheckpointFn>,
624        watermark_interval: Option<u32>,
625        max_subscribers: Option<usize>,
626        shards: Option<u32>,
627    ) -> SubscriptionServiceHandle {
628        let metrics = SubscriptionMetrics::new(registry);
629        let max_subscribers = max_subscribers.unwrap_or(DEFAULT_MAX_SUBSCRIBERS);
630        let (admission_sender, admission_mailbox) = subscription_admission_channel();
631        let counters = Arc::new(SubscriberCounts::new(max_subscribers));
632        let handle = SubscriptionServiceHandle {
633            admission_sender,
634            counters: Arc::clone(&counters),
635            metrics: metrics.clone(),
636        };
637
638        let watermark_interval = watermark_interval
639            .unwrap_or(DEFAULT_WATERMARK_INTERVAL)
640            .max(1);
641        let shards = shards.unwrap_or_else(default_shards).max(1);
642        let mut shard_senders = Vec::with_capacity(shards as usize);
643        for _ in 0..shards {
644            let (sender, shard_mailbox) = mpsc::channel(SHARD_MAILBOX_SIZE);
645            tokio::spawn(
646                SubscriptionShard {
647                    mailbox: shard_mailbox,
648                    matcher: matcher::SubscriptionMatcher::default(),
649                    watermark_interval,
650                }
651                .run(),
652            );
653            shard_senders.push(sender);
654        }
655
656        tokio::spawn(
657            Self {
658                checkpoint_mailbox,
659                admission_mailbox,
660                shards: shard_senders,
661                next_shard: 0,
662                counters,
663                indexed_checkpoint,
664                metrics,
665            }
666            .start(),
667        );
668
669        handle
670    }
671
672    async fn start(mut self) {
673        let mut admission_state = AdmissionState::Accepting;
674        loop {
675            let event = match &mut admission_state {
676                AdmissionState::Accepting => {
677                    tokio::select! {
678                        biased;
679
680                        result = self.checkpoint_mailbox.recv() => {
681                            SubscriptionServiceEvent::Checkpoint(result)
682                        },
683                        request = self.admission_mailbox.recv() => {
684                            match request {
685                                Some(request) => {
686                                    SubscriptionServiceEvent::AdmissionRequest(request)
687                                }
688                                None => SubscriptionServiceEvent::AdmissionClosed,
689                            }
690                        },
691                    }
692                }
693                AdmissionState::WaitingForShard(request) => {
694                    let shards: Vec<_> = (0..self.shards.len())
695                        .map(|offset| {
696                            let shard = (self.next_shard + offset) % self.shards.len();
697                            (shard, self.shards[shard].clone())
698                        })
699                        .collect();
700
701                    tokio::select! {
702                        biased;
703
704                        result = self.checkpoint_mailbox.recv() => {
705                            SubscriptionServiceEvent::Checkpoint(result)
706                        },
707                        _ = request.response_sender.closed() => {
708                            SubscriptionServiceEvent::AdmissionCanceled
709                        },
710                        (shard, permit) = Self::reserve_first_available_shard(shards) => {
711                            SubscriptionServiceEvent::ShardCapacity { shard, permit }
712                        },
713                    }
714                }
715                AdmissionState::Closed => {
716                    SubscriptionServiceEvent::Checkpoint(self.checkpoint_mailbox.recv().await)
717                }
718            };
719
720            match event {
721                SubscriptionServiceEvent::Checkpoint(Ok(checkpoint)) => {
722                    self.handle_checkpoint(checkpoint).await;
723                }
724                SubscriptionServiceEvent::Checkpoint(Err(broadcast::error::RecvError::Lagged(
725                    skipped,
726                ))) => {
727                    self.handle_lag(skipped).await;
728                }
729                // Once the publisher drops the sender this yields `Closed`
730                // and we can terminate the event loop.
731                SubscriptionServiceEvent::Checkpoint(Err(broadcast::error::RecvError::Closed)) => {
732                    break;
733                }
734                SubscriptionServiceEvent::AdmissionRequest(request) => {
735                    admission_state = self.admit_ready_requests(request);
736                }
737                SubscriptionServiceEvent::AdmissionClosed => {
738                    // Established subscribers remain live until the checkpoint
739                    // source closes.
740                    admission_state = AdmissionState::Closed;
741                }
742                SubscriptionServiceEvent::ShardCapacity { shard, permit } => {
743                    let AdmissionState::WaitingForShard(request) = admission_state else {
744                        unreachable!("shard capacity requires a waiting admission");
745                    };
746                    self.complete_admission(shard, permit, request);
747                    admission_state = AdmissionState::Accepting;
748                }
749                SubscriptionServiceEvent::AdmissionCanceled => {
750                    let AdmissionState::WaitingForShard(_) = admission_state else {
751                        unreachable!("admission cancellation requires a waiting admission");
752                    };
753                    admission_state = AdmissionState::Accepting;
754                }
755            }
756        }
757
758        for shard in &self.shards {
759            shard
760                .send(ShardMsg::Clear(
761                    SubscriptionTerminationReason::ServiceShutdown,
762                ))
763                .await
764                .expect("subscription shard terminated unexpectedly");
765        }
766
767        info!("RPC Subscription Services ended");
768    }
769
770    async fn handle_checkpoint(&mut self, checkpoint: SubscriptionCheckpoint) {
771        let sequence_number = checkpoint.summary().sequence_number;
772
773        // Check that we recieved checkpoints in-order. The broadcast stream
774        // preserves send order, and any gap surfaces separately as `Lagged`
775        // (see `handle_lag`), so reaching here out-of-order indicates a
776        // publisher bug. A locally built stream may also skip ahead (see
777        // `build_for_locally_built_checkpoints`), and runs on a validator,
778        // which must not abort over a streaming bug: any break in it drops
779        // the subscribers instead of panicking.
780        {
781            let last_sequence_number = self.metrics.last_recieved_checkpoint.get();
782            let expected_sequence_number = last_sequence_number + 1;
783
784            if last_sequence_number != 0 && expected_sequence_number != sequence_number as i64 {
785                match checkpoint {
786                    SubscriptionCheckpoint::LocallyBuilt(_) => {
787                        warn!(
788                            expected = expected_sequence_number,
789                            received = sequence_number,
790                            "locally built checkpoint stream broke sequence; \
791                             dropping all in-progress subscriptions"
792                        );
793                        self.clear_subscribers(SubscriptionTerminationReason::SourceGap)
794                            .await;
795                    }
796                    SubscriptionCheckpoint::Certified(_) => panic!(
797                        "recieved checkpoint out-of-order. expected checkpoint {}, recieved {}",
798                        expected_sequence_number, sequence_number
799                    ),
800                }
801            }
802
803            // Update the metric marking the latest checkpoint we've seen
804            self.metrics
805                .last_recieved_checkpoint
806                .set(sequence_number as i64);
807        }
808
809        // Hold the checkpoint back until the index has committed it, so a
810        // client that observes this checkpoint can immediately read its
811        // indexed state. No-op unless an index gate was configured.
812        self.wait_until_indexed(sequence_number).await;
813
814        // No live subscriber or admitted registration pending in a shard:
815        // skip fan-out. Requests merely queued in the admission lane are not
816        // counted and may establish their stream boundary after this checkpoint.
817        if self.counters.total.load(Ordering::Relaxed) == 0 {
818            return;
819        }
820
821        // Extract the checkpoint's dimension keys once, only for the key
822        // spaces with at least one filtered subscriber, then fan the
823        // checkpoint out to every shard. The bounded sends give backpressure:
824        // one slow shard stalls the dispatcher, which lags the broadcast
825        // receiver and tears everything down (see `handle_lag`).
826        let keys = Arc::new(matcher::extract_checkpoint_keys(
827            &checkpoint,
828            self.counters.filtered_tx.load(Ordering::Relaxed) > 0,
829            self.counters.filtered_event.load(Ordering::Relaxed) > 0,
830        ));
831        for shard in &self.shards {
832            shard
833                .send(ShardMsg::Checkpoint(checkpoint.clone(), Arc::clone(&keys)))
834                .await
835                .expect("subscription shard terminated unexpectedly");
836        }
837    }
838
839    /// Block until the index has committed `sequence_number`, polling the
840    /// configured [`IndexedCheckpointFn`]. Returns immediately when no gate is
841    /// configured or the index is already caught up. Gives up after
842    /// [`INDEX_WAIT_TIMEOUT`] -- a stalled indexer should not wedge delivery
843    /// forever -- delivering a (possibly not-yet-indexed) checkpoint rather
844    /// than stalling the stream.
845    async fn wait_until_indexed(&self, sequence_number: u64) {
846        let Some(indexed) = &self.indexed_checkpoint else {
847            return;
848        };
849
850        if indexed().is_some_and(|hi| hi >= sequence_number) {
851            return;
852        }
853
854        let wait_started = Instant::now();
855        let deadline = wait_started + INDEX_WAIT_TIMEOUT;
856        loop {
857            sleep(INDEX_WAIT_POLL_INTERVAL).await;
858            if indexed().is_some_and(|hi| hi >= sequence_number) {
859                self.metrics
860                    .index_wait_seconds
861                    .observe(wait_started.elapsed().as_secs_f64());
862                return;
863            }
864            if Instant::now() >= deadline {
865                self.metrics.index_wait_timeouts_total.inc();
866                self.metrics
867                    .index_wait_seconds
868                    .observe(wait_started.elapsed().as_secs_f64());
869                warn!(
870                    checkpoint = sequence_number,
871                    "index did not catch up within {INDEX_WAIT_TIMEOUT:?}; \
872                     delivering checkpoint anyway"
873                );
874                return;
875            }
876        }
877    }
878
879    /// Drop every in-progress subscription after the service fell behind the
880    /// broadcast stream. Having missed `skipped` checkpoints we can no longer
881    /// deliver an in-order, gap-free stream to any subscriber, and clients
882    /// already tolerate connection breaks and reconnect, so tearing them all
883    /// down is cheaper than trying to resynchronize each one.
884    async fn handle_lag(&mut self, skipped: u64) {
885        warn!(
886            skipped,
887            "subscription service lagged behind the checkpoint stream; \
888             dropping all in-progress subscriptions"
889        );
890        self.clear_subscribers(SubscriptionTerminationReason::SourceLag)
891            .await;
892        // The next delivered checkpoint jumps ahead by `skipped`; reset the
893        // in-order tracker so it is not mistaken for an out-of-order delivery.
894        self.metrics.last_recieved_checkpoint.set(0);
895    }
896
897    /// Drop every in-progress subscription before the next checkpoint, which
898    /// does not follow the last one delivered.
899    async fn clear_subscribers(&mut self, reason: SubscriptionTerminationReason) {
900        // Per-shard FIFO ordering guarantees no shard delivers a post-gap
901        // checkpoint to a pre-gap subscriber: Clear is enqueued behind all
902        // pre-gap checkpoints and ahead of all post-gap ones.
903        for shard in &self.shards {
904            shard
905                .send(ShardMsg::Clear(reason))
906                .await
907                .expect("subscription shard terminated unexpectedly");
908        }
909    }
910
911    fn try_reserve_least_backlogged_shard(&self) -> Option<(usize, mpsc::OwnedPermit<ShardMsg>)> {
912        let mut selected_shard = None;
913        let mut greatest_capacity = 0;
914
915        for offset in 0..self.shards.len() {
916            let shard = (self.next_shard + offset) % self.shards.len();
917            let sender = &self.shards[shard];
918            if sender.is_closed() {
919                panic!("subscription shard terminated unexpectedly");
920            }
921
922            let capacity = sender.capacity();
923            if selected_shard.is_none() || capacity > greatest_capacity {
924                selected_shard = Some(shard);
925                greatest_capacity = capacity;
926            }
927        }
928
929        let shard = selected_shard.expect("subscription service requires at least one shard");
930        match self.shards[shard].clone().try_reserve_owned() {
931            Ok(permit) => Some((shard, permit)),
932            Err(mpsc::error::TrySendError::Full(_)) => None,
933            Err(mpsc::error::TrySendError::Closed(_)) => {
934                panic!("subscription shard terminated unexpectedly")
935            }
936        }
937    }
938
939    async fn reserve_first_available_shard(
940        shards: Vec<(usize, mpsc::Sender<ShardMsg>)>,
941    ) -> (usize, mpsc::OwnedPermit<ShardMsg>) {
942        assert!(
943            !shards.is_empty(),
944            "subscription service requires at least one shard"
945        );
946        for (_, sender) in &shards {
947            if sender.is_closed() {
948                panic!("subscription shard terminated unexpectedly");
949            }
950        }
951
952        let mut reservations = FuturesUnordered::new();
953        for (shard, sender) in shards {
954            reservations.push(async move { (shard, sender.reserve_owned().await) });
955        }
956
957        match reservations
958            .next()
959            .await
960            .expect("subscription service requires at least one shard")
961        {
962            (shard, Ok(permit)) => (shard, permit),
963            (_, Err(_)) => panic!("subscription shard terminated unexpectedly"),
964        }
965    }
966
967    fn complete_admission(
968        &mut self,
969        shard: usize,
970        permit: mpsc::OwnedPermit<ShardMsg>,
971        request: SubscriptionRequest,
972    ) {
973        if request.response_sender.is_closed() {
974            trace!("failed to register new subscriber: request was cancelled");
975            return;
976        }
977
978        let (sender, receiver) = mpsc::channel(SUBSCRIPTION_CHANNEL_SIZE);
979        if request.response_sender.send(receiver).is_err() {
980            trace!("failed to register new subscriber: request was cancelled");
981            return;
982        }
983
984        trace!("successfully registered new subscriber");
985        let kind = request.spec.kind;
986        let filtered = request.spec.query.is_some();
987        let guard =
988            SubscriptionLifecycleGuard::new(kind, filtered, request.reservation, &self.metrics);
989        permit.send(ShardMsg::Register {
990            spec: request.spec,
991            sender,
992            guard,
993        });
994        self.next_shard = (shard + 1) % self.shards.len();
995    }
996
997    fn try_admit(&mut self, request: SubscriptionRequest) -> AdmissionState {
998        if request.response_sender.is_closed() {
999            trace!("failed to register new subscriber: request was cancelled");
1000            return AdmissionState::Accepting;
1001        }
1002
1003        let Some((shard, permit)) = self.try_reserve_least_backlogged_shard() else {
1004            trace!("waiting for a subscription shard to have capacity");
1005            return AdmissionState::WaitingForShard(request);
1006        };
1007
1008        self.complete_admission(shard, permit, request);
1009        AdmissionState::Accepting
1010    }
1011
1012    fn admit_ready_requests(&mut self, first_request: SubscriptionRequest) -> AdmissionState {
1013        let mut request = first_request;
1014        let mut remaining_attempts = ADMISSION_TURN_LIMIT;
1015
1016        loop {
1017            match self.try_admit(request) {
1018                AdmissionState::Accepting => {}
1019                state => return state,
1020            }
1021
1022            remaining_attempts -= 1;
1023            if remaining_attempts == 0 {
1024                return AdmissionState::Accepting;
1025            }
1026
1027            request = match self.admission_mailbox.try_recv() {
1028                Ok(request) => request,
1029                Err(mpsc::error::TryRecvError::Empty) => {
1030                    return AdmissionState::Accepting;
1031                }
1032                Err(mpsc::error::TryRecvError::Disconnected) => {
1033                    return AdmissionState::Closed;
1034                }
1035            };
1036        }
1037    }
1038}
1039
1040#[cfg(test)]
1041mod tests {
1042    use std::sync::atomic::AtomicU64;
1043
1044    use move_core_types::account_address::AccountAddress;
1045    use move_core_types::identifier::Identifier;
1046    use move_core_types::language_storage::StructTag;
1047    use sui_rpc::proto::sui::rpc::v2 as proto;
1048    use sui_types::base_types::ObjectID;
1049    use sui_types::base_types::SuiAddress;
1050    use sui_types::event::Event;
1051    use sui_types::test_checkpoint_data_builder::TestCheckpointBuilder;
1052
1053    use crate::ledger_history::filter::event_filter_to_query;
1054    use crate::ledger_history::filter::transaction_filter_to_query;
1055
1056    use super::*;
1057
1058    /// An unspawned dispatcher plus its shards, driven synchronously via
1059    /// [`drain`] so the actor tests are fully deterministic.
1060    fn test_service(shard_count: usize) -> (SubscriptionService, Vec<SubscriptionShard>) {
1061        test_service_with(shard_count, 25, None)
1062    }
1063
1064    fn test_service_with(
1065        shard_count: usize,
1066        watermark_interval: u32,
1067        indexed_checkpoint: Option<IndexedCheckpointFn>,
1068    ) -> (SubscriptionService, Vec<SubscriptionShard>) {
1069        let (service, _checkpoint_sender, _request_sender, shards) = actor_service_with(
1070            shard_count,
1071            watermark_interval,
1072            indexed_checkpoint,
1073            16,
1074            DEFAULT_MAX_SUBSCRIBERS,
1075        );
1076        (service, shards)
1077    }
1078
1079    fn actor_service_with(
1080        shard_count: usize,
1081        watermark_interval: u32,
1082        indexed_checkpoint: Option<IndexedCheckpointFn>,
1083        checkpoint_capacity: usize,
1084        max_subscribers: usize,
1085    ) -> (
1086        SubscriptionService,
1087        broadcast::Sender<Arc<Checkpoint>>,
1088        mpsc::UnboundedSender<SubscriptionRequest>,
1089        Vec<SubscriptionShard>,
1090    ) {
1091        let (checkpoint_sender, checkpoint_mailbox) = broadcast::channel(checkpoint_capacity);
1092        let (request_sender, admission_mailbox) = subscription_admission_channel();
1093        let metrics = SubscriptionMetrics::new(&prometheus::Registry::new());
1094        let counters = Arc::new(SubscriberCounts::new(max_subscribers));
1095
1096        let mut shard_senders = Vec::with_capacity(shard_count);
1097        let mut shards = Vec::with_capacity(shard_count);
1098        for _ in 0..shard_count {
1099            let (sender, shard_mailbox) = mpsc::channel(SHARD_MAILBOX_SIZE);
1100            shard_senders.push(sender);
1101            shards.push(SubscriptionShard {
1102                mailbox: shard_mailbox,
1103                matcher: matcher::SubscriptionMatcher::default(),
1104                watermark_interval,
1105            });
1106        }
1107
1108        let service = SubscriptionService {
1109            checkpoint_mailbox: CheckpointMailbox::Certified(checkpoint_mailbox),
1110            admission_mailbox,
1111            shards: shard_senders,
1112            next_shard: 0,
1113            counters,
1114            indexed_checkpoint,
1115            metrics,
1116        };
1117        (service, checkpoint_sender, request_sender, shards)
1118    }
1119
1120    fn checkpoint(sequence_number: u64) -> Arc<Checkpoint> {
1121        Arc::new(TestCheckpointBuilder::new(sequence_number).build_checkpoint())
1122    }
1123
1124    fn locally_built_checkpoint(sequence_number: u64) -> SubscriptionCheckpoint {
1125        let checkpoint = TestCheckpointBuilder::new(sequence_number).build_checkpoint();
1126        Arc::new(LocallyBuiltCheckpoint {
1127            summary: checkpoint.summary.into_data(),
1128            contents: checkpoint.contents,
1129            transactions: checkpoint.transactions,
1130            object_set: checkpoint.object_set,
1131        })
1132        .into()
1133    }
1134
1135    /// One checkpoint with one transaction per sender index, in order.
1136    fn checkpoint_with_senders(seq: u64, senders: &[u8]) -> Arc<Checkpoint> {
1137        let mut builder = TestCheckpointBuilder::new(seq);
1138        for &sender in senders {
1139            builder = builder.start_transaction(sender).finish_transaction();
1140        }
1141        Arc::new(builder.build_checkpoint())
1142    }
1143
1144    /// One checkpoint where tx 0 carries no events and tx 1 carries two.
1145    fn checkpoint_with_events(seq: u64) -> Arc<Checkpoint> {
1146        let package = AccountAddress::random();
1147        let event = |name: &str| Event {
1148            package_id: ObjectID::from(package),
1149            transaction_module: Identifier::new("emitter").unwrap(),
1150            sender: addr(1),
1151            type_: StructTag {
1152                address: package,
1153                module: Identifier::new("mod_t").unwrap(),
1154                name: Identifier::new(name).unwrap(),
1155                type_params: vec![],
1156            },
1157            contents: vec![],
1158        };
1159        let mut builder = TestCheckpointBuilder::new(seq);
1160        builder = builder.start_transaction(0).finish_transaction();
1161        builder = builder
1162            .start_transaction(1)
1163            .with_events(vec![event("EventA"), event("EventB")])
1164            .finish_transaction();
1165        Arc::new(builder.build_checkpoint())
1166    }
1167
1168    fn addr(idx: u8) -> SuiAddress {
1169        TestCheckpointBuilder::derive_address(idx)
1170    }
1171
1172    fn sender_query(address: SuiAddress, negated: bool) -> BitmapQuery {
1173        let mut sender = proto::SenderFilter::default();
1174        sender.address = Some(address.to_string());
1175        let mut literal = proto::TransactionLiteral::default();
1176        literal.predicate = Some(proto::transaction_literal::Predicate::Sender(sender));
1177        literal.negated = negated;
1178        let mut term = proto::TransactionTerm::default();
1179        term.literals = vec![literal];
1180        let mut filter = proto::TransactionFilter::default();
1181        filter.terms = vec![term];
1182        transaction_filter_to_query(&filter, 16).unwrap()
1183    }
1184
1185    fn event_type_query(type_str: &str) -> BitmapQuery {
1186        let mut event_type = proto::EventTypeFilter::default();
1187        event_type.event_type = Some(type_str.to_owned());
1188        let mut literal = proto::EventLiteral::default();
1189        literal.predicate = Some(proto::event_literal::Predicate::EventType(event_type));
1190        let mut term = proto::EventTerm::default();
1191        term.literals = vec![literal];
1192        let mut filter = proto::EventFilter::default();
1193        filter.terms = vec![term];
1194        event_filter_to_query(&filter, 16).unwrap()
1195    }
1196
1197    fn unfiltered() -> SubscriptionSpec {
1198        SubscriptionSpec {
1199            kind: SubscriptionKind::Checkpoints,
1200            query: None,
1201        }
1202    }
1203
1204    fn subscription_request(
1205        counters: &Arc<SubscriberCounts>,
1206        spec: SubscriptionSpec,
1207        response_sender: oneshot::Sender<mpsc::Receiver<SubscriptionUpdate>>,
1208    ) -> SubscriptionRequest {
1209        let reservation = counters
1210            .try_reserve()
1211            .expect("test request requires subscriber capacity");
1212        SubscriptionRequest {
1213            spec,
1214            response_sender,
1215            reservation,
1216        }
1217    }
1218
1219    /// Register a subscriber through the real admission path (cap check,
1220    /// gauge/space-counter increments, shard selection, and Register enqueue),
1221    /// returning the receiving half a client would hold, or `None` when the
1222    /// dispatcher rejected the registration.
1223    async fn register(
1224        service: &mut SubscriptionService,
1225        spec: SubscriptionSpec,
1226    ) -> Option<mpsc::Receiver<SubscriptionUpdate>> {
1227        let reservation = service.counters.try_reserve()?;
1228        let (response_sender, response_receiver) = oneshot::channel();
1229        assert!(matches!(
1230            service.try_admit(SubscriptionRequest {
1231                spec,
1232                response_sender,
1233                reservation,
1234            }),
1235            AdmissionState::Accepting
1236        ));
1237        response_receiver.await.ok()
1238    }
1239    fn inflight_subscribers(metrics: &SubscriptionMetrics) -> i64 {
1240        ["checkpoint", "transaction", "event"]
1241            .into_iter()
1242            .flat_map(|kind| {
1243                ["true", "false"].into_iter().map(move |filtered| {
1244                    metrics
1245                        .inflight_subscribers
1246                        .with_label_values(&[kind, filtered])
1247                        .get()
1248                })
1249            })
1250            .sum()
1251    }
1252
1253    fn terminations(
1254        metrics: &SubscriptionMetrics,
1255        kind: &'static str,
1256        reason: &'static str,
1257    ) -> u64 {
1258        metrics
1259            .terminations_total
1260            .with_label_values(&[kind, reason])
1261            .get()
1262    }
1263
1264    /// Synchronously run every shard's pending mailbox messages.
1265    fn drain(shards: &mut [SubscriptionShard]) {
1266        for shard in shards {
1267            shard.drain();
1268        }
1269    }
1270
1271    fn matched_sequence_number(update: SubscriptionUpdate) -> u64 {
1272        match update {
1273            SubscriptionUpdate::Matched(matched) => {
1274                assert!(matches!(matched.matches, SubscriptionMatches::Checkpoint));
1275                matched.checkpoint.summary().sequence_number
1276            }
1277            SubscriptionUpdate::WatermarkTick { .. } => {
1278                panic!("expected a matched checkpoint, got a watermark tick")
1279            }
1280        }
1281    }
1282
1283    fn matched_transactions(update: SubscriptionUpdate) -> Vec<u32> {
1284        match update {
1285            SubscriptionUpdate::Matched(matched) => match matched.matches {
1286                SubscriptionMatches::Transactions(ranges) => ranges.into_iter().flatten().collect(),
1287                _ => panic!("expected transaction matches"),
1288            },
1289            SubscriptionUpdate::WatermarkTick { .. } => {
1290                panic!("expected a matched checkpoint, got a watermark tick")
1291            }
1292        }
1293    }
1294
1295    #[test]
1296    fn subscriber_reservations_are_bounded_and_released() {
1297        let counters = Arc::new(SubscriberCounts::new(8));
1298        let attempted = Arc::new(std::sync::Barrier::new(65));
1299        let release = Arc::new(std::sync::Barrier::new(65));
1300
1301        std::thread::scope(|scope| {
1302            let mut handles = Vec::with_capacity(64);
1303            for _ in 0..64 {
1304                let counters = Arc::clone(&counters);
1305                let attempted = Arc::clone(&attempted);
1306                let release = Arc::clone(&release);
1307                handles.push(scope.spawn(move || {
1308                    let reservation = counters.try_reserve();
1309                    attempted.wait();
1310                    release.wait();
1311                    reservation.is_some()
1312                }));
1313            }
1314
1315            attempted.wait();
1316            assert_eq!(counters.reserved(), 8);
1317            release.wait();
1318
1319            let successful_reservations = handles
1320                .into_iter()
1321                .map(|handle| handle.join().unwrap())
1322                .filter(|success| *success)
1323                .count();
1324            assert_eq!(successful_reservations, 8);
1325        });
1326
1327        assert_eq!(counters.reserved(), 0);
1328    }
1329
1330    #[tokio::test]
1331    async fn pending_request_reserves_final_subscriber_slot() {
1332        let counters = Arc::new(SubscriberCounts::new(3));
1333        let active_reservations = [
1334            counters.try_reserve().unwrap(),
1335            counters.try_reserve().unwrap(),
1336        ];
1337        let (admission_sender, mailbox) = subscription_admission_channel();
1338        let handle = SubscriptionServiceHandle {
1339            admission_sender,
1340            counters: Arc::clone(&counters),
1341            metrics: SubscriptionMetrics::new(&prometheus::Registry::new()),
1342        };
1343
1344        let pending_handle = handle.clone();
1345        let pending_registration =
1346            tokio::spawn(async move { pending_handle.register_subscription(unfiltered()).await });
1347        tokio::time::timeout(Duration::from_secs(1), async {
1348            while mailbox.len() != 1 {
1349                tokio::task::yield_now().await;
1350            }
1351        })
1352        .await
1353        .expect("pending registration did not enter admission");
1354
1355        assert_eq!(counters.reserved(), 3);
1356        let rejected = tokio::time::timeout(
1357            Duration::from_secs(1),
1358            handle.register_subscription(unfiltered()),
1359        )
1360        .await
1361        .expect("the final subscriber slot is already reserved");
1362        assert!(rejected.is_none());
1363        assert_eq!(mailbox.len(), 1);
1364
1365        drop(mailbox);
1366        assert!(pending_registration.await.unwrap().is_none());
1367        assert_eq!(counters.reserved(), 2);
1368
1369        drop(active_reservations);
1370        assert_eq!(counters.reserved(), 0);
1371    }
1372
1373    #[tokio::test]
1374    async fn zero_subscriber_limit_rejects_before_ingress() {
1375        let (_checkpoint_sender, handle) =
1376            SubscriptionService::build(&prometheus::Registry::new(), None, None, Some(0), Some(1));
1377        assert!(handle.counters.try_reserve().is_none());
1378        assert!(handle.register_subscription(unfiltered()).await.is_none());
1379        assert_eq!(handle.counters.reserved(), 0);
1380    }
1381
1382    #[tokio::test]
1383    async fn public_admission_rejects_before_queue_at_limit() {
1384        let counters = Arc::new(SubscriberCounts::new(1));
1385        let held_reservation = counters.try_reserve().unwrap();
1386        let (admission_sender, mut mailbox) = subscription_admission_channel();
1387        let handle = SubscriptionServiceHandle {
1388            admission_sender,
1389            counters: Arc::clone(&counters),
1390            metrics: SubscriptionMetrics::new(&prometheus::Registry::new()),
1391        };
1392
1393        let result = tokio::time::timeout(
1394            Duration::from_secs(1),
1395            handle.register_subscription(unfiltered()),
1396        )
1397        .await
1398        .expect("a saturated service must reject before queueing");
1399        assert!(result.is_none());
1400        assert!(matches!(
1401            mailbox.try_recv(),
1402            Err(mpsc::error::TryRecvError::Empty)
1403        ));
1404
1405        drop(held_reservation);
1406        assert_eq!(counters.reserved(), 0);
1407    }
1408
1409    #[tokio::test]
1410    async fn closed_public_admission_queue_releases_reservation() {
1411        let counters = Arc::new(SubscriberCounts::new(1));
1412        let (admission_sender, mailbox) = subscription_admission_channel();
1413        drop(mailbox);
1414        let handle = SubscriptionServiceHandle {
1415            admission_sender,
1416            counters: Arc::clone(&counters),
1417            metrics: SubscriptionMetrics::new(&prometheus::Registry::new()),
1418        };
1419
1420        let result = handle.register_subscription(unfiltered()).await;
1421        assert!(result.is_none());
1422        assert_eq!(counters.reserved(), 0);
1423    }
1424
1425    #[test]
1426    fn admission_turn_is_bounded_and_drains_ready_requests() {
1427        let (mut service, _checkpoint_sender, request_sender, _shards) =
1428            actor_service_with(3, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1429        let mut response_receivers = Vec::with_capacity(ADMISSION_TURN_LIMIT + 1);
1430
1431        for _ in 0..=ADMISSION_TURN_LIMIT {
1432            let (response_sender, response_receiver) = oneshot::channel();
1433            assert!(
1434                request_sender
1435                    .send(subscription_request(
1436                        &service.counters,
1437                        unfiltered(),
1438                        response_sender,
1439                    ))
1440                    .is_ok()
1441            );
1442            response_receivers.push(response_receiver);
1443        }
1444
1445        let first_request = service.admission_mailbox.try_recv().unwrap();
1446        assert!(matches!(
1447            service.admit_ready_requests(first_request),
1448            AdmissionState::Accepting
1449        ));
1450        assert_eq!(service.admission_mailbox.len(), 1);
1451
1452        for response_receiver in response_receivers.iter_mut().take(ADMISSION_TURN_LIMIT) {
1453            response_receiver
1454                .try_recv()
1455                .expect("request should be admitted");
1456        }
1457        assert!(matches!(
1458            response_receivers.last_mut().unwrap().try_recv(),
1459            Err(oneshot::error::TryRecvError::Empty)
1460        ));
1461
1462        let final_request = service.admission_mailbox.try_recv().unwrap();
1463        assert!(matches!(
1464            service.admit_ready_requests(final_request),
1465            AdmissionState::Accepting
1466        ));
1467        response_receivers
1468            .last_mut()
1469            .unwrap()
1470            .try_recv()
1471            .expect("final request should be admitted");
1472    }
1473
1474    #[test]
1475    fn canceled_ready_admission_releases_reservation() {
1476        let (mut service, _checkpoint_sender, _request_sender, _shards) =
1477            actor_service_with(1, 25, None, 4, 1);
1478        let (response_sender, response_receiver) = oneshot::channel();
1479        let request = subscription_request(&service.counters, unfiltered(), response_sender);
1480        assert_eq!(service.counters.reserved(), 1);
1481
1482        drop(response_receiver);
1483        assert!(matches!(
1484            service.try_admit(request),
1485            AdmissionState::Accepting
1486        ));
1487        assert_eq!(service.counters.reserved(), 0);
1488    }
1489
1490    #[tokio::test]
1491    async fn checkpoint_backlog_drains_before_registration() {
1492        // Queue one admission and two checkpoints before starting the actor,
1493        // making both select branches ready on its first poll.
1494        let (service, checkpoint_sender, request_sender, mut shards) =
1495            actor_service_with(1, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1496        let metrics = service.metrics.clone();
1497        let (reply_sender, reply_receiver) = oneshot::channel();
1498        assert!(
1499            request_sender
1500                .send(subscription_request(
1501                    &service.counters,
1502                    unfiltered(),
1503                    reply_sender,
1504                ))
1505                .is_ok()
1506        );
1507        assert_eq!(checkpoint_sender.send(checkpoint(1)).unwrap(), 1);
1508        assert_eq!(checkpoint_sender.send(checkpoint(2)).unwrap(), 1);
1509
1510        // The biased select must process both checkpoints before acknowledging
1511        // the registration.
1512        let actor = tokio::spawn(service.start());
1513        let mut receiver = tokio::time::timeout(Duration::from_secs(1), reply_receiver)
1514            .await
1515            .expect("admission remained blocked after the checkpoint backlog drained")
1516            .unwrap();
1517        assert_eq!(metrics.last_recieved_checkpoint.get(), 2);
1518
1519        // The subscriber was admitted after checkpoints 1 and 2, so its first
1520        // deliverable checkpoint is 3.
1521        assert_eq!(checkpoint_sender.send(checkpoint(3)).unwrap(), 1);
1522        drop(checkpoint_sender);
1523        actor.await.unwrap();
1524
1525        drain(&mut shards);
1526        assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 3);
1527        assert!(receiver.recv().await.is_none());
1528    }
1529
1530    #[tokio::test]
1531    async fn queued_admission_releases_reservation_when_actor_shuts_down() {
1532        let (service, checkpoint_sender, request_sender, _shards) =
1533            actor_service_with(1, 25, None, 4, 1);
1534        let counters = Arc::clone(&service.counters);
1535        let (response_sender, response_receiver) = oneshot::channel();
1536        assert!(
1537            request_sender
1538                .send(subscription_request(
1539                    &counters,
1540                    unfiltered(),
1541                    response_sender,
1542                ))
1543                .is_ok()
1544        );
1545        assert_eq!(counters.reserved(), 1);
1546
1547        drop(checkpoint_sender);
1548        service.start().await;
1549
1550        assert!(response_receiver.await.is_err());
1551        assert_eq!(counters.reserved(), 0);
1552    }
1553
1554    #[tokio::test]
1555    async fn admission_prefers_least_backlogged_shard() {
1556        // Seed mailbox depths of two, one, and zero. Immediate admission should
1557        // choose shard 2 because it has the greatest remaining capacity.
1558        let (mut service, mut shards) = test_service(3);
1559        for _ in 0..2 {
1560            assert!(
1561                service.shards[0]
1562                    .try_send(ShardMsg::Clear(
1563                        SubscriptionTerminationReason::ServiceShutdown
1564                    ))
1565                    .is_ok()
1566            );
1567        }
1568        assert!(
1569            service.shards[1]
1570                .try_send(ShardMsg::Clear(
1571                    SubscriptionTerminationReason::ServiceShutdown
1572                ))
1573                .is_ok()
1574        );
1575
1576        // Processing the selected registration also verifies that admission
1577        // installed its lifecycle accounting.
1578        let receiver = register(&mut service, unfiltered()).await.unwrap();
1579        let registration = shards[2].mailbox.try_recv().unwrap();
1580        assert!(matches!(registration, ShardMsg::Register { .. }));
1581        shards[2].handle_msg(registration);
1582        assert_eq!(service.counters.total.load(Ordering::Relaxed), 1);
1583        assert_eq!(inflight_subscribers(&service.metrics), 1);
1584        drop(receiver);
1585    }
1586
1587    #[tokio::test]
1588    async fn saturated_admission_waits_for_any_shard() {
1589        // Fill every shard mailbox so the request enters WaitingForShard rather
1590        // than taking the immediate admission path.
1591        let (service, _checkpoint_sender, request_sender, mut shards) =
1592            actor_service_with(2, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1593        for shard in &service.shards {
1594            for _ in 0..SHARD_MAILBOX_SIZE {
1595                assert!(
1596                    shard
1597                        .try_send(ShardMsg::Clear(
1598                            SubscriptionTerminationReason::ServiceShutdown
1599                        ))
1600                        .is_ok()
1601                );
1602            }
1603        }
1604
1605        // Polling once consumes the request and reaches the pending
1606        // shard-capacity wait before a mailbox slot is released.
1607        let (reply_sender, reply_receiver) = oneshot::channel();
1608        assert!(
1609            request_sender
1610                .send(subscription_request(
1611                    &service.counters,
1612                    unfiltered(),
1613                    reply_sender,
1614                ))
1615                .is_ok()
1616        );
1617        let mut actor = Box::pin(service.start());
1618        assert!(futures::poll!(&mut actor).is_pending());
1619        let actor = tokio::spawn(actor);
1620
1621        // Free only shard 1. Waiting on all shard permits must wake and route
1622        // the registration there even though the cursor starts at shard 0.
1623        assert!(matches!(
1624            shards[1].mailbox.try_recv(),
1625            Ok(ShardMsg::Clear(_))
1626        ));
1627        let _receiver = tokio::time::timeout(Duration::from_secs(1), reply_receiver)
1628            .await
1629            .expect("admission did not wake when shard capacity became available")
1630            .unwrap();
1631
1632        // The registration follows the existing shard 1 messages, while shard
1633        // 0 remains full and untouched.
1634        for _ in 1..SHARD_MAILBOX_SIZE {
1635            assert!(matches!(
1636                shards[1].mailbox.try_recv(),
1637                Ok(ShardMsg::Clear(_))
1638            ));
1639        }
1640        assert!(matches!(
1641            shards[1].mailbox.try_recv(),
1642            Ok(ShardMsg::Register { .. })
1643        ));
1644        assert_eq!(shards[0].mailbox.len(), SHARD_MAILBOX_SIZE);
1645
1646        actor.abort();
1647        let _ = actor.await;
1648    }
1649
1650    #[tokio::test]
1651    async fn saturated_head_blocks_newer_request_until_canceled() {
1652        // Fill every shard, then queue a head request followed by a newer one.
1653        let (service, _checkpoint_sender, request_sender, mut shards) =
1654            actor_service_with(2, 25, None, 4, DEFAULT_MAX_SUBSCRIBERS);
1655        let counters = Arc::clone(&service.counters);
1656        for shard in &service.shards {
1657            for _ in 0..SHARD_MAILBOX_SIZE {
1658                assert!(
1659                    shard
1660                        .try_send(ShardMsg::Clear(
1661                            SubscriptionTerminationReason::ServiceShutdown
1662                        ))
1663                        .is_ok()
1664                );
1665            }
1666        }
1667
1668        let (first_sender, first_reply) = oneshot::channel();
1669        let (second_sender, second_reply) = oneshot::channel();
1670        assert!(
1671            request_sender
1672                .send(subscription_request(
1673                    &service.counters,
1674                    unfiltered(),
1675                    first_sender,
1676                ))
1677                .is_ok()
1678        );
1679        assert!(
1680            request_sender
1681                .send(subscription_request(
1682                    &service.counters,
1683                    unfiltered(),
1684                    second_sender,
1685                ))
1686                .is_ok()
1687        );
1688        assert_eq!(counters.reserved(), 2);
1689
1690        // One poll consumes only the FIFO head and reaches the pending
1691        // shard-capacity wait, leaving the newer request queued.
1692        let mut actor = Box::pin(service.start());
1693        assert!(futures::poll!(&mut actor).is_pending());
1694
1695        drop(first_reply);
1696        // The next poll drops the canceled head and moves the newer request
1697        // into the shard-capacity wait.
1698        assert!(futures::poll!(&mut actor).is_pending());
1699        assert_eq!(counters.reserved(), 1);
1700        let actor = tokio::spawn(actor);
1701
1702        // Free one shard slot so the newer request can complete admission.
1703        assert!(matches!(
1704            shards[0].mailbox.try_recv(),
1705            Ok(ShardMsg::Clear(_))
1706        ));
1707        let _receiver = tokio::time::timeout(Duration::from_secs(1), second_reply)
1708            .await
1709            .expect("newer admission remained blocked after canceling the waiting request")
1710            .unwrap();
1711        assert_eq!(counters.reserved(), 1);
1712
1713        actor.abort();
1714        let _ = actor.await;
1715        drop(shards);
1716        assert_eq!(counters.reserved(), 0);
1717    }
1718
1719    #[tokio::test]
1720    async fn run_loop_lag_clears_subscribers_and_resets_sequence_tracker() {
1721        // Register a subscriber, then overflow a one-slot checkpoint broadcast.
1722        // The actor first observes Lagged and then the retained checkpoint 2.
1723        let (mut service, checkpoint_sender, _request_sender, mut shards) =
1724            actor_service_with(1, 25, None, 1, DEFAULT_MAX_SUBSCRIBERS);
1725        let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1726        let counters = Arc::clone(&service.counters);
1727        let metrics = service.metrics.clone();
1728        // A deliberately incompatible prior sequence proves lag handling resets
1729        // the tracker before checkpoint 2 is processed.
1730        metrics.last_recieved_checkpoint.set(99);
1731
1732        assert_eq!(checkpoint_sender.send(checkpoint(1)).unwrap(), 1);
1733        assert_eq!(checkpoint_sender.send(checkpoint(2)).unwrap(), 1);
1734        let actor = tokio::spawn(service.start());
1735        drop(checkpoint_sender);
1736        actor.await.unwrap();
1737
1738        // Draining applies the queued registration, source-lag clear, retained
1739        // checkpoint, and shutdown clear in shard FIFO order.
1740        drain(&mut shards);
1741        assert!(receiver.recv().await.is_none());
1742        assert_eq!(counters.total.load(Ordering::Relaxed), 0);
1743        assert_eq!(inflight_subscribers(&metrics), 0);
1744        assert_eq!(terminations(&metrics, "checkpoint", "source_lag"), 1);
1745        // The retained checkpoint can be accepted only because the Lagged
1746        // branch reset the deliberately incompatible prior sequence number.
1747        assert_eq!(metrics.last_recieved_checkpoint.get(), 2);
1748    }
1749
1750    #[tokio::test]
1751    async fn labeled_inflight_gauge_covers_all_types_and_filter_states() {
1752        let (mut service, mut shards) = test_service(1);
1753        let kinds = [
1754            SubscriptionKind::Checkpoints,
1755            SubscriptionKind::Transactions,
1756            SubscriptionKind::Events,
1757        ];
1758        let mut receivers = Vec::new();
1759
1760        for kind in kinds {
1761            for filtered in [false, true] {
1762                let query = filtered.then(|| match kind {
1763                    SubscriptionKind::Checkpoints | SubscriptionKind::Transactions => {
1764                        sender_query(addr(0), false)
1765                    }
1766                    SubscriptionKind::Events => event_type_query(
1767                        "0x0000000000000000000000000000000000000000000000000000000000000002::coin::CoinEvent",
1768                    ),
1769                });
1770                receivers.push(
1771                    register(&mut service, SubscriptionSpec { kind, query })
1772                        .await
1773                        .unwrap(),
1774                );
1775            }
1776        }
1777        drain(&mut shards);
1778
1779        for kind in kinds {
1780            for filtered in ["false", "true"] {
1781                assert_eq!(
1782                    service
1783                        .metrics
1784                        .inflight_subscribers
1785                        .with_label_values(&[kind.metric_label(), filtered])
1786                        .get(),
1787                    1
1788                );
1789            }
1790        }
1791
1792        drop(receivers);
1793        service.handle_checkpoint(checkpoint(1).into()).await;
1794        drain(&mut shards);
1795        assert_eq!(inflight_subscribers(&service.metrics), 0);
1796        assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1797
1798        shards[0].handle_msg(ShardMsg::Clear(
1799            SubscriptionTerminationReason::ServiceShutdown,
1800        ));
1801        assert_eq!(inflight_subscribers(&service.metrics), 0);
1802        assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1803        for kind in ["checkpoint", "transaction", "event"] {
1804            assert_eq!(terminations(&service.metrics, kind, "client_closed"), 2);
1805        }
1806    }
1807
1808    #[tokio::test]
1809    async fn handle_checkpoint_fans_out_in_order() {
1810        let (mut service, mut shards) = test_service(1);
1811        let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1812
1813        service.handle_checkpoint(checkpoint(1).into()).await;
1814        service.handle_checkpoint(checkpoint(2).into()).await;
1815        drain(&mut shards);
1816
1817        assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 1);
1818        assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 2);
1819        assert_eq!(shards[0].matcher.len(), 1);
1820    }
1821
1822    #[tokio::test]
1823    async fn handle_checkpoint_drops_departed_subscriber() {
1824        let (mut service, mut shards) = test_service(1);
1825        let receiver = register(&mut service, unfiltered()).await.unwrap();
1826        drop(receiver);
1827
1828        service.handle_checkpoint(checkpoint(1).into()).await;
1829        drain(&mut shards);
1830
1831        assert!(shards[0].matcher.is_empty());
1832        assert_eq!(inflight_subscribers(&service.metrics), 0);
1833        assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1834        assert_eq!(
1835            terminations(&service.metrics, "checkpoint", "client_closed"),
1836            1
1837        );
1838    }
1839
1840    #[tokio::test(start_paused = true)]
1841    async fn handle_checkpoint_waits_for_index_before_delivering() {
1842        // The first checkpoint is already indexed and does not count as a wait.
1843        let indexed = Arc::new(AtomicU64::new(4));
1844        let gate = indexed.clone();
1845        let (mut service, mut shards) = test_service_with(
1846            1,
1847            25,
1848            Some(Arc::new(move || Some(gate.load(Ordering::SeqCst)))),
1849        );
1850        let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1851
1852        service.handle_checkpoint(checkpoint(4).into()).await;
1853        drain(&mut shards);
1854        assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 4);
1855        assert_eq!(service.metrics.index_wait_seconds.get_sample_count(), 0);
1856        assert_eq!(service.metrics.index_wait_timeouts_total.get(), 0);
1857
1858        // Delivery of checkpoint 5 blocks until the index catches up to it.
1859        {
1860            let mut deliver = std::pin::pin!(service.handle_checkpoint(checkpoint(5).into()));
1861            assert!(
1862                futures::poll!(&mut deliver).is_pending(),
1863                "delivery should block while checkpoint 5 is unindexed"
1864            );
1865            assert!(receiver.try_recv().is_err());
1866
1867            indexed.store(5, Ordering::SeqCst);
1868            tokio::time::advance(INDEX_WAIT_POLL_INTERVAL).await;
1869            deliver.await;
1870        }
1871        drain(&mut shards);
1872
1873        assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 5);
1874        assert_eq!(service.metrics.index_wait_seconds.get_sample_count(), 1);
1875        assert!(
1876            service.metrics.index_wait_seconds.get_sample_sum()
1877                >= INDEX_WAIT_POLL_INTERVAL.as_secs_f64()
1878        );
1879        assert_eq!(service.metrics.index_wait_timeouts_total.get(), 0);
1880    }
1881
1882    #[tokio::test(start_paused = true)]
1883    async fn index_wait_timeout_records_and_delivers_with_paused_time() {
1884        let indexed = Arc::new(AtomicU64::new(4));
1885        let gate = indexed.clone();
1886        let (mut service, mut shards) = test_service_with(
1887            1,
1888            25,
1889            Some(Arc::new(move || Some(gate.load(Ordering::SeqCst)))),
1890        );
1891        let mut receiver = register(&mut service, unfiltered()).await.unwrap();
1892
1893        {
1894            let mut deliver = std::pin::pin!(service.handle_checkpoint(checkpoint(5).into()));
1895            assert!(
1896                futures::poll!(&mut deliver).is_pending(),
1897                "delivery should block while checkpoint 5 is unindexed"
1898            );
1899            tokio::time::advance(INDEX_WAIT_TIMEOUT).await;
1900            deliver.await;
1901        }
1902        drain(&mut shards);
1903
1904        assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 5);
1905        assert_eq!(service.metrics.index_wait_seconds.get_sample_count(), 1);
1906        assert!(
1907            service.metrics.index_wait_seconds.get_sample_sum() >= INDEX_WAIT_TIMEOUT.as_secs_f64()
1908        );
1909        assert_eq!(service.metrics.index_wait_timeouts_total.get(), 1);
1910    }
1911
1912    #[tokio::test]
1913    async fn handle_lag_drops_all_subscribers_and_resets_tracker() {
1914        let (mut service, mut shards) = test_service(2);
1915        let mut receiver_1 = register(&mut service, unfiltered()).await.unwrap();
1916        let mut receiver_2 = register(&mut service, unfiltered()).await.unwrap();
1917
1918        service.handle_checkpoint(checkpoint(5).into()).await;
1919        drain(&mut shards);
1920        assert_eq!(service.metrics.last_recieved_checkpoint.get(), 5);
1921
1922        service.handle_lag(10).await;
1923        drain(&mut shards);
1924
1925        // Clear empties every shard.
1926        assert!(shards[0].matcher.is_empty());
1927        assert!(shards[1].matcher.is_empty());
1928        assert_eq!(inflight_subscribers(&service.metrics), 0);
1929        assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
1930        // Both subscriptions are torn down, so the client streams close.
1931        assert!(receiver_1.recv().await.is_some()); // checkpoint 5, then closed
1932        assert!(receiver_1.recv().await.is_none());
1933        assert!(receiver_2.recv().await.is_some());
1934        assert!(receiver_2.recv().await.is_none());
1935        // The tracker is reset so the next, jumped-ahead checkpoint is not
1936        // mistaken for an out-of-order delivery (which would panic).
1937        assert_eq!(service.metrics.last_recieved_checkpoint.get(), 0);
1938        assert_eq!(
1939            terminations(&service.metrics, "checkpoint", "source_lag"),
1940            2
1941        );
1942        service.handle_lag(1).await;
1943        drain(&mut shards);
1944        assert_eq!(
1945            terminations(&service.metrics, "checkpoint", "source_lag"),
1946            2
1947        );
1948
1949        let mut receiver_3 = register(&mut service, unfiltered()).await.unwrap();
1950        service.handle_checkpoint(checkpoint(100).into()).await;
1951        drain(&mut shards);
1952        assert_eq!(
1953            matched_sequence_number(receiver_3.recv().await.unwrap()),
1954            100
1955        );
1956    }
1957
1958    #[tokio::test]
1959    async fn locally_built_gap_drops_subscribers() {
1960        let (mut service, mut shards) = test_service(2);
1961        let mut before_gap = register(&mut service, unfiltered()).await.unwrap();
1962
1963        service.handle_checkpoint(locally_built_checkpoint(5)).await;
1964        service.handle_checkpoint(locally_built_checkpoint(9)).await;
1965        drain(&mut shards);
1966
1967        // The subscriber sees checkpoint 5, then its stream closes without
1968        // ever delivering checkpoint 9.
1969        assert_eq!(matched_sequence_number(before_gap.recv().await.unwrap()), 5);
1970        assert!(before_gap.recv().await.is_none());
1971        assert_eq!(
1972            terminations(&service.metrics, "checkpoint", "source_gap"),
1973            1
1974        );
1975        assert_eq!(service.metrics.last_recieved_checkpoint.get(), 9);
1976
1977        let mut after_gap = register(&mut service, unfiltered()).await.unwrap();
1978        service
1979            .handle_checkpoint(locally_built_checkpoint(10))
1980            .await;
1981        drain(&mut shards);
1982        assert_eq!(matched_sequence_number(after_gap.recv().await.unwrap()), 10);
1983    }
1984
1985    #[tokio::test]
1986    #[should_panic(expected = "out-of-order")]
1987    async fn certified_gap_panics() {
1988        let (mut service, _shards) = test_service(1);
1989        service.handle_checkpoint(checkpoint(5).into()).await;
1990        service.handle_checkpoint(checkpoint(9).into()).await;
1991    }
1992
1993    #[tokio::test]
1994    async fn service_shutdown_records_each_subscription_once() {
1995        let (mut service, mut shards) = test_service(2);
1996        let receiver_explicit = register(&mut service, unfiltered()).await.unwrap();
1997        let receiver_fallback = register(&mut service, unfiltered()).await.unwrap();
1998        drain(&mut shards);
1999        let metrics = service.metrics.clone();
2000        let counters = Arc::clone(&service.counters);
2001        assert_eq!(counters.reserved(), 2);
2002
2003        service.shards[0]
2004            .send(ShardMsg::Clear(
2005                SubscriptionTerminationReason::ServiceShutdown,
2006            ))
2007            .await
2008            .unwrap();
2009        shards[0].drain();
2010        assert_eq!(terminations(&metrics, "checkpoint", "service_shutdown"), 1);
2011        assert_eq!(counters.reserved(), 1);
2012
2013        let fallback_shard = shards.pop().unwrap();
2014        drop(service);
2015        fallback_shard.run().await;
2016        assert!(receiver_explicit.is_closed());
2017        assert!(receiver_fallback.is_closed());
2018        assert_eq!(inflight_subscribers(&metrics), 0);
2019        assert_eq!(counters.total.load(Ordering::Relaxed), 0);
2020        assert_eq!(counters.reserved(), 0);
2021        assert_eq!(counters.filtered_tx.load(Ordering::Relaxed), 0);
2022        assert_eq!(counters.filtered_event.load(Ordering::Relaxed), 0);
2023        assert_eq!(terminations(&metrics, "checkpoint", "service_shutdown"), 2);
2024
2025        drop(shards);
2026        assert_eq!(terminations(&metrics, "checkpoint", "service_shutdown"), 2);
2027    }
2028
2029    #[tokio::test]
2030    async fn subscribers_on_every_shard_receive_a_matched_checkpoint() {
2031        let (mut service, mut shards) = test_service(2);
2032        let mut receivers = Vec::new();
2033        for _ in 0..4 {
2034            receivers.push(register(&mut service, unfiltered()).await.unwrap());
2035        }
2036        drain(&mut shards);
2037        // Equal-capacity tie-breaking spreads subscribers across the shards.
2038        assert_eq!(shards[0].matcher.len(), 2);
2039        assert_eq!(shards[1].matcher.len(), 2);
2040
2041        service.handle_checkpoint(checkpoint(1).into()).await;
2042        drain(&mut shards);
2043        for receiver in &mut receivers {
2044            assert_eq!(matched_sequence_number(receiver.recv().await.unwrap()), 1);
2045        }
2046    }
2047
2048    #[tokio::test]
2049    async fn filtered_subscribers_match_via_dispatcher_extracted_keys() {
2050        let (mut service, mut shards) = test_service(2);
2051        // Include filter on sender 0...
2052        let mut include_rx = register(
2053            &mut service,
2054            SubscriptionSpec {
2055                kind: SubscriptionKind::Transactions,
2056                query: Some(sender_query(addr(0), false)),
2057            },
2058        )
2059        .await
2060        .unwrap();
2061        // ...and an exclude-only filter, anchored on the synthetic
2062        // `TxUniverse` key that dispatcher-side extraction must insert into
2063        // every transaction's key set.
2064        let mut exclude_rx = register(
2065            &mut service,
2066            SubscriptionSpec {
2067                kind: SubscriptionKind::Transactions,
2068                query: Some(sender_query(addr(0), true)),
2069            },
2070        )
2071        .await
2072        .unwrap();
2073        assert_eq!(service.counters.filtered_tx.load(Ordering::Relaxed), 2);
2074
2075        let checkpoint = checkpoint_with_senders(1, &[0, 1]);
2076        let tx_lo = checkpoint.summary.data().network_total_transactions
2077            - checkpoint.transactions.len() as u64;
2078        service.handle_checkpoint(checkpoint.into()).await;
2079        drain(&mut shards);
2080
2081        assert!(matches!(
2082            include_rx.recv().await.unwrap(),
2083            SubscriptionUpdate::WatermarkTick {
2084                checkpoint: 0,
2085                tx_hi
2086            } if tx_hi == tx_lo
2087        ));
2088        assert!(matches!(
2089            exclude_rx.recv().await.unwrap(),
2090            SubscriptionUpdate::WatermarkTick {
2091                checkpoint: 0,
2092                tx_hi
2093            } if tx_hi == tx_lo
2094        ));
2095
2096        assert_eq!(
2097            matched_transactions(include_rx.recv().await.unwrap()),
2098            vec![0]
2099        );
2100        assert_eq!(
2101            matched_transactions(exclude_rx.recv().await.unwrap()),
2102            vec![1]
2103        );
2104    }
2105
2106    #[tokio::test]
2107    async fn configured_cap_is_enforced_globally_across_shards() {
2108        let max_subscribers = 3;
2109        let (mut service, _, _, mut shards) = actor_service_with(2, 25, None, 16, max_subscribers);
2110        let mut receivers = Vec::with_capacity(max_subscribers);
2111        for _ in 0..max_subscribers {
2112            receivers.push(register(&mut service, unfiltered()).await.unwrap());
2113        }
2114        drain(&mut shards);
2115        assert_eq!(
2116            service.counters.total.load(Ordering::Relaxed),
2117            max_subscribers
2118        );
2119        assert_eq!(service.counters.reserved(), max_subscribers);
2120        assert_eq!(
2121            inflight_subscribers(&service.metrics),
2122            max_subscribers as i64
2123        );
2124        assert!(!shards[0].matcher.is_empty());
2125        assert!(!shards[1].matcher.is_empty());
2126
2127        // Reservation failure occurs before constructing an admission request,
2128        // so the cap cannot add queued work or lifecycle accounting.
2129        assert!(register(&mut service, unfiltered()).await.is_none());
2130        assert_eq!(
2131            service.counters.total.load(Ordering::Relaxed),
2132            max_subscribers
2133        );
2134        assert_eq!(service.counters.reserved(), max_subscribers);
2135        assert_eq!(
2136            inflight_subscribers(&service.metrics),
2137            max_subscribers as i64
2138        );
2139        assert_eq!(
2140            terminations(&service.metrics, "checkpoint", "service_shutdown"),
2141            0
2142        );
2143
2144        drop(receivers);
2145        service.handle_checkpoint(checkpoint(1).into()).await;
2146        drain(&mut shards);
2147        assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
2148        assert_eq!(service.counters.reserved(), 0);
2149        assert_eq!(inflight_subscribers(&service.metrics), 0);
2150
2151        let replacement = register(&mut service, unfiltered()).await.unwrap();
2152        drain(&mut shards);
2153        assert_eq!(service.counters.total.load(Ordering::Relaxed), 1);
2154        assert_eq!(service.counters.reserved(), 1);
2155
2156        drop(replacement);
2157        service.handle_checkpoint(checkpoint(2).into()).await;
2158        drain(&mut shards);
2159        assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
2160        assert_eq!(service.counters.reserved(), 0);
2161    }
2162
2163    #[tokio::test]
2164    async fn unprocessed_register_finalizes_guard_when_shard_drops() {
2165        let (mut service, shards) = test_service(1);
2166        let _receiver = register(&mut service, unfiltered()).await.unwrap();
2167        assert_eq!(service.counters.total.load(Ordering::Relaxed), 1);
2168        assert_eq!(inflight_subscribers(&service.metrics), 1);
2169        assert_eq!(service.counters.reserved(), 1);
2170
2171        // Drop the shard with the Register message still queued in its
2172        // mailbox: the guard travelling inside the message must finalize
2173        // with the default service_shutdown reason and rebalance the
2174        // counts and gauge exactly once.
2175        drop(shards);
2176        assert_eq!(service.counters.total.load(Ordering::Relaxed), 0);
2177        assert_eq!(service.counters.reserved(), 0);
2178        assert_eq!(inflight_subscribers(&service.metrics), 0);
2179        assert_eq!(
2180            terminations(&service.metrics, "checkpoint", "service_shutdown"),
2181            1
2182        );
2183    }
2184
2185    #[tokio::test]
2186    async fn registration_racing_dispatch_lands_at_the_next_checkpoint() {
2187        let (mut service, mut shards) = test_service(1);
2188        let mut receiver_a = register(&mut service, unfiltered()).await.unwrap();
2189        service.handle_checkpoint(checkpoint(1).into()).await;
2190        let mut receiver_b = register(&mut service, unfiltered()).await.unwrap();
2191        service.handle_checkpoint(checkpoint(2).into()).await;
2192        drain(&mut shards);
2193
2194        // Per-shard FIFO: B's Register is queued behind checkpoint 1, so B
2195        // sees only checkpoint 2.
2196        assert_eq!(matched_sequence_number(receiver_a.recv().await.unwrap()), 1);
2197        assert_eq!(matched_sequence_number(receiver_a.recv().await.unwrap()), 2);
2198        assert_eq!(matched_sequence_number(receiver_b.recv().await.unwrap()), 2);
2199        assert!(receiver_b.try_recv().is_err());
2200    }
2201
2202    #[tokio::test]
2203    async fn space_counters_track_filtered_subscribers() {
2204        let (mut service, mut shards) = test_service_with(2, 1, None);
2205        let tx_rx = register(
2206            &mut service,
2207            SubscriptionSpec {
2208                kind: SubscriptionKind::Transactions,
2209                query: Some(sender_query(addr(0), false)),
2210            },
2211        )
2212        .await
2213        .unwrap();
2214        assert_eq!(service.counters.filtered_tx.load(Ordering::Relaxed), 1);
2215        let event_rx = register(
2216            &mut service,
2217            SubscriptionSpec {
2218                kind: SubscriptionKind::Events,
2219                query: Some(event_type_query(
2220                    "0x0000000000000000000000000000000000000000000000000000000000000002::coin::CoinEvent",
2221                )),
2222            },
2223        )
2224        .await
2225        .unwrap();
2226        assert_eq!(service.counters.filtered_event.load(Ordering::Relaxed), 1);
2227
2228        // Departed clients are finalized by their shard-owned lifecycle
2229        // guards when the next dispatch observes the closed channels.
2230        drop(tx_rx);
2231        drop(event_rx);
2232        service.handle_checkpoint(checkpoint(1).into()).await;
2233        drain(&mut shards);
2234
2235        assert_eq!(service.counters.filtered_tx.load(Ordering::Relaxed), 0);
2236        assert_eq!(service.counters.filtered_event.load(Ordering::Relaxed), 0);
2237        assert_eq!(inflight_subscribers(&service.metrics), 0);
2238    }
2239
2240    #[tokio::test]
2241    async fn unfiltered_subscribers_receive_all_matches() {
2242        let (mut service, mut shards) = test_service(1);
2243        let mut tx_rx = register(
2244            &mut service,
2245            SubscriptionSpec {
2246                kind: SubscriptionKind::Transactions,
2247                query: None,
2248            },
2249        )
2250        .await
2251        .unwrap();
2252        let mut ev_rx = register(
2253            &mut service,
2254            SubscriptionSpec {
2255                kind: SubscriptionKind::Events,
2256                query: None,
2257            },
2258        )
2259        .await
2260        .unwrap();
2261
2262        let event_checkpoint = checkpoint_with_events(1);
2263        let tx_lo = event_checkpoint.summary.data().network_total_transactions
2264            - event_checkpoint.transactions.len() as u64;
2265        service.handle_checkpoint(event_checkpoint.into()).await;
2266        drain(&mut shards);
2267
2268        assert!(matches!(
2269            tx_rx.recv().await.unwrap(),
2270            SubscriptionUpdate::WatermarkTick {
2271                checkpoint: 0,
2272                tx_hi
2273            } if tx_hi == tx_lo
2274        ));
2275        assert!(matches!(
2276            ev_rx.recv().await.unwrap(),
2277            SubscriptionUpdate::WatermarkTick {
2278                checkpoint: 0,
2279                tx_hi
2280            } if tx_hi == tx_lo
2281        ));
2282
2283        // Unfiltered matches arrive as O(1) "all" payloads that the index
2284        // accessors expand to every transaction / event.
2285        let matched = match tx_rx.recv().await.unwrap() {
2286            SubscriptionUpdate::Matched(matched) => matched,
2287            SubscriptionUpdate::WatermarkTick { .. } => panic!("expected a match, got a tick"),
2288        };
2289        assert!(matches!(
2290            matched.matches,
2291            SubscriptionMatches::AllTransactions
2292        ));
2293        let tx_count = matched.checkpoint.transactions().len() as u32;
2294        assert_eq!(
2295            matched
2296                .matches
2297                .transaction_indices(tx_count)
2298                .unwrap()
2299                .collect::<Vec<_>>(),
2300            vec![0, 1]
2301        );
2302
2303        let matched = match ev_rx.recv().await.unwrap() {
2304            SubscriptionUpdate::Matched(matched) => matched,
2305            SubscriptionUpdate::WatermarkTick { .. } => panic!("expected a match, got a tick"),
2306        };
2307        assert!(matches!(matched.matches, SubscriptionMatches::AllEvents));
2308        assert_eq!(
2309            matched
2310                .matches
2311                .event_indices(&matched.checkpoint)
2312                .unwrap()
2313                .collect::<Vec<_>>(),
2314            vec![(1, 0), (1, 1)]
2315        );
2316
2317        // A checkpoint with no transactions yields no frame for either
2318        // subscriber: empty match sets stay on the watermark-tick path.
2319        service.handle_checkpoint(checkpoint(2).into()).await;
2320        drain(&mut shards);
2321        assert!(tx_rx.try_recv().is_err());
2322        assert!(ev_rx.try_recv().is_err());
2323    }
2324
2325    #[test]
2326    fn event_indices_flatten_sparse_matches_in_order() {
2327        let matches = SubscriptionMatches::Events(vec![(0, vec![1, 2]), (3, vec![0])]);
2328        let checkpoint = checkpoint(1).into();
2329        assert_eq!(
2330            matches
2331                .event_indices(&checkpoint)
2332                .unwrap()
2333                .collect::<Vec<_>>(),
2334            vec![(0, 1), (0, 2), (3, 0)]
2335        );
2336    }
2337}