Skip to main content

sui_indexer_alt_framework/
lib.rs

1// Copyright (c) Mysten Labs, Inc.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::collections::BTreeSet;
5use std::ops::Bound;
6use std::sync::Arc;
7use std::time::Duration;
8
9use anyhow::Context;
10use anyhow::bail;
11use anyhow::ensure;
12use cohort::cohorts;
13use ingestion::ArcStreamingClient;
14use ingestion::ClientArgs;
15use ingestion::IngestionConfig;
16use ingestion::IngestionFactory;
17use ingestion::IngestionService;
18use ingestion::ingestion_client::IngestionClient;
19use metrics::IndexerMetrics;
20use prometheus::Registry;
21use sui_indexer_alt_framework_store_traits::ConcurrentStore;
22use sui_indexer_alt_framework_store_traits::Connection;
23use sui_indexer_alt_framework_store_traits::SequentialStore;
24use sui_indexer_alt_framework_store_traits::Store;
25use sui_indexer_alt_framework_store_traits::pipeline_task;
26use tracing::info;
27
28use crate::metrics::IngestionMetrics;
29use crate::pipeline::Processor;
30use crate::pipeline::concurrent::ConcurrentConfig;
31use crate::pipeline::concurrent::{self};
32use crate::pipeline::sequential::SequentialConfig;
33use crate::pipeline::sequential::{self};
34use crate::service::Service;
35
36pub use sui_field_count::FieldCount;
37pub use sui_futures::service;
38/// External users access the store trait through framework::store
39pub use sui_indexer_alt_framework_store_traits as store;
40pub use sui_types as types;
41
42#[cfg(feature = "cluster")]
43pub mod cluster;
44mod cohort;
45pub mod config;
46pub mod ingestion;
47pub mod metrics;
48pub mod pipeline;
49#[cfg(feature = "postgres")]
50pub mod postgres;
51
52#[cfg(test)]
53pub mod mocks;
54
55/// Command-line arguments for the indexer
56#[derive(clap::Args, Default, Debug, Clone)]
57pub struct IndexerArgs {
58    /// Override the next checkpoint for all pipelines without a committer watermark to start
59    /// processing from, which is 0 by default. Pipelines with existing watermarks will ignore this
60    /// setting and always resume from their committer watermark + 1.
61    ///
62    /// Setting this value indirectly affects ingestion, as the checkpoint to start ingesting from
63    /// is the minimum across all pipelines' next checkpoints.
64    #[arg(long)]
65    pub first_checkpoint: Option<u64>,
66
67    /// Override for the checkpoint to end ingestion at (inclusive) -- useful for backfills. By
68    /// default, ingestion will not stop, and will continue to poll for new checkpoints.
69    #[arg(long)]
70    pub last_checkpoint: Option<u64>,
71
72    /// Only run the following pipelines. If not provided, all pipelines found in the
73    /// configuration file will be run.
74    #[arg(long, action = clap::ArgAction::Append)]
75    pub pipeline: Vec<String>,
76
77    /// Additional configurations for running a tasked indexer.
78    #[clap(flatten)]
79    pub task: TaskArgs,
80}
81
82/// Command-line arguments for configuring a tasked indexer.
83#[derive(clap::Parser, Default, Debug, Clone)]
84pub struct TaskArgs {
85    /// An optional task name for this indexer. When set, pipelines will record watermarks using the
86    /// delimiter defined on the store. This allows the same pipelines to run under multiple
87    /// indexers (e.g. for backfills or temporary workflows) while maintaining separate watermark
88    /// entries in the database.
89    ///
90    /// By default there is no task name, and watermarks are keyed only by `pipeline`.
91    ///
92    /// Sequential pipelines cannot be attached to a tasked indexer, unless `--pipeline` excludes
93    /// them.
94    ///
95    /// The framework ensures that tasked pipelines never commit checkpoints below the main
96    /// pipeline’s pruner watermark. Requires `--reader-interval-ms`.
97    #[arg(long, requires = "reader_interval_ms")]
98    task: Option<String>,
99
100    /// The interval in milliseconds at which each of the pipelines on a tasked indexer should
101    /// refetch its main pipeline's reader watermark.
102    ///
103    /// This is required when `--task` is set and should should ideally be set to a value that is
104    /// an order of magnitude smaller than the main pipeline's pruning interval, to ensure this
105    /// task pipeline can pick up the new reader watermark before the main pipeline prunes up to
106    /// it.
107    ///
108    /// If the main pipeline does not have pruning enabled, this value can be set to some high
109    /// value, as the tasked pipeline will never see an updated reader watermark.
110    #[arg(long, requires = "task")]
111    reader_interval_ms: Option<u64>,
112}
113
114pub struct Indexer<S: Store> {
115    /// The storage backend that the indexer uses to write and query indexed data. This
116    /// generic implementation allows for plugging in different storage solutions that implement the
117    /// `Store` trait.
118    store: S,
119
120    /// Prometheus Metrics.
121    metrics: Arc<IndexerMetrics>,
122
123    /// Creates the services that download and disseminate checkpoint data -- one per cohort of
124    /// pipelines with similar distances from the network tip, determined in [Self::run].
125    ingestion_factory: IngestionFactory,
126
127    /// Optional override of the checkpoint lowerbound. When set, pipelines without a committer
128    /// watermark will start processing at this checkpoint.
129    first_checkpoint: Option<u64>,
130
131    /// Optional override of the checkpoint upperbound. When set, the indexer will stop ingestion at
132    /// this checkpoint.
133    last_checkpoint: Option<u64>,
134
135    /// The network's latest checkpoint, when the indexer was started.
136    latest_checkpoint: u64,
137
138    /// The minimum `next_checkpoint` across all pipelines. This is the checkpoint for the indexer
139    /// to start ingesting from.
140    next_checkpoint: u64,
141
142    /// The minimum `next_checkpoint` across all sequential pipelines. This is used to initialize
143    /// the regulator to prevent ingestion from running too far ahead of sequential pipelines.
144    next_sequential_checkpoint: Option<u64>,
145
146    /// An optional task name for this indexer. When set, pipelines will record watermarks using the
147    /// delimiter defined on the store. This allows the same pipelines to run under multiple
148    /// indexers (e.g. for backfills or temporary workflows) while maintaining separate watermark
149    /// entries in the database.
150    ///
151    /// By default there is no task name, and watermarks are keyed only by `pipeline`.
152    ///
153    /// Sequential pipelines cannot be attached to a tasked indexer, unless `--pipeline` excludes
154    /// them.
155    ///
156    /// The framework ensures that tasked pipelines never commit checkpoints below the main
157    /// pipeline’s pruner watermark.
158    task: Option<Task>,
159
160    /// Optional filter for pipelines to run. If `None`, all pipelines added to the indexer will
161    /// run. Any pipelines that are present in this filter but not added to the indexer will yield
162    /// a warning when the indexer is run.
163    enabled_pipelines: Option<BTreeSet<String>>,
164
165    /// Pipelines that have already been registered with the indexer. Used to make sure a pipeline
166    /// with the same name isn't added twice.
167    added_pipelines: BTreeSet<&'static str>,
168
169    /// Pipelines that are registered and enabled, waiting to be grouped into ingestion cohorts and
170    /// started when the indexer is run.
171    pipelines: Vec<PendingPipeline>,
172}
173
174/// Configuration for a tasked indexer.
175#[derive(Clone)]
176pub(crate) struct Task {
177    /// Name of the tasked indexer, to be used with the delimiter defined on the indexer's store to
178    /// record pipeline watermarks.
179    task: String,
180    /// The interval at which each of the pipelines on a tasked indexer should refecth its main
181    /// pipeline's reader watermark.
182    reader_interval: Duration,
183}
184
185/// A pipeline that has been registered but whose tasks have not been started yet. Pipelines are
186/// held in this form until [Indexer::run], when they are grouped into ingestion cohorts by how
187/// far behind the network tip they will resume from.
188struct PendingPipeline {
189    /// The pipeline's name, for logging cohort composition.
190    name: &'static str,
191
192    /// The checkpoint this pipeline will resume processing from.
193    next_checkpoint: u64,
194
195    /// Deferred constructor, invoked in [Indexer::run] once the pipeline's cohort has an
196    /// ingestion service to subscribe to.
197    build: PipelineBuilder,
198}
199
200/// Subscribes a pipeline to its cohort's ingestion service and starts the pipeline's tasks,
201/// returning the service handle over those tasks.
202///
203/// `Sync` is required (in addition to `Send`) because the `Indexer` holding these builders is kept
204/// alive across await points, and the simulator test runtime requires the resulting futures to be
205/// `Sync`. The builder closures only capture `Send + Sync` state (the handler, store, config, and
206/// metrics), so this bound is satisfied.
207type PipelineBuilder = Box<dyn FnOnce(&mut IngestionService) -> Service + Send + Sync>;
208
209impl TaskArgs {
210    pub fn tasked(task: String, reader_interval_ms: u64) -> Self {
211        Self {
212            task: Some(task),
213            reader_interval_ms: Some(reader_interval_ms),
214        }
215    }
216
217    fn into_task(self) -> Option<Task> {
218        Some(Task {
219            task: self.task?,
220            reader_interval: Duration::from_millis(self.reader_interval_ms?),
221        })
222    }
223}
224
225impl<S: Store> Indexer<S> {
226    /// Create a new instance of the indexer framework from a store that implements the `Store`
227    /// trait, along with `indexer_args`, `client_args`, and `ingestion_config`. Together, these
228    /// arguments configure the following:
229    ///
230    /// - What is indexed (which checkpoints, which pipelines, whether to track and update
231    ///   watermarks) and where to serve metrics from,
232    /// - Where to download checkpoints from,
233    /// - Concurrency and buffering parameters for downloading checkpoints.
234    ///
235    /// After initialization, at least one pipeline must be added using [Self::concurrent_pipeline]
236    /// or [Self::sequential_pipeline], before the indexer is started using [Self::run].
237    pub async fn new(
238        store: S,
239        indexer_args: IndexerArgs,
240        client_args: ClientArgs,
241        ingestion_config: IngestionConfig,
242        metrics_prefix: Option<&str>,
243        registry: &Registry,
244    ) -> anyhow::Result<Self> {
245        let ingestion_factory =
246            IngestionFactory::new(client_args, ingestion_config, metrics_prefix, registry).await?;
247        Self::with_ingestion_factory(
248            store,
249            indexer_args,
250            ingestion_factory,
251            metrics_prefix,
252            registry,
253        )
254        .await
255    }
256
257    /// Variant of [`Self::new`] that accepts pre-built ingestion clients, bypassing
258    /// [`ClientArgs`]-driven construction. Callers that supply their own
259    /// [`IngestionClientTrait`] / [`CheckpointStreamingClient`] implementations — for example,
260    /// when embedding the indexer in a fullnode that already has checkpoint data on hand — hand
261    /// them in here, and the indexer creates its ingestion services from them.
262    ///
263    /// All the indexer's ingestion services clone `ingestion_client` and report to its metrics
264    /// handle, and each gets its own clone of the streaming client (if any). If
265    /// `gap_fill_client` is provided, it fetches the checkpoints between each service's position
266    /// and the start of a newly connected stream, instead of `ingestion_client`.
267    ///
268    /// [`IngestionClientTrait`]: crate::ingestion::ingestion_client::IngestionClientTrait
269    /// [`CheckpointStreamingClient`]: crate::ingestion::streaming_client::CheckpointStreamingClient
270    pub async fn with_ingestion_clients(
271        store: S,
272        indexer_args: IndexerArgs,
273        ingestion_client: IngestionClient,
274        streaming_client: Option<ArcStreamingClient>,
275        gap_fill_client: Option<IngestionClient>,
276        ingestion_config: IngestionConfig,
277        metrics_prefix: Option<&str>,
278        registry: &Registry,
279    ) -> anyhow::Result<Self> {
280        let ingestion_factory = IngestionFactory::with_clients(
281            ingestion_client,
282            streaming_client,
283            gap_fill_client,
284            ingestion_config,
285        );
286        Self::with_ingestion_factory(
287            store,
288            indexer_args,
289            ingestion_factory,
290            metrics_prefix,
291            registry,
292        )
293        .await
294    }
295
296    /// Common assembly point for [`Self::new`] and [`Self::with_ingestion_clients`]: probes the
297    /// network tip through the factory and stamps the fields onto the struct.
298    async fn with_ingestion_factory(
299        store: S,
300        indexer_args: IndexerArgs,
301        ingestion_factory: IngestionFactory,
302        metrics_prefix: Option<&str>,
303        registry: &Registry,
304    ) -> anyhow::Result<Self> {
305        let IndexerArgs {
306            first_checkpoint,
307            last_checkpoint,
308            pipeline,
309            task,
310        } = indexer_args;
311
312        let metrics = IndexerMetrics::new(metrics_prefix, registry);
313
314        let latest_checkpoint = ingestion_factory.latest_checkpoint_number().await?;
315
316        info!(latest_checkpoint);
317
318        Ok(Self {
319            store,
320            metrics,
321            ingestion_factory,
322            first_checkpoint,
323            last_checkpoint,
324            latest_checkpoint,
325            next_checkpoint: u64::MAX,
326            next_sequential_checkpoint: None,
327            task: task.into_task(),
328            enabled_pipelines: if pipeline.is_empty() {
329                None
330            } else {
331                Some(pipeline.into_iter().collect())
332            },
333            added_pipelines: BTreeSet::new(),
334            pipelines: vec![],
335        })
336    }
337
338    /// The store used by the indexer.
339    pub fn store(&self) -> &S {
340        &self.store
341    }
342
343    /// The ingestion client used by the indexer to fetch checkpoints.
344    pub fn ingestion_client(&self) -> &IngestionClient {
345        self.ingestion_factory.ingestion_client()
346    }
347
348    /// The indexer's metrics.
349    pub fn indexer_metrics(&self) -> &Arc<IndexerMetrics> {
350        &self.metrics
351    }
352
353    /// The ingestion metrics shared by all of this indexer's ingestion services.
354    pub fn ingestion_metrics(&self) -> &Arc<IngestionMetrics> {
355        self.ingestion_factory.metrics()
356    }
357
358    /// The pipelines that this indexer will run: those registered with it and not excluded by
359    /// `--pipeline`.
360    pub fn pipelines(&self) -> impl Iterator<Item = &'static str> + '_ {
361        self.pipelines.iter().map(|p| p.name)
362    }
363
364    /// The minimum next checkpoint across all sequential pipelines. This value is used to
365    /// initialize the ingestion regulator's high watermark to prevent ingestion from running
366    /// too far ahead of sequential pipelines.
367    pub fn next_sequential_checkpoint(&self) -> Option<u64> {
368        self.next_sequential_checkpoint
369    }
370
371    /// Group the registered pipelines into ingestion cohorts by how far behind the network tip
372    /// they will resume from, and start one ingestion service per cohort. Each cohort ingests
373    /// from the smallest `next_checkpoint` among its members, so pipelines near the tip are not
374    /// held back (through channel backpressure) by pipelines that have a long backfill ahead of
375    /// them.
376    ///
377    /// Ingestion will stop after consuming the configured `last_checkpoint` if one is provided.
378    /// Note that a pipeline dropping its checkpoint subscription only winds down its own
379    /// cohort's ingestion service; other cohorts keep running.
380    pub async fn run(self) -> anyhow::Result<Service> {
381        let Self {
382            ingestion_factory,
383            last_checkpoint,
384            latest_checkpoint,
385            enabled_pipelines,
386            pipelines,
387            ..
388        } = self;
389
390        if let Some(enabled_pipelines) = enabled_pipelines {
391            ensure!(
392                enabled_pipelines.is_empty(),
393                "Tried to enable pipelines that this indexer does not know about: \
394                {enabled_pipelines:#?}",
395            );
396        }
397
398        ensure!(!pipelines.is_empty(), "No pipelines registered to run");
399
400        let groups = cohorts(
401            pipelines,
402            latest_checkpoint,
403            ingestion_factory.config().min_cohort_boundary,
404        );
405
406        info!(
407            cohorts = groups.len(),
408            latest_checkpoint, "Grouped pipelines into ingestion cohorts"
409        );
410
411        let end = last_checkpoint.map_or(Bound::Unbounded, Bound::Included);
412
413        let mut service = Service::new();
414        for (cohort, group) in groups.into_iter().enumerate() {
415            let mut ingestion = ingestion_factory.create(cohort);
416
417            let mut start = u64::MAX;
418            let mut names = Vec::with_capacity(group.len());
419            let mut members = Vec::with_capacity(group.len());
420            for pending in group {
421                start = start.min(pending.next_checkpoint);
422                names.push(pending.name);
423                members.push((pending.build)(&mut ingestion));
424            }
425
426            info!(cohort, start, end = last_checkpoint, pipelines = ?names, "Ingestion range");
427
428            let mut cohort_service = ingestion
429                .run((Bound::Included(start), end))
430                .await
431                .context("Failed to start ingestion service")?;
432
433            for member in members {
434                cohort_service = cohort_service.merge(member);
435            }
436
437            service = service.merge(cohort_service);
438        }
439
440        Ok(service)
441    }
442
443    /// Determine the checkpoint for the pipeline to resume processing from. This is either the
444    /// checkpoint after its watermark, or if that doesn't exist, then the provided
445    /// [Self::first_checkpoint], and if that is not set, then 0 (genesis).
446    ///
447    /// Update the starting ingestion checkpoint as the minimum across all the next checkpoints
448    /// calculated above.
449    ///
450    /// Returns `Ok(None)` if the pipeline is disabled.
451    async fn add_pipeline<P: Processor + 'static>(
452        &mut self,
453        pipeline_task: String,
454        retention: Option<u64>,
455    ) -> anyhow::Result<Option<u64>> {
456        ensure!(
457            self.added_pipelines.insert(P::NAME),
458            "Pipeline {:?} already added",
459            P::NAME,
460        );
461
462        if let Some(enabled_pipelines) = &mut self.enabled_pipelines
463            && !enabled_pipelines.remove(P::NAME)
464        {
465            info!(pipeline = P::NAME, "Skipping");
466            return Ok(None);
467        }
468
469        // Create a new record based on `proposed_next_checkpoint` if one does not exist.
470        // Otherwise, use the existing record and disregard the proposed value.
471        let proposed_next_checkpoint = if let Some(first_checkpoint) = self.first_checkpoint {
472            first_checkpoint
473        } else if let Some(retention) = retention {
474            self.latest_checkpoint.saturating_sub(retention)
475        } else {
476            0
477        };
478        let mut conn = self.store.connect().await?;
479        let init_watermark = conn
480            .init_watermark(&pipeline_task, proposed_next_checkpoint.checked_sub(1))
481            .await
482            .with_context(|| format!("Failed to init watermark for {pipeline_task}"))?;
483
484        let next_checkpoint = if let Some(init_watermark) = init_watermark {
485            if let Some(checkpoint_hi_inclusive) = init_watermark.checkpoint_hi_inclusive {
486                checkpoint_hi_inclusive + 1
487            } else {
488                0
489            }
490        } else {
491            proposed_next_checkpoint
492        };
493
494        self.next_checkpoint = next_checkpoint.min(self.next_checkpoint);
495
496        Ok(Some(next_checkpoint))
497    }
498}
499
500impl<S: ConcurrentStore> Indexer<S> {
501    /// Adds a new pipeline to this indexer. Its tasks are started when the indexer is run, and
502    /// will be idle until its cohort's ingestion service serves it checkpoint data.
503    ///
504    /// Concurrent pipelines commit checkpoint data out-of-order to maximise throughput, and they
505    /// keep the watermark table up-to-date with the highest point they can guarantee all data
506    /// exists for, for their pipeline.
507    pub async fn concurrent_pipeline<H: concurrent::Handler<Store = S>>(
508        &mut self,
509        handler: H,
510        config: ConcurrentConfig,
511    ) -> anyhow::Result<()> {
512        let pipeline_task =
513            pipeline_task::<S>(H::NAME, self.task.as_ref().map(|t| t.task.as_str()))?;
514        let retention = config.pruner.as_ref().map(|p| p.retention);
515        let Some(next_checkpoint) = self.add_pipeline::<H>(pipeline_task, retention).await? else {
516            return Ok(());
517        };
518
519        let store = self.store.clone();
520        let task = self.task.clone();
521        let metrics = self.metrics.clone();
522        self.pipelines.push(PendingPipeline {
523            name: H::NAME,
524            next_checkpoint,
525            build: Box::new(move |ingestion| {
526                let checkpoint_rx =
527                    ingestion.subscribe_bounded(config.ingestion.subscriber_channel_size());
528                concurrent::pipeline::<H>(
529                    handler,
530                    next_checkpoint,
531                    config,
532                    store,
533                    task,
534                    checkpoint_rx,
535                    metrics,
536                )
537            }),
538        });
539
540        Ok(())
541    }
542}
543
544impl<T: SequentialStore> Indexer<T> {
545    /// Adds a new pipeline to this indexer. Its tasks are started when the indexer is run, and
546    /// will be idle until its cohort's ingestion service serves it checkpoint data.
547    ///
548    /// Sequential pipelines commit checkpoint data in-order which sacrifices throughput, but may be
549    /// required to handle pipelines that modify data in-place (where each update is not an insert,
550    /// but could be a modification of an existing row, where ordering between updates is
551    /// important).
552    pub async fn sequential_pipeline<H: sequential::Handler<Store = T>>(
553        &mut self,
554        handler: H,
555        config: SequentialConfig,
556    ) -> anyhow::Result<()> {
557        // A pipeline excluded by `--pipeline` is skipped by `add_pipeline` below, so it does not
558        // stop a tasked indexer from running the pipelines that are enabled.
559        let enabled = self
560            .enabled_pipelines
561            .as_ref()
562            .is_none_or(|e| e.contains(H::NAME));
563        if self.task.is_some() && enabled {
564            bail!(
565                "Sequential pipelines do not support pipeline tasks. \
566                These pipelines guarantee that each checkpoint is committed exactly once and in order. \
567                Running the same pipeline under a different task would violate these guarantees."
568            );
569        }
570
571        let Some(next_checkpoint) = self.add_pipeline::<H>(H::NAME.to_owned(), None).await? else {
572            return Ok(());
573        };
574
575        // Track the minimum next_checkpoint across all sequential pipelines
576        self.next_sequential_checkpoint = Some(
577            self.next_sequential_checkpoint
578                .map_or(next_checkpoint, |n| n.min(next_checkpoint)),
579        );
580
581        let store = self.store.clone();
582        let metrics = self.metrics.clone();
583        self.pipelines.push(PendingPipeline {
584            name: H::NAME,
585            next_checkpoint,
586            build: Box::new(move |ingestion| {
587                let checkpoint_rx =
588                    ingestion.subscribe_bounded(config.ingestion.subscriber_channel_size());
589                sequential::pipeline::<H>(
590                    handler,
591                    next_checkpoint,
592                    config,
593                    store,
594                    checkpoint_rx,
595                    metrics,
596                )
597            }),
598        });
599
600        Ok(())
601    }
602}
603
604#[cfg(test)]
605mod tests {
606    use std::sync::Arc;
607
608    use async_trait::async_trait;
609    use clap::Parser;
610    use sui_indexer_alt_framework_store_traits::PrunerWatermark;
611    use sui_synthetic_ingestion::synthetic_ingestion;
612    use tokio::sync::watch;
613
614    use crate::FieldCount;
615    use crate::config::ConcurrencyConfig;
616    use crate::ingestion::ingestion_client::IngestionClientArgs;
617    use crate::ingestion::store_client::ObjectStoreWatermark;
618    use crate::ingestion::store_client::WATERMARK_PATH;
619    use crate::mocks::store::FallibleMockStore;
620    use crate::pipeline::CommitterConfig;
621    use crate::pipeline::Processor;
622    use crate::pipeline::concurrent::ConcurrentConfig;
623    use crate::store::CommitterWatermark;
624    use crate::store::ConcurrentConnection as _;
625
626    use super::*;
627
628    #[allow(dead_code)]
629    #[derive(Clone, FieldCount)]
630    struct MockValue(u64);
631
632    /// A handler that can be controlled externally to block checkpoint processing.
633    struct ControllableHandler {
634        /// Process checkpoints less than or equal to this value.
635        process_below: watch::Receiver<u64>,
636    }
637
638    impl ControllableHandler {
639        fn with_limit(limit: u64) -> (Self, watch::Sender<u64>) {
640            let (tx, rx) = watch::channel(limit);
641            (Self { process_below: rx }, tx)
642        }
643    }
644
645    #[async_trait]
646    impl Processor for ControllableHandler {
647        const NAME: &'static str = "controllable";
648        type Value = MockValue;
649
650        async fn process(
651            &self,
652            checkpoint: &Arc<sui_types::full_checkpoint_content::Checkpoint>,
653        ) -> anyhow::Result<Vec<Self::Value>> {
654            let cp_num = checkpoint.summary.sequence_number;
655
656            // Wait until the checkpoint is allowed to be processed
657            self.process_below
658                .clone()
659                .wait_for(|&limit| cp_num <= limit)
660                .await
661                .ok();
662
663            Ok(vec![MockValue(cp_num)])
664        }
665    }
666
667    #[async_trait]
668    impl concurrent::Handler for ControllableHandler {
669        type Store = FallibleMockStore;
670        type Batch = Vec<MockValue>;
671
672        fn batch(
673            &self,
674            batch: &mut Self::Batch,
675            values: &mut std::vec::IntoIter<Self::Value>,
676        ) -> concurrent::BatchStatus {
677            batch.extend(values);
678            concurrent::BatchStatus::Ready
679        }
680
681        async fn commit<'a>(
682            &self,
683            batch: &Self::Batch,
684            conn: &mut <Self::Store as Store>::Connection<'a>,
685        ) -> anyhow::Result<usize> {
686            for value in batch {
687                conn.0
688                    .commit_data(Self::NAME, value.0, vec![value.0])
689                    .await?;
690            }
691            Ok(batch.len())
692        }
693    }
694
695    macro_rules! test_pipeline {
696        ($handler:ident, $name:literal) => {
697            struct $handler;
698
699            #[async_trait]
700            impl Processor for $handler {
701                const NAME: &'static str = $name;
702                type Value = MockValue;
703                async fn process(
704                    &self,
705                    checkpoint: &Arc<sui_types::full_checkpoint_content::Checkpoint>,
706                ) -> anyhow::Result<Vec<Self::Value>> {
707                    Ok(vec![MockValue(checkpoint.summary.sequence_number)])
708                }
709            }
710
711            #[async_trait]
712            impl crate::pipeline::concurrent::Handler for $handler {
713                type Store = FallibleMockStore;
714                type Batch = Vec<Self::Value>;
715
716                fn batch(
717                    &self,
718                    batch: &mut Self::Batch,
719                    values: &mut std::vec::IntoIter<Self::Value>,
720                ) -> crate::pipeline::concurrent::BatchStatus {
721                    batch.extend(values);
722                    crate::pipeline::concurrent::BatchStatus::Pending
723                }
724
725                async fn commit<'a>(
726                    &self,
727                    batch: &Self::Batch,
728                    conn: &mut <Self::Store as Store>::Connection<'a>,
729                ) -> anyhow::Result<usize> {
730                    for value in batch {
731                        conn.0
732                            .commit_data(Self::NAME, value.0, vec![value.0])
733                            .await?;
734                    }
735                    Ok(batch.len())
736                }
737            }
738
739            #[async_trait]
740            impl crate::pipeline::sequential::Handler for $handler {
741                type Store = FallibleMockStore;
742                type Batch = Vec<Self::Value>;
743
744                fn batch(&self, batch: &mut Self::Batch, values: std::vec::IntoIter<Self::Value>) {
745                    batch.extend(values);
746                }
747
748                async fn commit<'a>(
749                    &self,
750                    _batch: &Self::Batch,
751                    _conn: &mut <Self::Store as Store>::Connection<'a>,
752                ) -> anyhow::Result<usize> {
753                    Ok(1)
754                }
755            }
756        };
757    }
758
759    test_pipeline!(MockHandler, "test_processor");
760    test_pipeline!(SequentialHandler, "sequential_handler");
761    test_pipeline!(MockCheckpointSequenceNumberHandler, "test");
762
763    fn init_ingestion_dir(latest_checkpoint: Option<u64>) -> tempfile::TempDir {
764        let dir = tempfile::tempdir().unwrap();
765        if let Some(cp) = latest_checkpoint {
766            let watermark_path = dir.path().join(WATERMARK_PATH);
767            std::fs::create_dir_all(watermark_path.parent().unwrap()).unwrap();
768            let watermark = ObjectStoreWatermark {
769                checkpoint_hi_inclusive: Some(cp),
770            };
771            std::fs::write(watermark_path, serde_json::to_string(&watermark).unwrap()).unwrap();
772        }
773        dir
774    }
775
776    /// If `ingestion_data` is `Some((num_checkpoints, checkpoint_size))`, synthetic ingestion
777    /// data will be generated in the temp directory before creating the indexer.
778    async fn create_test_indexer(
779        store: FallibleMockStore,
780        indexer_args: IndexerArgs,
781        registry: &Registry,
782        ingestion_data: Option<(u64, u64)>,
783    ) -> (Indexer<FallibleMockStore>, tempfile::TempDir) {
784        let temp_dir = init_ingestion_dir(None);
785        if let Some((num_checkpoints, checkpoint_size)) = ingestion_data {
786            synthetic_ingestion::generate_ingestion(synthetic_ingestion::Config {
787                ingestion_dir: temp_dir.path().to_owned(),
788                starting_checkpoint: 0,
789                num_checkpoints,
790                checkpoint_size,
791            })
792            .await;
793        }
794        let client_args = ClientArgs {
795            ingestion: IngestionClientArgs {
796                local_ingestion_path: Some(temp_dir.path().to_owned()),
797                ..Default::default()
798            },
799            ..Default::default()
800        };
801        let indexer = Indexer::new(
802            store,
803            indexer_args,
804            client_args,
805            IngestionConfig::default(),
806            None,
807            registry,
808        )
809        .await
810        .unwrap();
811        (indexer, temp_dir)
812    }
813
814    async fn set_committer_watermark(
815        conn: &mut <FallibleMockStore as Store>::Connection<'_>,
816        name: &str,
817        hi: u64,
818    ) {
819        conn.set_committer_watermark(
820            name,
821            CommitterWatermark {
822                checkpoint_hi_inclusive: hi,
823                ..Default::default()
824            },
825        )
826        .await
827        .unwrap();
828    }
829
830    async fn add_concurrent<H: concurrent::Handler<Store = FallibleMockStore>>(
831        indexer: &mut Indexer<FallibleMockStore>,
832        handler: H,
833    ) {
834        indexer
835            .concurrent_pipeline(handler, ConcurrentConfig::default())
836            .await
837            .unwrap();
838    }
839
840    async fn add_sequential<H: sequential::Handler<Store = FallibleMockStore>>(
841        indexer: &mut Indexer<FallibleMockStore>,
842        handler: H,
843    ) {
844        indexer
845            .sequential_pipeline(handler, SequentialConfig::default())
846            .await
847            .unwrap();
848    }
849
850    /// Seed `store` so `near` resumes just past a fake network tip of 100,000 (its distance
851    /// from the tip saturates to zero) while `far` resumes 99,990 checkpoints behind it, and
852    /// return an ingestion directory advertising that tip with checkpoints 0..60 on disk.
853    /// Cohorts are determined by distance from the advertised tip, but ingestion stops at
854    /// `last_checkpoint`. Checkpoint 0 must exist because the ingestion client derives the
855    /// chain identifier from it, and retries indefinitely until it appears.
856    async fn multi_cohort_setup(
857        store: &FallibleMockStore,
858        near: &str,
859        far: &str,
860    ) -> tempfile::TempDir {
861        let mut conn = store.connect().await.unwrap();
862        set_committer_watermark(&mut conn, near, 100_100).await;
863        set_committer_watermark(&mut conn, far, 9).await;
864
865        let temp_dir = init_ingestion_dir(Some(100_000));
866        synthetic_ingestion::generate_ingestion(synthetic_ingestion::Config {
867            ingestion_dir: temp_dir.path().to_owned(),
868            starting_checkpoint: 0,
869            num_checkpoints: 60,
870            checkpoint_size: 1,
871        })
872        .await;
873
874        temp_dir
875    }
876
877    macro_rules! assert_out_of_order {
878        ($metrics:expr, $pipeline:expr, $expected:expr) => {
879            assert_eq!(
880                $metrics
881                    .total_watermarks_out_of_order
882                    .get_metric_with_label_values(&[$pipeline])
883                    .unwrap()
884                    .get(),
885                $expected,
886            );
887        };
888    }
889
890    async fn test_init_watermark(
891        first_checkpoint: Option<u64>,
892        is_concurrent: bool,
893    ) -> (Option<CommitterWatermark>, Option<PrunerWatermark>) {
894        let registry = Registry::new();
895        let store = FallibleMockStore::default();
896
897        test_pipeline!(A, "pipeline_name");
898
899        let mut conn = store.connect().await.unwrap();
900
901        let indexer_args = IndexerArgs {
902            first_checkpoint,
903            ..IndexerArgs::default()
904        };
905        let (mut indexer, _temp_dir) =
906            create_test_indexer(store.clone(), indexer_args, &registry, None).await;
907
908        if is_concurrent {
909            add_concurrent(&mut indexer, A).await;
910        } else {
911            add_sequential(&mut indexer, A).await;
912        }
913
914        (
915            conn.committer_watermark(A::NAME).await.unwrap(),
916            conn.pruner_watermark(A::NAME, Duration::ZERO)
917                .await
918                .unwrap(),
919        )
920    }
921
922    const LATEST_CHECKPOINT: u64 = 10;
923
924    /// Set up an indexer as if the network is at `latest_checkpoint` (the next checkpoint to
925    /// ingest). Runs a single concurrent pipeline with the given config. If `watermark` is
926    /// provided, the pipeline's high watermark is pre-set; `first_checkpoint` controls where
927    /// ingestion begins.
928    async fn test_next_checkpoint(
929        watermark: Option<u64>,
930        first_checkpoint: Option<u64>,
931        concurrent_config: ConcurrentConfig,
932    ) -> Indexer<FallibleMockStore> {
933        let registry = Registry::new();
934        let store = FallibleMockStore::default();
935
936        test_pipeline!(A, "concurrent_a");
937
938        if let Some(checkpoint_hi_inclusive) = watermark {
939            let mut conn = store.connect().await.unwrap();
940            conn.set_committer_watermark(
941                A::NAME,
942                CommitterWatermark {
943                    checkpoint_hi_inclusive,
944                    ..Default::default()
945                },
946            )
947            .await
948            .unwrap();
949        }
950
951        let temp_dir = init_ingestion_dir(Some(LATEST_CHECKPOINT));
952        let mut indexer = Indexer::new(
953            store,
954            IndexerArgs {
955                first_checkpoint,
956                ..Default::default()
957            },
958            ClientArgs {
959                ingestion: IngestionClientArgs {
960                    local_ingestion_path: Some(temp_dir.path().to_owned()),
961                    ..Default::default()
962                },
963                ..Default::default()
964            },
965            IngestionConfig::default(),
966            None,
967            &registry,
968        )
969        .await
970        .unwrap();
971
972        assert_eq!(indexer.latest_checkpoint, LATEST_CHECKPOINT);
973
974        indexer
975            .concurrent_pipeline::<A>(A, concurrent_config)
976            .await
977            .unwrap();
978
979        indexer
980    }
981
982    fn pruner_config(retention: u64) -> ConcurrentConfig {
983        ConcurrentConfig {
984            pruner: Some(concurrent::PrunerConfig {
985                retention,
986                ..Default::default()
987            }),
988            ..Default::default()
989        }
990    }
991
992    #[test]
993    fn test_arg_parsing() {
994        #[derive(Parser)]
995        struct Args {
996            #[clap(flatten)]
997            indexer: IndexerArgs,
998        }
999
1000        let args = Args::try_parse_from([
1001            "cmd",
1002            "--first-checkpoint",
1003            "10",
1004            "--last-checkpoint",
1005            "100",
1006            "--pipeline",
1007            "a",
1008            "--pipeline",
1009            "b",
1010            "--task",
1011            "t",
1012            "--reader-interval-ms",
1013            "5000",
1014        ])
1015        .unwrap();
1016
1017        assert_eq!(args.indexer.first_checkpoint, Some(10));
1018        assert_eq!(args.indexer.last_checkpoint, Some(100));
1019        assert_eq!(args.indexer.pipeline, vec!["a", "b"]);
1020        assert_eq!(args.indexer.task.task, Some("t".to_owned()));
1021        assert_eq!(args.indexer.task.reader_interval_ms, Some(5000));
1022    }
1023
1024    /// next_checkpoint is smallest among existing watermarks + 1.
1025    #[tokio::test]
1026    async fn test_next_checkpoint_all_pipelines_have_watermarks() {
1027        let registry = Registry::new();
1028        let store = FallibleMockStore::default();
1029
1030        test_pipeline!(A, "concurrent_a");
1031        test_pipeline!(B, "concurrent_b");
1032        test_pipeline!(C, "sequential_c");
1033        test_pipeline!(D, "sequential_d");
1034
1035        let mut conn = store.connect().await.unwrap();
1036
1037        conn.init_watermark(A::NAME, Some(0)).await.unwrap();
1038        set_committer_watermark(&mut conn, A::NAME, 100).await;
1039
1040        conn.init_watermark(B::NAME, Some(0)).await.unwrap();
1041        set_committer_watermark(&mut conn, B::NAME, 10).await;
1042
1043        conn.init_watermark(C::NAME, Some(0)).await.unwrap();
1044        set_committer_watermark(&mut conn, C::NAME, 1).await;
1045
1046        conn.init_watermark(D::NAME, Some(0)).await.unwrap();
1047        set_committer_watermark(&mut conn, D::NAME, 50).await;
1048
1049        let (mut indexer, _temp_dir) =
1050            create_test_indexer(store, IndexerArgs::default(), &registry, None).await;
1051
1052        add_concurrent(&mut indexer, A).await;
1053        add_concurrent(&mut indexer, B).await;
1054        add_sequential(&mut indexer, C).await;
1055        add_sequential(&mut indexer, D).await;
1056
1057        assert_eq!(indexer.first_checkpoint, None);
1058        assert_eq!(indexer.last_checkpoint, None);
1059        assert_eq!(indexer.latest_checkpoint, 0);
1060        assert_eq!(indexer.next_checkpoint, 2);
1061        assert_eq!(indexer.next_sequential_checkpoint, Some(2));
1062    }
1063
1064    /// next_checkpoint is 0 when at least one pipeline has no watermark.
1065    #[tokio::test]
1066    async fn test_next_checkpoint_not_all_pipelines_have_watermarks() {
1067        let registry = Registry::new();
1068        let store = FallibleMockStore::default();
1069
1070        test_pipeline!(A, "concurrent_a");
1071        test_pipeline!(B, "concurrent_b");
1072        test_pipeline!(C, "sequential_c");
1073        test_pipeline!(D, "sequential_d");
1074
1075        let mut conn = store.connect().await.unwrap();
1076        set_committer_watermark(&mut conn, B::NAME, 10).await;
1077        set_committer_watermark(&mut conn, C::NAME, 1).await;
1078
1079        let (mut indexer, _temp_dir) =
1080            create_test_indexer(store, IndexerArgs::default(), &registry, None).await;
1081
1082        add_concurrent(&mut indexer, A).await;
1083        add_concurrent(&mut indexer, B).await;
1084        add_sequential(&mut indexer, C).await;
1085        add_sequential(&mut indexer, D).await;
1086
1087        assert_eq!(indexer.first_checkpoint, None);
1088        assert_eq!(indexer.last_checkpoint, None);
1089        assert_eq!(indexer.latest_checkpoint, 0);
1090        assert_eq!(indexer.next_checkpoint, 0);
1091        assert_eq!(indexer.next_sequential_checkpoint, Some(0));
1092    }
1093
1094    /// `pipelines()` reports every registered pipeline without `--pipeline`, and exactly the
1095    /// registered pipelines that `--pipeline` names with it.
1096    #[tokio::test]
1097    async fn test_pipelines_reports_enabled_pipelines() {
1098        test_pipeline!(A, "concurrent_a");
1099        test_pipeline!(B, "concurrent_b");
1100        test_pipeline!(C, "sequential_c");
1101        test_pipeline!(D, "sequential_d");
1102
1103        let cases = [
1104            (vec![], vec![A::NAME, B::NAME, C::NAME, D::NAME]),
1105            (vec![A::NAME, C::NAME], vec![A::NAME, C::NAME]),
1106        ];
1107
1108        for (filter, expected) in cases {
1109            let args = IndexerArgs {
1110                pipeline: filter.into_iter().map(String::from).collect(),
1111                ..Default::default()
1112            };
1113            let (mut indexer, _temp_dir) =
1114                create_test_indexer(FallibleMockStore::default(), args, &Registry::new(), None)
1115                    .await;
1116
1117            add_concurrent(&mut indexer, A).await;
1118            add_concurrent(&mut indexer, B).await;
1119            add_sequential(&mut indexer, C).await;
1120            add_sequential(&mut indexer, D).await;
1121
1122            let pipelines: BTreeSet<_> = indexer.pipelines().collect();
1123            assert_eq!(pipelines, BTreeSet::from_iter(expected));
1124        }
1125    }
1126
1127    /// next_checkpoint is 1 when smallest committer watermark is 0.
1128    #[tokio::test]
1129    async fn test_next_checkpoint_smallest_is_0() {
1130        let registry = Registry::new();
1131        let store = FallibleMockStore::default();
1132
1133        test_pipeline!(A, "concurrent_a");
1134        test_pipeline!(B, "concurrent_b");
1135        test_pipeline!(C, "sequential_c");
1136        test_pipeline!(D, "sequential_d");
1137
1138        let mut conn = store.connect().await.unwrap();
1139        set_committer_watermark(&mut conn, A::NAME, 100).await;
1140        set_committer_watermark(&mut conn, B::NAME, 10).await;
1141        set_committer_watermark(&mut conn, C::NAME, 1).await;
1142        set_committer_watermark(&mut conn, D::NAME, 0).await;
1143
1144        let (mut indexer, _temp_dir) =
1145            create_test_indexer(store, IndexerArgs::default(), &registry, None).await;
1146
1147        add_concurrent(&mut indexer, A).await;
1148        add_concurrent(&mut indexer, B).await;
1149        add_sequential(&mut indexer, C).await;
1150        add_sequential(&mut indexer, D).await;
1151
1152        assert_eq!(indexer.next_checkpoint, 1);
1153    }
1154
1155    /// next_checkpoint is first_checkpoint when at least one pipeline has no
1156    /// watermark, and first_checkpoint is smallest.
1157    #[tokio::test]
1158    async fn test_next_checkpoint_first_checkpoint_and_no_watermark() {
1159        let registry = Registry::new();
1160        let store = FallibleMockStore::default();
1161
1162        test_pipeline!(A, "concurrent_a");
1163        test_pipeline!(B, "concurrent_b");
1164        test_pipeline!(C, "sequential_c");
1165        test_pipeline!(D, "sequential_d");
1166
1167        let mut conn = store.connect().await.unwrap();
1168        set_committer_watermark(&mut conn, B::NAME, 50).await;
1169        set_committer_watermark(&mut conn, C::NAME, 10).await;
1170
1171        let indexer_args = IndexerArgs {
1172            first_checkpoint: Some(5),
1173            ..Default::default()
1174        };
1175        let (mut indexer, _temp_dir) =
1176            create_test_indexer(store, indexer_args, &registry, None).await;
1177
1178        add_concurrent(&mut indexer, A).await;
1179        add_concurrent(&mut indexer, B).await;
1180        add_sequential(&mut indexer, C).await;
1181        add_sequential(&mut indexer, D).await;
1182
1183        assert_eq!(indexer.first_checkpoint, Some(5));
1184        assert_eq!(indexer.last_checkpoint, None);
1185        assert_eq!(indexer.latest_checkpoint, 0);
1186        assert_eq!(indexer.next_checkpoint, 5);
1187        assert_eq!(indexer.next_sequential_checkpoint, Some(5));
1188    }
1189
1190    /// next_checkpoint is smallest among existing watermarks + 1 if
1191    /// all pipelines have watermarks (ignores first_checkpoint).
1192    #[tokio::test]
1193    async fn test_next_checkpoint_ignore_first_checkpoint() {
1194        let registry = Registry::new();
1195        let store = FallibleMockStore::default();
1196
1197        test_pipeline!(B, "concurrent_b");
1198        test_pipeline!(C, "sequential_c");
1199
1200        let mut conn = store.connect().await.unwrap();
1201        set_committer_watermark(&mut conn, B::NAME, 50).await;
1202        set_committer_watermark(&mut conn, C::NAME, 10).await;
1203
1204        let indexer_args = IndexerArgs {
1205            first_checkpoint: Some(5),
1206            ..Default::default()
1207        };
1208        let (mut indexer, _temp_dir) =
1209            create_test_indexer(store, indexer_args, &registry, None).await;
1210
1211        add_concurrent(&mut indexer, B).await;
1212        add_sequential(&mut indexer, C).await;
1213
1214        assert_eq!(indexer.first_checkpoint, Some(5));
1215        assert_eq!(indexer.last_checkpoint, None);
1216        assert_eq!(indexer.latest_checkpoint, 0);
1217        assert_eq!(indexer.next_checkpoint, 11);
1218        assert_eq!(indexer.next_sequential_checkpoint, Some(11));
1219    }
1220
1221    /// If the first_checkpoint is being considered, because pipelines are missing watermarks, it
1222    /// will not be used as the starting point if it is not the smallest valid committer watermark
1223    /// to resume ingesting from.
1224    #[tokio::test]
1225    async fn test_next_checkpoint_large_first_checkpoint() {
1226        let registry = Registry::new();
1227        let store = FallibleMockStore::default();
1228
1229        test_pipeline!(A, "concurrent_a");
1230        test_pipeline!(B, "concurrent_b");
1231        test_pipeline!(C, "sequential_c");
1232
1233        let mut conn = store.connect().await.unwrap();
1234        set_committer_watermark(&mut conn, B::NAME, 50).await;
1235        set_committer_watermark(&mut conn, C::NAME, 10).await;
1236
1237        let indexer_args = IndexerArgs {
1238            first_checkpoint: Some(24),
1239            ..Default::default()
1240        };
1241        let (mut indexer, _temp_dir) =
1242            create_test_indexer(store, indexer_args, &registry, None).await;
1243
1244        add_concurrent(&mut indexer, A).await;
1245        add_concurrent(&mut indexer, B).await;
1246        add_sequential(&mut indexer, C).await;
1247
1248        assert_eq!(indexer.first_checkpoint, Some(24));
1249        assert_eq!(indexer.last_checkpoint, None);
1250        assert_eq!(indexer.latest_checkpoint, 0);
1251        assert_eq!(indexer.next_checkpoint, 11);
1252        assert_eq!(indexer.next_sequential_checkpoint, Some(11));
1253    }
1254
1255    /// latest_checkpoint is read from the watermark file.
1256    #[tokio::test]
1257    async fn test_latest_checkpoint_from_watermark() {
1258        let registry = Registry::new();
1259        let store = FallibleMockStore::default();
1260        let temp_dir = init_ingestion_dir(Some(30));
1261        let indexer = Indexer::new(
1262            store,
1263            IndexerArgs::default(),
1264            ClientArgs {
1265                ingestion: IngestionClientArgs {
1266                    local_ingestion_path: Some(temp_dir.path().to_owned()),
1267                    ..Default::default()
1268                },
1269                ..Default::default()
1270            },
1271            IngestionConfig::default(),
1272            None,
1273            &registry,
1274        )
1275        .await
1276        .unwrap();
1277
1278        assert_eq!(indexer.latest_checkpoint, 30);
1279    }
1280
1281    /// No watermark, no first_checkpoint, pruner with retention:
1282    /// next_checkpoint = LATEST_CHECKPOINT - retention.
1283    #[tokio::test]
1284    async fn test_next_checkpoint_with_pruner_uses_retention() {
1285        let retention = LATEST_CHECKPOINT - 1;
1286        let indexer = test_next_checkpoint(None, None, pruner_config(retention)).await;
1287        assert_eq!(indexer.next_checkpoint, LATEST_CHECKPOINT - retention);
1288    }
1289
1290    /// No watermark, no first_checkpoint, no pruner: falls back to 0.
1291    #[tokio::test]
1292    async fn test_next_checkpoint_without_pruner_falls_back_to_genesis() {
1293        let indexer = test_next_checkpoint(None, None, ConcurrentConfig::default()).await;
1294        assert_eq!(indexer.next_checkpoint, 0);
1295    }
1296
1297    /// Watermark at 5 takes priority over latest_checkpoint - retention.
1298    #[tokio::test]
1299    async fn test_next_checkpoint_watermark_takes_priority_over_pruner() {
1300        let retention = LATEST_CHECKPOINT - 1;
1301        let indexer = test_next_checkpoint(Some(5), None, pruner_config(retention)).await;
1302        assert_eq!(indexer.next_checkpoint, 6);
1303    }
1304
1305    /// first_checkpoint takes priority over latest_checkpoint - retention when
1306    /// there's no watermark.
1307    #[tokio::test]
1308    async fn test_next_checkpoint_first_checkpoint_takes_priority_over_pruner() {
1309        let retention = LATEST_CHECKPOINT - 1;
1310        let indexer = test_next_checkpoint(None, Some(2), pruner_config(retention)).await;
1311        assert_eq!(indexer.next_checkpoint, 2);
1312    }
1313
1314    /// When retention exceeds latest_checkpoint, saturating_sub clamps to 0.
1315    #[tokio::test]
1316    async fn test_next_checkpoint_retention_exceeds_latest_checkpoint() {
1317        let retention = LATEST_CHECKPOINT + 1;
1318        let indexer = test_next_checkpoint(None, None, pruner_config(retention)).await;
1319        assert_eq!(indexer.next_checkpoint, 0);
1320    }
1321
1322    // test ingestion, all pipelines have watermarks, no first_checkpoint provided
1323    #[tokio::test]
1324    async fn test_indexer_ingestion_existing_watermarks_no_first_checkpoint() {
1325        let registry = Registry::new();
1326        let store = FallibleMockStore::default();
1327
1328        test_pipeline!(A, "concurrent_a");
1329        test_pipeline!(B, "concurrent_b");
1330        test_pipeline!(C, "sequential_c");
1331        test_pipeline!(D, "sequential_d");
1332
1333        let mut conn = store.connect().await.unwrap();
1334        set_committer_watermark(&mut conn, A::NAME, 5).await;
1335        set_committer_watermark(&mut conn, B::NAME, 10).await;
1336        set_committer_watermark(&mut conn, C::NAME, 15).await;
1337        set_committer_watermark(&mut conn, D::NAME, 20).await;
1338
1339        let indexer_args = IndexerArgs {
1340            last_checkpoint: Some(29),
1341            ..Default::default()
1342        };
1343        let (mut indexer, _temp_dir) =
1344            create_test_indexer(store.clone(), indexer_args, &registry, Some((30, 1))).await;
1345
1346        add_concurrent(&mut indexer, A).await;
1347        add_concurrent(&mut indexer, B).await;
1348        add_sequential(&mut indexer, C).await;
1349        add_sequential(&mut indexer, D).await;
1350
1351        let ingestion_metrics = indexer.ingestion_metrics().clone();
1352        let indexer_metrics = indexer.indexer_metrics().clone();
1353
1354        indexer.run().await.unwrap().join().await.unwrap();
1355
1356        assert_eq!(
1357            ingestion_metrics
1358                .total_ingested_checkpoints
1359                .with_label_values(&["0"])
1360                .get(),
1361            24
1362        );
1363        assert_out_of_order!(indexer_metrics, A::NAME, 0);
1364        assert_out_of_order!(indexer_metrics, B::NAME, 5);
1365        assert_out_of_order!(indexer_metrics, C::NAME, 10);
1366        assert_out_of_order!(indexer_metrics, D::NAME, 15);
1367    }
1368
1369    // test ingestion, no pipelines missing watermarks, first_checkpoint provided
1370    #[tokio::test]
1371    async fn test_indexer_ingestion_existing_watermarks_ignore_first_checkpoint() {
1372        let registry = Registry::new();
1373        let store = FallibleMockStore::default();
1374
1375        test_pipeline!(A, "concurrent_a");
1376        test_pipeline!(B, "concurrent_b");
1377        test_pipeline!(C, "sequential_c");
1378        test_pipeline!(D, "sequential_d");
1379
1380        let mut conn = store.connect().await.unwrap();
1381        set_committer_watermark(&mut conn, A::NAME, 5).await;
1382        set_committer_watermark(&mut conn, B::NAME, 10).await;
1383        set_committer_watermark(&mut conn, C::NAME, 15).await;
1384        set_committer_watermark(&mut conn, D::NAME, 20).await;
1385
1386        let indexer_args = IndexerArgs {
1387            first_checkpoint: Some(3),
1388            last_checkpoint: Some(29),
1389            ..Default::default()
1390        };
1391        let (mut indexer, _temp_dir) =
1392            create_test_indexer(store.clone(), indexer_args, &registry, Some((30, 1))).await;
1393
1394        add_concurrent(&mut indexer, A).await;
1395        add_concurrent(&mut indexer, B).await;
1396        add_sequential(&mut indexer, C).await;
1397        add_sequential(&mut indexer, D).await;
1398
1399        let ingestion_metrics = indexer.ingestion_metrics().clone();
1400        let metrics = indexer.indexer_metrics().clone();
1401        indexer.run().await.unwrap().join().await.unwrap();
1402
1403        assert_eq!(
1404            ingestion_metrics
1405                .total_ingested_checkpoints
1406                .with_label_values(&["0"])
1407                .get(),
1408            24
1409        );
1410        assert_out_of_order!(metrics, A::NAME, 0);
1411        assert_out_of_order!(metrics, B::NAME, 5);
1412        assert_out_of_order!(metrics, C::NAME, 10);
1413        assert_out_of_order!(metrics, D::NAME, 15);
1414    }
1415
1416    // test ingestion, some pipelines missing watermarks, no first_checkpoint provided
1417    #[tokio::test]
1418    async fn test_indexer_ingestion_missing_watermarks_no_first_checkpoint() {
1419        let registry = Registry::new();
1420        let store = FallibleMockStore::default();
1421
1422        test_pipeline!(A, "concurrent_a");
1423        test_pipeline!(B, "concurrent_b");
1424        test_pipeline!(C, "sequential_c");
1425        test_pipeline!(D, "sequential_d");
1426
1427        let mut conn = store.connect().await.unwrap();
1428        set_committer_watermark(&mut conn, B::NAME, 10).await;
1429        set_committer_watermark(&mut conn, C::NAME, 15).await;
1430        set_committer_watermark(&mut conn, D::NAME, 20).await;
1431
1432        let indexer_args = IndexerArgs {
1433            last_checkpoint: Some(29),
1434            ..Default::default()
1435        };
1436        let (mut indexer, _temp_dir) =
1437            create_test_indexer(store.clone(), indexer_args, &registry, Some((30, 1))).await;
1438
1439        add_concurrent(&mut indexer, A).await;
1440        add_concurrent(&mut indexer, B).await;
1441        add_sequential(&mut indexer, C).await;
1442        add_sequential(&mut indexer, D).await;
1443
1444        let ingestion_metrics = indexer.ingestion_metrics().clone();
1445        let metrics = indexer.indexer_metrics().clone();
1446        indexer.run().await.unwrap().join().await.unwrap();
1447
1448        assert_eq!(
1449            ingestion_metrics
1450                .total_ingested_checkpoints
1451                .with_label_values(&["0"])
1452                .get(),
1453            30
1454        );
1455        assert_out_of_order!(metrics, A::NAME, 0);
1456        assert_out_of_order!(metrics, B::NAME, 11);
1457        assert_out_of_order!(metrics, C::NAME, 16);
1458        assert_out_of_order!(metrics, D::NAME, 21);
1459    }
1460
1461    // test ingestion, some pipelines missing watermarks, use first_checkpoint
1462    #[tokio::test]
1463    async fn test_indexer_ingestion_use_first_checkpoint() {
1464        let registry = Registry::new();
1465        let store = FallibleMockStore::default();
1466
1467        test_pipeline!(A, "concurrent_a");
1468        test_pipeline!(B, "concurrent_b");
1469        test_pipeline!(C, "sequential_c");
1470        test_pipeline!(D, "sequential_d");
1471
1472        let mut conn = store.connect().await.unwrap();
1473        set_committer_watermark(&mut conn, B::NAME, 10).await;
1474        set_committer_watermark(&mut conn, C::NAME, 15).await;
1475        set_committer_watermark(&mut conn, D::NAME, 20).await;
1476
1477        let indexer_args = IndexerArgs {
1478            first_checkpoint: Some(10),
1479            last_checkpoint: Some(29),
1480            ..Default::default()
1481        };
1482        let (mut indexer, _temp_dir) =
1483            create_test_indexer(store.clone(), indexer_args, &registry, Some((30, 1))).await;
1484
1485        add_concurrent(&mut indexer, A).await;
1486        add_concurrent(&mut indexer, B).await;
1487        add_sequential(&mut indexer, C).await;
1488        add_sequential(&mut indexer, D).await;
1489
1490        let ingestion_metrics = indexer.ingestion_metrics().clone();
1491        let metrics = indexer.indexer_metrics().clone();
1492        indexer.run().await.unwrap().join().await.unwrap();
1493
1494        assert_eq!(
1495            ingestion_metrics
1496                .total_ingested_checkpoints
1497                .with_label_values(&["0"])
1498                .get(),
1499            20
1500        );
1501        assert_out_of_order!(metrics, A::NAME, 0);
1502        assert_out_of_order!(metrics, B::NAME, 1);
1503        assert_out_of_order!(metrics, C::NAME, 6);
1504        assert_out_of_order!(metrics, D::NAME, 11);
1505    }
1506
1507    #[tokio::test]
1508    async fn test_init_watermark_concurrent_no_first_checkpoint() {
1509        let (committer_watermark, pruner_watermark) = test_init_watermark(None, true).await;
1510        assert_eq!(committer_watermark, None);
1511        assert_eq!(pruner_watermark, None);
1512    }
1513
1514    #[tokio::test]
1515    async fn test_init_watermark_concurrent_first_checkpoint_0() {
1516        let (committer_watermark, pruner_watermark) = test_init_watermark(Some(0), true).await;
1517        assert_eq!(committer_watermark, None);
1518        assert_eq!(pruner_watermark, None);
1519    }
1520
1521    #[tokio::test]
1522    async fn test_init_watermark_concurrent_first_checkpoint_1() {
1523        let (committer_watermark, pruner_watermark) = test_init_watermark(Some(1), true).await;
1524
1525        let committer_watermark = committer_watermark.unwrap();
1526        assert_eq!(committer_watermark.checkpoint_hi_inclusive, 0);
1527
1528        let pruner_watermark = pruner_watermark.unwrap();
1529        assert_eq!(pruner_watermark.reader_lo, 1);
1530        assert_eq!(pruner_watermark.pruner_hi, 1);
1531    }
1532
1533    #[tokio::test]
1534    async fn test_init_watermark_sequential() {
1535        let (committer_watermark, pruner_watermark) = test_init_watermark(Some(1), false).await;
1536
1537        let committer_watermark = committer_watermark.unwrap();
1538        assert_eq!(committer_watermark.checkpoint_hi_inclusive, 0);
1539
1540        let pruner_watermark = pruner_watermark.unwrap();
1541        assert_eq!(pruner_watermark.reader_lo, 1);
1542        assert_eq!(pruner_watermark.pruner_hi, 1);
1543    }
1544
1545    #[tokio::test]
1546    async fn test_multiple_sequential_pipelines_next_checkpoint() {
1547        let registry = Registry::new();
1548        let store = FallibleMockStore::default();
1549
1550        let mut conn = store.connect().await.unwrap();
1551        set_committer_watermark(&mut conn, MockHandler::NAME, 10).await;
1552        set_committer_watermark(&mut conn, SequentialHandler::NAME, 5).await;
1553
1554        let indexer_args = IndexerArgs {
1555            first_checkpoint: None,
1556            last_checkpoint: Some(19),
1557            pipeline: vec![],
1558            ..Default::default()
1559        };
1560        let (mut indexer, _temp_dir) =
1561            create_test_indexer(store.clone(), indexer_args, &registry, Some((20, 2))).await;
1562
1563        // Add first sequential pipeline
1564        add_sequential(&mut indexer, MockHandler).await;
1565
1566        // Verify next_sequential_checkpoint is set correctly (10 + 1 = 11)
1567        assert_eq!(
1568            indexer.next_sequential_checkpoint(),
1569            Some(11),
1570            "next_sequential_checkpoint should be 11"
1571        );
1572
1573        // Add second sequential pipeline
1574        add_sequential(&mut indexer, SequentialHandler).await;
1575
1576        // Should change to 6 (minimum of 6 and 11)
1577        assert_eq!(
1578            indexer.next_sequential_checkpoint(),
1579            Some(6),
1580            "next_sequential_checkpoint should still be 6"
1581        );
1582
1583        // Run indexer to verify it can make progress past the initial hi and finish ingesting.
1584        indexer.run().await.unwrap().join().await.unwrap();
1585
1586        // Verify each pipeline made some progress independently
1587        let watermark1 = conn.committer_watermark(MockHandler::NAME).await.unwrap();
1588        let watermark2 = conn
1589            .committer_watermark(SequentialHandler::NAME)
1590            .await
1591            .unwrap();
1592
1593        assert_eq!(watermark1.unwrap().checkpoint_hi_inclusive, 19);
1594        assert_eq!(watermark2.unwrap().checkpoint_hi_inclusive, 19);
1595    }
1596
1597    /// When a tasked indexer is initialized such that a tasked pipeline is run with a
1598    /// `first_checkpoint` less than the main pipeline's reader_lo, the indexer will correctly skip
1599    /// committing checkpoints less than the main pipeline's reader watermark.
1600    #[tokio::test]
1601    async fn test_tasked_pipelines_ignore_below_main_reader_lo() {
1602        let registry = Registry::new();
1603        let store = FallibleMockStore::default();
1604
1605        // Mock the store as if we have a main pipeline with a committer watermark at `10` and a
1606        // reader watermark at `7`.
1607        let mut conn = store.connect().await.unwrap();
1608        set_committer_watermark(&mut conn, MockCheckpointSequenceNumberHandler::NAME, 10).await;
1609        conn.set_reader_watermark(MockCheckpointSequenceNumberHandler::NAME, 7)
1610            .await
1611            .unwrap();
1612
1613        // Start a tasked indexer that will ingest from checkpoint 0. Checkpoints 0 through 6 should
1614        // be ignored by the tasked indexer.
1615        let indexer_args = IndexerArgs {
1616            first_checkpoint: Some(0),
1617            last_checkpoint: Some(15),
1618            task: TaskArgs::tasked("task".to_string(), 10),
1619            ..Default::default()
1620        };
1621        let (mut tasked_indexer, _temp_dir) =
1622            create_test_indexer(store.clone(), indexer_args, &registry, Some((16, 2))).await;
1623
1624        add_concurrent(&mut tasked_indexer, MockCheckpointSequenceNumberHandler).await;
1625
1626        let ingestion_metrics = tasked_indexer.ingestion_metrics().clone();
1627        let metrics = tasked_indexer.indexer_metrics().clone();
1628
1629        tasked_indexer.run().await.unwrap().join().await.unwrap();
1630
1631        assert_eq!(
1632            ingestion_metrics
1633                .total_ingested_checkpoints
1634                .with_label_values(&["0"])
1635                .get(),
1636            16
1637        );
1638        assert_eq!(
1639            metrics
1640                .total_collector_skipped_checkpoints
1641                .get_metric_with_label_values(&[MockCheckpointSequenceNumberHandler::NAME])
1642                .unwrap()
1643                .get(),
1644            7
1645        );
1646        let data = store
1647            .data
1648            .get(MockCheckpointSequenceNumberHandler::NAME)
1649            .unwrap();
1650        assert_eq!(data.len(), 9);
1651        for i in 0..7 {
1652            assert!(data.get(&i).is_none());
1653        }
1654        for i in 7..16 {
1655            assert!(data.get(&i).is_some());
1656        }
1657    }
1658
1659    /// A sequential pipeline that `--pipeline` excludes does not stop a tasked indexer from
1660    /// running its concurrent pipelines, but a tasked indexer still rejects a sequential pipeline
1661    /// that is enabled, whether by name or by an empty `--pipeline` list.
1662    #[tokio::test]
1663    async fn test_tasked_indexer_skips_excluded_sequential_pipeline() {
1664        let store = FallibleMockStore::default();
1665        let tasked = |pipeline: Vec<&str>| IndexerArgs {
1666            first_checkpoint: Some(0),
1667            last_checkpoint: Some(4),
1668            pipeline: pipeline.into_iter().map(String::from).collect(),
1669            task: TaskArgs::tasked("task".to_string(), 10),
1670        };
1671
1672        let (mut indexer, _temp_dir) = create_test_indexer(
1673            store.clone(),
1674            tasked(vec![MockCheckpointSequenceNumberHandler::NAME]),
1675            &Registry::new(),
1676            Some((5, 1)),
1677        )
1678        .await;
1679        add_sequential(&mut indexer, SequentialHandler).await;
1680        add_concurrent(&mut indexer, MockCheckpointSequenceNumberHandler).await;
1681        indexer.run().await.unwrap().join().await.unwrap();
1682
1683        let tasked_watermark = store.watermark("test@task").unwrap();
1684        assert_eq!(tasked_watermark.checkpoint_hi_inclusive, Some(4));
1685        assert!(
1686            store.watermark(SequentialHandler::NAME).is_none(),
1687            "the excluded sequential pipeline ran"
1688        );
1689
1690        for pipeline in [vec![SequentialHandler::NAME], vec![]] {
1691            let (mut indexer, _temp_dir) =
1692                create_test_indexer(store.clone(), tasked(pipeline), &Registry::new(), None).await;
1693            let err = indexer
1694                .sequential_pipeline(SequentialHandler, SequentialConfig::default())
1695                .await
1696                .unwrap_err();
1697            assert!(
1698                err.to_string()
1699                    .contains("Sequential pipelines do not support pipeline tasks"),
1700                "unexpected error: {err:#}"
1701            );
1702        }
1703    }
1704
1705    /// Tasked pipelines can run ahead of the main pipeline's committer watermark.
1706    #[tokio::test]
1707    async fn test_tasked_pipelines_surpass_main_pipeline_committer_hi() {
1708        let registry = Registry::new();
1709        let store = FallibleMockStore::default();
1710
1711        let mut conn = store.connect().await.unwrap();
1712        set_committer_watermark(&mut conn, "test", 10).await;
1713        conn.set_reader_watermark("test", 5).await.unwrap();
1714
1715        // Start a tasked indexer that will ingest from checkpoint 9 and go past the main pipeline's
1716        // watermarks.
1717        let indexer_args = IndexerArgs {
1718            first_checkpoint: Some(9),
1719            last_checkpoint: Some(25),
1720            task: TaskArgs::tasked("task".to_string(), 10),
1721            ..Default::default()
1722        };
1723        let (mut tasked_indexer, _temp_dir) =
1724            create_test_indexer(store.clone(), indexer_args, &registry, Some((26, 2))).await;
1725
1726        add_concurrent(&mut tasked_indexer, MockCheckpointSequenceNumberHandler).await;
1727
1728        let ingestion_metrics = tasked_indexer.ingestion_metrics().clone();
1729        let metrics = tasked_indexer.indexer_metrics().clone();
1730
1731        tasked_indexer.run().await.unwrap().join().await.unwrap();
1732
1733        assert_eq!(
1734            ingestion_metrics
1735                .total_ingested_checkpoints
1736                .with_label_values(&["0"])
1737                .get(),
1738            17
1739        );
1740        assert_out_of_order!(metrics, "test", 0);
1741        assert_eq!(
1742            metrics
1743                .total_collector_skipped_checkpoints
1744                .get_metric_with_label_values(&[MockCheckpointSequenceNumberHandler::NAME])
1745                .unwrap()
1746                .get(),
1747            0
1748        );
1749
1750        let data = store.data.get("test").unwrap();
1751        assert_eq!(data.len(), 17);
1752        for i in 0..9 {
1753            assert!(data.get(&i).is_none());
1754        }
1755        for i in 9..26 {
1756            assert!(data.get(&i).is_some());
1757        }
1758        let main_pipeline_watermark = store.watermark("test").unwrap();
1759        // assert that the main pipeline's watermarks are not updated
1760        assert_eq!(main_pipeline_watermark.checkpoint_hi_inclusive, Some(10));
1761        assert_eq!(main_pipeline_watermark.reader_lo, 5);
1762        let tasked_pipeline_watermark = store.watermark("test@task").unwrap();
1763        assert_eq!(tasked_pipeline_watermark.checkpoint_hi_inclusive, Some(25));
1764        assert_eq!(tasked_pipeline_watermark.reader_lo, 9);
1765    }
1766
1767    /// Test that when the collector observes `reader_lo = X`, that all checkpoints >= X will be
1768    /// committed, and any checkpoints inflight < X will be skipped.
1769    #[tokio::test]
1770    async fn test_tasked_pipelines_skip_checkpoints_trailing_main_reader_lo() {
1771        let registry = Registry::new();
1772        let store = FallibleMockStore::default();
1773        let mut conn = store.connect().await.unwrap();
1774        // Set the main pipeline watermark.
1775        set_committer_watermark(&mut conn, ControllableHandler::NAME, 11).await;
1776
1777        // Generate 500 checkpoints upfront, for the indexer to process all at once.
1778        let indexer_args = IndexerArgs {
1779            first_checkpoint: Some(0),
1780            last_checkpoint: Some(500),
1781            task: TaskArgs::tasked("task".to_string(), 10 /* reader_interval_ms */),
1782            ..Default::default()
1783        };
1784        let (mut tasked_indexer, _temp_dir) =
1785            create_test_indexer(store.clone(), indexer_args, &registry, Some((501, 2))).await;
1786        let mut allow_process = 10;
1787        // Limit the pipeline to process only checkpoints `[0, 10]`.
1788        let (controllable_handler, process_below) = ControllableHandler::with_limit(allow_process);
1789        let _ = tasked_indexer
1790            .concurrent_pipeline(
1791                controllable_handler,
1792                ConcurrentConfig {
1793                    committer: CommitterConfig {
1794                        watermark_interval_ms: 10,
1795                        ..Default::default()
1796                    },
1797                    // High fixed concurrency so all checkpoints can be processed
1798                    // concurrently despite out-of-order arrival.
1799                    fanout: Some(ConcurrencyConfig::Fixed { value: 501 }),
1800                    ..Default::default()
1801                },
1802            )
1803            .await;
1804        let metrics = tasked_indexer.indexer_metrics().clone();
1805
1806        let mut s_indexer = tasked_indexer.run().await.unwrap();
1807
1808        // Wait for pipeline to commit up to configured checkpoint 10 inclusive. With the main
1809        // pipeline `reader_lo` currently unset, all checkpoints are allowed and should be
1810        // committed.
1811        store
1812            .wait_for_watermark(
1813                &pipeline_task::<FallibleMockStore>(ControllableHandler::NAME, Some("task"))
1814                    .unwrap(),
1815                10,
1816                Duration::from_secs(10),
1817            )
1818            .await;
1819
1820        // Set the reader_lo to 250, simulating the main pipeline getting ahead. The
1821        // track_main_reader_lo task will eventually pick this up and update the atomic. The
1822        // collector reads from the atomic when it receives checkpoints, so we release checkpoints
1823        // one at a time until the collector_reader_lo metric shows the new value.
1824        conn.set_reader_watermark(ControllableHandler::NAME, 250)
1825            .await
1826            .unwrap();
1827
1828        let reader_lo = metrics
1829            .collector_reader_lo
1830            .with_label_values(&[ControllableHandler::NAME]);
1831
1832        // Send checkpoints one at a time at 10ms intervals. The tasked indexer has a reader refresh
1833        // interval of 10ms as well, so the collector should pick up the new reader_lo after a few
1834        // checkpoints have been processed.
1835        let mut interval = tokio::time::interval(Duration::from_millis(10));
1836        while reader_lo.get() != 250 {
1837            interval.tick().await;
1838            // allow_process is initialized to 11, bump to 11 for the next checkpoint
1839            allow_process += 1;
1840            assert!(
1841                allow_process <= 500,
1842                "Released all checkpoints but collector never observed new reader_lo"
1843            );
1844            process_below.send(allow_process).ok();
1845        }
1846
1847        // At this point, the collector has observed reader_lo = 250. Release all remaining
1848        // checkpoints. Guarantees:
1849        // - [0, 10]: committed (before reader_lo was set)
1850        // - [11, allow_process]: some committed, some skipped (timing-dependent during detection)
1851        // - (allow_process, 250): skipped (in-flight, filtered by collector)
1852        // - [250, 500]: committed (>= reader_lo)
1853        process_below.send(500).ok();
1854
1855        s_indexer.join().await.unwrap();
1856
1857        let data = store.data.get(ControllableHandler::NAME).unwrap();
1858
1859        // Checkpoints (allow_process, 250) must be skipped.
1860        for chkpt in (allow_process + 1)..250 {
1861            assert!(
1862                data.get(&chkpt).is_none(),
1863                "Checkpoint {chkpt} should have been skipped"
1864            );
1865        }
1866
1867        // Checkpoints >= reader_lo must be committed.
1868        for chkpt in 250..=500 {
1869            assert!(
1870                data.get(&chkpt).is_some(),
1871                "Checkpoint {chkpt} should have been committed (>= reader_lo)"
1872            );
1873        }
1874
1875        // Baseline: checkpoints [0, 10] were committed before reader_lo was set.
1876        for chkpt in 0..=10 {
1877            assert!(
1878                data.get(&chkpt).is_some(),
1879                "Checkpoint {chkpt} should have been committed (baseline)"
1880            );
1881        }
1882    }
1883
1884    /// Pipelines whose distances from the network tip are far apart are split into separate
1885    /// ingestion cohorts, each served by its own ingestion service, and the indexer completes
1886    /// even though the cohorts' ingestion services finish at different times.
1887    #[tokio::test]
1888    async fn test_multi_cohort_ingestion() {
1889        let registry = Registry::new();
1890        let store = FallibleMockStore::default();
1891
1892        test_pipeline!(A, "near_tip");
1893        test_pipeline!(B, "far_behind");
1894
1895        let temp_dir = multi_cohort_setup(&store, A::NAME, B::NAME).await;
1896
1897        let mut indexer = Indexer::new(
1898            store.clone(),
1899            IndexerArgs {
1900                last_checkpoint: Some(59),
1901                ..Default::default()
1902            },
1903            ClientArgs {
1904                ingestion: IngestionClientArgs {
1905                    local_ingestion_path: Some(temp_dir.path().to_owned()),
1906                    ..Default::default()
1907                },
1908                ..Default::default()
1909            },
1910            IngestionConfig::default(),
1911            None,
1912            &registry,
1913        )
1914        .await
1915        .unwrap();
1916
1917        add_concurrent(&mut indexer, A).await;
1918        add_concurrent(&mut indexer, B).await;
1919
1920        let ingestion_metrics = indexer.ingestion_metrics().clone();
1921        let indexer_metrics = indexer.indexer_metrics().clone();
1922        indexer.run().await.unwrap().join().await.unwrap();
1923
1924        // Both cohorts' services report to the shared metrics, each under its own cohort label.
1925        // Only the lagging cohort (1) had anything to ingest; the near-tip cohort (0)'s range
1926        // [100_101, 59] is empty.
1927        assert_eq!(
1928            ingestion_metrics
1929                .total_ingested_checkpoints
1930                .with_label_values(&["0"])
1931                .get(),
1932            0
1933        );
1934        assert_eq!(
1935            ingestion_metrics
1936                .total_ingested_checkpoints
1937                .with_label_values(&["1"])
1938                .get(),
1939            50
1940        );
1941
1942        // The near-tip pipeline received no checkpoints at all, proving it was not subscribed
1943        // to the lagging cohort's backfill (unsplit, it would have received all 50).
1944        let received = |name| {
1945            indexer_metrics
1946                .total_handler_checkpoints_received
1947                .get_metric_with_label_values(&[name])
1948                .unwrap()
1949                .get()
1950        };
1951        assert_eq!(received(A::NAME), 0);
1952        assert_eq!(received(B::NAME), 50);
1953
1954        // The lagging pipeline committed everything up to `last_checkpoint`, while the near-tip
1955        // pipeline had nothing to process.
1956        let watermark = store.watermark(B::NAME).unwrap();
1957        assert_eq!(watermark.checkpoint_hi_inclusive, Some(59));
1958        let watermark = store.watermark(A::NAME).unwrap();
1959        assert_eq!(watermark.checkpoint_hi_inclusive, Some(100_100));
1960    }
1961
1962    /// The client-driven constructor groups pipelines into cohorts too, with all cohorts
1963    /// sharing the supplied ingestion client and its metrics handle.
1964    #[tokio::test]
1965    async fn test_multi_cohort_ingestion_with_clients() {
1966        let registry = Registry::new();
1967        let store = FallibleMockStore::default();
1968
1969        test_pipeline!(A, "near_tip");
1970        test_pipeline!(B, "far_behind");
1971
1972        let temp_dir = multi_cohort_setup(&store, A::NAME, B::NAME).await;
1973
1974        let ingestion_metrics = IngestionMetrics::new(None, &registry);
1975        let ingestion_client = IngestionClient::new(
1976            IngestionClientArgs {
1977                local_ingestion_path: Some(temp_dir.path().to_owned()),
1978                ..Default::default()
1979            },
1980            ingestion_metrics.clone(),
1981        )
1982        .unwrap();
1983
1984        let mut indexer = Indexer::with_ingestion_clients(
1985            store.clone(),
1986            IndexerArgs {
1987                last_checkpoint: Some(59),
1988                ..Default::default()
1989            },
1990            ingestion_client,
1991            None,
1992            None,
1993            IngestionConfig::default(),
1994            None,
1995            &registry,
1996        )
1997        .await
1998        .unwrap();
1999
2000        add_concurrent(&mut indexer, A).await;
2001        add_concurrent(&mut indexer, B).await;
2002
2003        let indexer_metrics = indexer.indexer_metrics().clone();
2004        indexer.run().await.unwrap().join().await.unwrap();
2005
2006        // Both cohorts' services report to the one shared metrics handle, each under its own
2007        // cohort label, and only the lagging cohort (1) ingested checkpoints.
2008        assert_eq!(
2009            ingestion_metrics
2010                .total_ingested_checkpoints
2011                .with_label_values(&["0"])
2012                .get(),
2013            0
2014        );
2015        assert_eq!(
2016            ingestion_metrics
2017                .total_ingested_checkpoints
2018                .with_label_values(&["1"])
2019                .get(),
2020            50
2021        );
2022        assert_eq!(
2023            indexer_metrics
2024                .total_handler_checkpoints_received
2025                .get_metric_with_label_values(&[A::NAME])
2026                .unwrap()
2027                .get(),
2028            0
2029        );
2030
2031        let watermark = store.watermark(B::NAME).unwrap();
2032        assert_eq!(watermark.checkpoint_hi_inclusive, Some(59));
2033    }
2034
2035    /// Hard isolation: a pipeline at the tip runs to completion even while a far-behind pipeline in
2036    /// a separate cohort is wedged and backpressuring its own ingestion service. Before cohorts,
2037    /// both pipelines shared one ingestion service, so the far-behind pipeline's full subscriber
2038    /// channel would have throttled the shared broadcaster and held the tip pipeline back too.
2039    #[tokio::test]
2040    async fn test_near_tip_cohort_progresses_while_far_behind_stalls() {
2041        const TIP: u64 = 100;
2042
2043        let registry = Registry::new();
2044        let store = FallibleMockStore::default();
2045
2046        test_pipeline!(A, "near_tip");
2047
2048        // The far-behind pipeline's handler blocks every checkpoint in its range (all > 9), so its
2049        // cohort's ingestion service fills its subscriber channel and backpressures to a halt. Keep
2050        // the release sender alive for the whole test -- dropping it would unblock the handler.
2051        let (far_behind, _release) = ControllableHandler::with_limit(9);
2052
2053        let temp_dir = init_ingestion_dir(Some(TIP));
2054        let mut conn = store.connect().await.unwrap();
2055        set_committer_watermark(&mut conn, A::NAME, 94).await; // near tip: resumes at 95
2056        set_committer_watermark(&mut conn, ControllableHandler::NAME, 9).await; // far behind: resumes at 10
2057        synthetic_ingestion::generate_ingestion(synthetic_ingestion::Config {
2058            ingestion_dir: temp_dir.path().to_owned(),
2059            starting_checkpoint: 0,
2060            num_checkpoints: TIP + 1, // checkpoints [0, TIP]
2061            checkpoint_size: 1,
2062        })
2063        .await;
2064
2065        let mut indexer = Indexer::new(
2066            store.clone(),
2067            IndexerArgs {
2068                last_checkpoint: Some(TIP),
2069                ..Default::default()
2070            },
2071            ClientArgs {
2072                ingestion: IngestionClientArgs {
2073                    local_ingestion_path: Some(temp_dir.path().to_owned()),
2074                    ..Default::default()
2075                },
2076                ..Default::default()
2077            },
2078            // A small cohort boundary splits the near-tip and far-behind pipelines into separate
2079            // cohorts without having to generate tens of thousands of checkpoints.
2080            IngestionConfig {
2081                min_cohort_boundary: 10,
2082                ..Default::default()
2083            },
2084            None,
2085            &registry,
2086        )
2087        .await
2088        .unwrap();
2089
2090        // Near-tip pipeline: write watermarks promptly so it advances without waiting on the
2091        // default watermark interval.
2092        indexer
2093            .concurrent_pipeline(
2094                A,
2095                ConcurrentConfig {
2096                    committer: CommitterConfig {
2097                        watermark_interval_ms: 10,
2098                        ..Default::default()
2099                    },
2100                    ..Default::default()
2101                },
2102            )
2103            .await
2104            .unwrap();
2105        // Far-behind pipeline: its handler blocks, so it never commits regardless of config.
2106        add_concurrent(&mut indexer, far_behind).await;
2107
2108        let service = indexer.run().await.unwrap();
2109
2110        // The near-tip cohort ingests [95, 100] and commits to completion even though the
2111        // far-behind cohort is wedged. Were the two not isolated, the far-behind pipeline's
2112        // backpressure would stall this too and the wait would time out.
2113        store
2114            .wait_for_watermark(A::NAME, TIP, Duration::from_secs(10))
2115            .await;
2116
2117        // The far-behind pipeline made no progress: still at its initial watermark, nothing
2118        // committed -- it is stalled inside its own cohort, not holding the tip pipeline back.
2119        assert_eq!(
2120            store
2121                .watermark(ControllableHandler::NAME)
2122                .unwrap()
2123                .checkpoint_hi_inclusive,
2124            Some(9),
2125        );
2126        assert!(store.data.get(ControllableHandler::NAME).is_none());
2127
2128        // The merged service never completes on its own while a cohort is wedged; drop aborts it.
2129        drop(service);
2130    }
2131}