1use std::collections::BTreeSet;
5use std::ops::Bound;
6use std::sync::Arc;
7use std::time::Duration;
8
9use anyhow::Context;
10use anyhow::bail;
11use anyhow::ensure;
12use cohort::cohorts;
13use ingestion::ArcStreamingClient;
14use ingestion::ClientArgs;
15use ingestion::IngestionConfig;
16use ingestion::IngestionFactory;
17use ingestion::IngestionService;
18use ingestion::ingestion_client::IngestionClient;
19use metrics::IndexerMetrics;
20use prometheus::Registry;
21use sui_indexer_alt_framework_store_traits::ConcurrentStore;
22use sui_indexer_alt_framework_store_traits::Connection;
23use sui_indexer_alt_framework_store_traits::SequentialStore;
24use sui_indexer_alt_framework_store_traits::Store;
25use sui_indexer_alt_framework_store_traits::pipeline_task;
26use tracing::info;
27
28use crate::metrics::IngestionMetrics;
29use crate::pipeline::Processor;
30use crate::pipeline::concurrent::ConcurrentConfig;
31use crate::pipeline::concurrent::{self};
32use crate::pipeline::sequential::SequentialConfig;
33use crate::pipeline::sequential::{self};
34use crate::service::Service;
35
36pub use sui_field_count::FieldCount;
37pub use sui_futures::service;
38pub use sui_indexer_alt_framework_store_traits as store;
40pub use sui_types as types;
41
42#[cfg(feature = "cluster")]
43pub mod cluster;
44mod cohort;
45pub mod config;
46pub mod ingestion;
47pub mod metrics;
48pub mod pipeline;
49#[cfg(feature = "postgres")]
50pub mod postgres;
51
52#[cfg(test)]
53pub mod mocks;
54
55#[derive(clap::Args, Default, Debug, Clone)]
57pub struct IndexerArgs {
58 #[arg(long)]
65 pub first_checkpoint: Option<u64>,
66
67 #[arg(long)]
70 pub last_checkpoint: Option<u64>,
71
72 #[arg(long, action = clap::ArgAction::Append)]
75 pub pipeline: Vec<String>,
76
77 #[clap(flatten)]
79 pub task: TaskArgs,
80}
81
82#[derive(clap::Parser, Default, Debug, Clone)]
84pub struct TaskArgs {
85 #[arg(long, requires = "reader_interval_ms")]
98 task: Option<String>,
99
100 #[arg(long, requires = "task")]
111 reader_interval_ms: Option<u64>,
112}
113
114pub struct Indexer<S: Store> {
115 store: S,
119
120 metrics: Arc<IndexerMetrics>,
122
123 ingestion_factory: IngestionFactory,
126
127 first_checkpoint: Option<u64>,
130
131 last_checkpoint: Option<u64>,
134
135 latest_checkpoint: u64,
137
138 next_checkpoint: u64,
141
142 next_sequential_checkpoint: Option<u64>,
145
146 task: Option<Task>,
159
160 enabled_pipelines: Option<BTreeSet<String>>,
164
165 added_pipelines: BTreeSet<&'static str>,
168
169 pipelines: Vec<PendingPipeline>,
172}
173
174#[derive(Clone)]
176pub(crate) struct Task {
177 task: String,
180 reader_interval: Duration,
183}
184
185struct PendingPipeline {
189 name: &'static str,
191
192 next_checkpoint: u64,
194
195 build: PipelineBuilder,
198}
199
200type PipelineBuilder = Box<dyn FnOnce(&mut IngestionService) -> Service + Send + Sync>;
208
209impl TaskArgs {
210 pub fn tasked(task: String, reader_interval_ms: u64) -> Self {
211 Self {
212 task: Some(task),
213 reader_interval_ms: Some(reader_interval_ms),
214 }
215 }
216
217 fn into_task(self) -> Option<Task> {
218 Some(Task {
219 task: self.task?,
220 reader_interval: Duration::from_millis(self.reader_interval_ms?),
221 })
222 }
223}
224
225impl<S: Store> Indexer<S> {
226 pub async fn new(
238 store: S,
239 indexer_args: IndexerArgs,
240 client_args: ClientArgs,
241 ingestion_config: IngestionConfig,
242 metrics_prefix: Option<&str>,
243 registry: &Registry,
244 ) -> anyhow::Result<Self> {
245 let ingestion_factory =
246 IngestionFactory::new(client_args, ingestion_config, metrics_prefix, registry).await?;
247 Self::with_ingestion_factory(
248 store,
249 indexer_args,
250 ingestion_factory,
251 metrics_prefix,
252 registry,
253 )
254 .await
255 }
256
257 pub async fn with_ingestion_clients(
271 store: S,
272 indexer_args: IndexerArgs,
273 ingestion_client: IngestionClient,
274 streaming_client: Option<ArcStreamingClient>,
275 gap_fill_client: Option<IngestionClient>,
276 ingestion_config: IngestionConfig,
277 metrics_prefix: Option<&str>,
278 registry: &Registry,
279 ) -> anyhow::Result<Self> {
280 let ingestion_factory = IngestionFactory::with_clients(
281 ingestion_client,
282 streaming_client,
283 gap_fill_client,
284 ingestion_config,
285 );
286 Self::with_ingestion_factory(
287 store,
288 indexer_args,
289 ingestion_factory,
290 metrics_prefix,
291 registry,
292 )
293 .await
294 }
295
296 async fn with_ingestion_factory(
299 store: S,
300 indexer_args: IndexerArgs,
301 ingestion_factory: IngestionFactory,
302 metrics_prefix: Option<&str>,
303 registry: &Registry,
304 ) -> anyhow::Result<Self> {
305 let IndexerArgs {
306 first_checkpoint,
307 last_checkpoint,
308 pipeline,
309 task,
310 } = indexer_args;
311
312 let metrics = IndexerMetrics::new(metrics_prefix, registry);
313
314 let latest_checkpoint = ingestion_factory.latest_checkpoint_number().await?;
315
316 info!(latest_checkpoint);
317
318 Ok(Self {
319 store,
320 metrics,
321 ingestion_factory,
322 first_checkpoint,
323 last_checkpoint,
324 latest_checkpoint,
325 next_checkpoint: u64::MAX,
326 next_sequential_checkpoint: None,
327 task: task.into_task(),
328 enabled_pipelines: if pipeline.is_empty() {
329 None
330 } else {
331 Some(pipeline.into_iter().collect())
332 },
333 added_pipelines: BTreeSet::new(),
334 pipelines: vec![],
335 })
336 }
337
338 pub fn store(&self) -> &S {
340 &self.store
341 }
342
343 pub fn ingestion_client(&self) -> &IngestionClient {
345 self.ingestion_factory.ingestion_client()
346 }
347
348 pub fn indexer_metrics(&self) -> &Arc<IndexerMetrics> {
350 &self.metrics
351 }
352
353 pub fn ingestion_metrics(&self) -> &Arc<IngestionMetrics> {
355 self.ingestion_factory.metrics()
356 }
357
358 pub fn pipelines(&self) -> impl Iterator<Item = &'static str> + '_ {
361 self.pipelines.iter().map(|p| p.name)
362 }
363
364 pub fn next_sequential_checkpoint(&self) -> Option<u64> {
368 self.next_sequential_checkpoint
369 }
370
371 pub async fn run(self) -> anyhow::Result<Service> {
381 let Self {
382 ingestion_factory,
383 last_checkpoint,
384 latest_checkpoint,
385 enabled_pipelines,
386 pipelines,
387 ..
388 } = self;
389
390 if let Some(enabled_pipelines) = enabled_pipelines {
391 ensure!(
392 enabled_pipelines.is_empty(),
393 "Tried to enable pipelines that this indexer does not know about: \
394 {enabled_pipelines:#?}",
395 );
396 }
397
398 ensure!(!pipelines.is_empty(), "No pipelines registered to run");
399
400 let groups = cohorts(
401 pipelines,
402 latest_checkpoint,
403 ingestion_factory.config().min_cohort_boundary,
404 );
405
406 info!(
407 cohorts = groups.len(),
408 latest_checkpoint, "Grouped pipelines into ingestion cohorts"
409 );
410
411 let end = last_checkpoint.map_or(Bound::Unbounded, Bound::Included);
412
413 let mut service = Service::new();
414 for (cohort, group) in groups.into_iter().enumerate() {
415 let mut ingestion = ingestion_factory.create(cohort);
416
417 let mut start = u64::MAX;
418 let mut names = Vec::with_capacity(group.len());
419 let mut members = Vec::with_capacity(group.len());
420 for pending in group {
421 start = start.min(pending.next_checkpoint);
422 names.push(pending.name);
423 members.push((pending.build)(&mut ingestion));
424 }
425
426 info!(cohort, start, end = last_checkpoint, pipelines = ?names, "Ingestion range");
427
428 let mut cohort_service = ingestion
429 .run((Bound::Included(start), end))
430 .await
431 .context("Failed to start ingestion service")?;
432
433 for member in members {
434 cohort_service = cohort_service.merge(member);
435 }
436
437 service = service.merge(cohort_service);
438 }
439
440 Ok(service)
441 }
442
443 async fn add_pipeline<P: Processor + 'static>(
452 &mut self,
453 pipeline_task: String,
454 retention: Option<u64>,
455 ) -> anyhow::Result<Option<u64>> {
456 ensure!(
457 self.added_pipelines.insert(P::NAME),
458 "Pipeline {:?} already added",
459 P::NAME,
460 );
461
462 if let Some(enabled_pipelines) = &mut self.enabled_pipelines
463 && !enabled_pipelines.remove(P::NAME)
464 {
465 info!(pipeline = P::NAME, "Skipping");
466 return Ok(None);
467 }
468
469 let proposed_next_checkpoint = if let Some(first_checkpoint) = self.first_checkpoint {
472 first_checkpoint
473 } else if let Some(retention) = retention {
474 self.latest_checkpoint.saturating_sub(retention)
475 } else {
476 0
477 };
478 let mut conn = self.store.connect().await?;
479 let init_watermark = conn
480 .init_watermark(&pipeline_task, proposed_next_checkpoint.checked_sub(1))
481 .await
482 .with_context(|| format!("Failed to init watermark for {pipeline_task}"))?;
483
484 let next_checkpoint = if let Some(init_watermark) = init_watermark {
485 if let Some(checkpoint_hi_inclusive) = init_watermark.checkpoint_hi_inclusive {
486 checkpoint_hi_inclusive + 1
487 } else {
488 0
489 }
490 } else {
491 proposed_next_checkpoint
492 };
493
494 self.next_checkpoint = next_checkpoint.min(self.next_checkpoint);
495
496 Ok(Some(next_checkpoint))
497 }
498}
499
500impl<S: ConcurrentStore> Indexer<S> {
501 pub async fn concurrent_pipeline<H: concurrent::Handler<Store = S>>(
508 &mut self,
509 handler: H,
510 config: ConcurrentConfig,
511 ) -> anyhow::Result<()> {
512 let pipeline_task =
513 pipeline_task::<S>(H::NAME, self.task.as_ref().map(|t| t.task.as_str()))?;
514 let retention = config.pruner.as_ref().map(|p| p.retention);
515 let Some(next_checkpoint) = self.add_pipeline::<H>(pipeline_task, retention).await? else {
516 return Ok(());
517 };
518
519 let store = self.store.clone();
520 let task = self.task.clone();
521 let metrics = self.metrics.clone();
522 self.pipelines.push(PendingPipeline {
523 name: H::NAME,
524 next_checkpoint,
525 build: Box::new(move |ingestion| {
526 let checkpoint_rx =
527 ingestion.subscribe_bounded(config.ingestion.subscriber_channel_size());
528 concurrent::pipeline::<H>(
529 handler,
530 next_checkpoint,
531 config,
532 store,
533 task,
534 checkpoint_rx,
535 metrics,
536 )
537 }),
538 });
539
540 Ok(())
541 }
542}
543
544impl<T: SequentialStore> Indexer<T> {
545 pub async fn sequential_pipeline<H: sequential::Handler<Store = T>>(
553 &mut self,
554 handler: H,
555 config: SequentialConfig,
556 ) -> anyhow::Result<()> {
557 let enabled = self
560 .enabled_pipelines
561 .as_ref()
562 .is_none_or(|e| e.contains(H::NAME));
563 if self.task.is_some() && enabled {
564 bail!(
565 "Sequential pipelines do not support pipeline tasks. \
566 These pipelines guarantee that each checkpoint is committed exactly once and in order. \
567 Running the same pipeline under a different task would violate these guarantees."
568 );
569 }
570
571 let Some(next_checkpoint) = self.add_pipeline::<H>(H::NAME.to_owned(), None).await? else {
572 return Ok(());
573 };
574
575 self.next_sequential_checkpoint = Some(
577 self.next_sequential_checkpoint
578 .map_or(next_checkpoint, |n| n.min(next_checkpoint)),
579 );
580
581 let store = self.store.clone();
582 let metrics = self.metrics.clone();
583 self.pipelines.push(PendingPipeline {
584 name: H::NAME,
585 next_checkpoint,
586 build: Box::new(move |ingestion| {
587 let checkpoint_rx =
588 ingestion.subscribe_bounded(config.ingestion.subscriber_channel_size());
589 sequential::pipeline::<H>(
590 handler,
591 next_checkpoint,
592 config,
593 store,
594 checkpoint_rx,
595 metrics,
596 )
597 }),
598 });
599
600 Ok(())
601 }
602}
603
604#[cfg(test)]
605mod tests {
606 use std::sync::Arc;
607
608 use async_trait::async_trait;
609 use clap::Parser;
610 use sui_indexer_alt_framework_store_traits::PrunerWatermark;
611 use sui_synthetic_ingestion::synthetic_ingestion;
612 use tokio::sync::watch;
613
614 use crate::FieldCount;
615 use crate::config::ConcurrencyConfig;
616 use crate::ingestion::ingestion_client::IngestionClientArgs;
617 use crate::ingestion::store_client::ObjectStoreWatermark;
618 use crate::ingestion::store_client::WATERMARK_PATH;
619 use crate::mocks::store::FallibleMockStore;
620 use crate::pipeline::CommitterConfig;
621 use crate::pipeline::Processor;
622 use crate::pipeline::concurrent::ConcurrentConfig;
623 use crate::store::CommitterWatermark;
624 use crate::store::ConcurrentConnection as _;
625
626 use super::*;
627
628 #[allow(dead_code)]
629 #[derive(Clone, FieldCount)]
630 struct MockValue(u64);
631
632 struct ControllableHandler {
634 process_below: watch::Receiver<u64>,
636 }
637
638 impl ControllableHandler {
639 fn with_limit(limit: u64) -> (Self, watch::Sender<u64>) {
640 let (tx, rx) = watch::channel(limit);
641 (Self { process_below: rx }, tx)
642 }
643 }
644
645 #[async_trait]
646 impl Processor for ControllableHandler {
647 const NAME: &'static str = "controllable";
648 type Value = MockValue;
649
650 async fn process(
651 &self,
652 checkpoint: &Arc<sui_types::full_checkpoint_content::Checkpoint>,
653 ) -> anyhow::Result<Vec<Self::Value>> {
654 let cp_num = checkpoint.summary.sequence_number;
655
656 self.process_below
658 .clone()
659 .wait_for(|&limit| cp_num <= limit)
660 .await
661 .ok();
662
663 Ok(vec![MockValue(cp_num)])
664 }
665 }
666
667 #[async_trait]
668 impl concurrent::Handler for ControllableHandler {
669 type Store = FallibleMockStore;
670 type Batch = Vec<MockValue>;
671
672 fn batch(
673 &self,
674 batch: &mut Self::Batch,
675 values: &mut std::vec::IntoIter<Self::Value>,
676 ) -> concurrent::BatchStatus {
677 batch.extend(values);
678 concurrent::BatchStatus::Ready
679 }
680
681 async fn commit<'a>(
682 &self,
683 batch: &Self::Batch,
684 conn: &mut <Self::Store as Store>::Connection<'a>,
685 ) -> anyhow::Result<usize> {
686 for value in batch {
687 conn.0
688 .commit_data(Self::NAME, value.0, vec![value.0])
689 .await?;
690 }
691 Ok(batch.len())
692 }
693 }
694
695 macro_rules! test_pipeline {
696 ($handler:ident, $name:literal) => {
697 struct $handler;
698
699 #[async_trait]
700 impl Processor for $handler {
701 const NAME: &'static str = $name;
702 type Value = MockValue;
703 async fn process(
704 &self,
705 checkpoint: &Arc<sui_types::full_checkpoint_content::Checkpoint>,
706 ) -> anyhow::Result<Vec<Self::Value>> {
707 Ok(vec![MockValue(checkpoint.summary.sequence_number)])
708 }
709 }
710
711 #[async_trait]
712 impl crate::pipeline::concurrent::Handler for $handler {
713 type Store = FallibleMockStore;
714 type Batch = Vec<Self::Value>;
715
716 fn batch(
717 &self,
718 batch: &mut Self::Batch,
719 values: &mut std::vec::IntoIter<Self::Value>,
720 ) -> crate::pipeline::concurrent::BatchStatus {
721 batch.extend(values);
722 crate::pipeline::concurrent::BatchStatus::Pending
723 }
724
725 async fn commit<'a>(
726 &self,
727 batch: &Self::Batch,
728 conn: &mut <Self::Store as Store>::Connection<'a>,
729 ) -> anyhow::Result<usize> {
730 for value in batch {
731 conn.0
732 .commit_data(Self::NAME, value.0, vec![value.0])
733 .await?;
734 }
735 Ok(batch.len())
736 }
737 }
738
739 #[async_trait]
740 impl crate::pipeline::sequential::Handler for $handler {
741 type Store = FallibleMockStore;
742 type Batch = Vec<Self::Value>;
743
744 fn batch(&self, batch: &mut Self::Batch, values: std::vec::IntoIter<Self::Value>) {
745 batch.extend(values);
746 }
747
748 async fn commit<'a>(
749 &self,
750 _batch: &Self::Batch,
751 _conn: &mut <Self::Store as Store>::Connection<'a>,
752 ) -> anyhow::Result<usize> {
753 Ok(1)
754 }
755 }
756 };
757 }
758
759 test_pipeline!(MockHandler, "test_processor");
760 test_pipeline!(SequentialHandler, "sequential_handler");
761 test_pipeline!(MockCheckpointSequenceNumberHandler, "test");
762
763 fn init_ingestion_dir(latest_checkpoint: Option<u64>) -> tempfile::TempDir {
764 let dir = tempfile::tempdir().unwrap();
765 if let Some(cp) = latest_checkpoint {
766 let watermark_path = dir.path().join(WATERMARK_PATH);
767 std::fs::create_dir_all(watermark_path.parent().unwrap()).unwrap();
768 let watermark = ObjectStoreWatermark {
769 checkpoint_hi_inclusive: Some(cp),
770 };
771 std::fs::write(watermark_path, serde_json::to_string(&watermark).unwrap()).unwrap();
772 }
773 dir
774 }
775
776 async fn create_test_indexer(
779 store: FallibleMockStore,
780 indexer_args: IndexerArgs,
781 registry: &Registry,
782 ingestion_data: Option<(u64, u64)>,
783 ) -> (Indexer<FallibleMockStore>, tempfile::TempDir) {
784 let temp_dir = init_ingestion_dir(None);
785 if let Some((num_checkpoints, checkpoint_size)) = ingestion_data {
786 synthetic_ingestion::generate_ingestion(synthetic_ingestion::Config {
787 ingestion_dir: temp_dir.path().to_owned(),
788 starting_checkpoint: 0,
789 num_checkpoints,
790 checkpoint_size,
791 })
792 .await;
793 }
794 let client_args = ClientArgs {
795 ingestion: IngestionClientArgs {
796 local_ingestion_path: Some(temp_dir.path().to_owned()),
797 ..Default::default()
798 },
799 ..Default::default()
800 };
801 let indexer = Indexer::new(
802 store,
803 indexer_args,
804 client_args,
805 IngestionConfig::default(),
806 None,
807 registry,
808 )
809 .await
810 .unwrap();
811 (indexer, temp_dir)
812 }
813
814 async fn set_committer_watermark(
815 conn: &mut <FallibleMockStore as Store>::Connection<'_>,
816 name: &str,
817 hi: u64,
818 ) {
819 conn.set_committer_watermark(
820 name,
821 CommitterWatermark {
822 checkpoint_hi_inclusive: hi,
823 ..Default::default()
824 },
825 )
826 .await
827 .unwrap();
828 }
829
830 async fn add_concurrent<H: concurrent::Handler<Store = FallibleMockStore>>(
831 indexer: &mut Indexer<FallibleMockStore>,
832 handler: H,
833 ) {
834 indexer
835 .concurrent_pipeline(handler, ConcurrentConfig::default())
836 .await
837 .unwrap();
838 }
839
840 async fn add_sequential<H: sequential::Handler<Store = FallibleMockStore>>(
841 indexer: &mut Indexer<FallibleMockStore>,
842 handler: H,
843 ) {
844 indexer
845 .sequential_pipeline(handler, SequentialConfig::default())
846 .await
847 .unwrap();
848 }
849
850 async fn multi_cohort_setup(
857 store: &FallibleMockStore,
858 near: &str,
859 far: &str,
860 ) -> tempfile::TempDir {
861 let mut conn = store.connect().await.unwrap();
862 set_committer_watermark(&mut conn, near, 100_100).await;
863 set_committer_watermark(&mut conn, far, 9).await;
864
865 let temp_dir = init_ingestion_dir(Some(100_000));
866 synthetic_ingestion::generate_ingestion(synthetic_ingestion::Config {
867 ingestion_dir: temp_dir.path().to_owned(),
868 starting_checkpoint: 0,
869 num_checkpoints: 60,
870 checkpoint_size: 1,
871 })
872 .await;
873
874 temp_dir
875 }
876
877 macro_rules! assert_out_of_order {
878 ($metrics:expr, $pipeline:expr, $expected:expr) => {
879 assert_eq!(
880 $metrics
881 .total_watermarks_out_of_order
882 .get_metric_with_label_values(&[$pipeline])
883 .unwrap()
884 .get(),
885 $expected,
886 );
887 };
888 }
889
890 async fn test_init_watermark(
891 first_checkpoint: Option<u64>,
892 is_concurrent: bool,
893 ) -> (Option<CommitterWatermark>, Option<PrunerWatermark>) {
894 let registry = Registry::new();
895 let store = FallibleMockStore::default();
896
897 test_pipeline!(A, "pipeline_name");
898
899 let mut conn = store.connect().await.unwrap();
900
901 let indexer_args = IndexerArgs {
902 first_checkpoint,
903 ..IndexerArgs::default()
904 };
905 let (mut indexer, _temp_dir) =
906 create_test_indexer(store.clone(), indexer_args, ®istry, None).await;
907
908 if is_concurrent {
909 add_concurrent(&mut indexer, A).await;
910 } else {
911 add_sequential(&mut indexer, A).await;
912 }
913
914 (
915 conn.committer_watermark(A::NAME).await.unwrap(),
916 conn.pruner_watermark(A::NAME, Duration::ZERO)
917 .await
918 .unwrap(),
919 )
920 }
921
922 const LATEST_CHECKPOINT: u64 = 10;
923
924 async fn test_next_checkpoint(
929 watermark: Option<u64>,
930 first_checkpoint: Option<u64>,
931 concurrent_config: ConcurrentConfig,
932 ) -> Indexer<FallibleMockStore> {
933 let registry = Registry::new();
934 let store = FallibleMockStore::default();
935
936 test_pipeline!(A, "concurrent_a");
937
938 if let Some(checkpoint_hi_inclusive) = watermark {
939 let mut conn = store.connect().await.unwrap();
940 conn.set_committer_watermark(
941 A::NAME,
942 CommitterWatermark {
943 checkpoint_hi_inclusive,
944 ..Default::default()
945 },
946 )
947 .await
948 .unwrap();
949 }
950
951 let temp_dir = init_ingestion_dir(Some(LATEST_CHECKPOINT));
952 let mut indexer = Indexer::new(
953 store,
954 IndexerArgs {
955 first_checkpoint,
956 ..Default::default()
957 },
958 ClientArgs {
959 ingestion: IngestionClientArgs {
960 local_ingestion_path: Some(temp_dir.path().to_owned()),
961 ..Default::default()
962 },
963 ..Default::default()
964 },
965 IngestionConfig::default(),
966 None,
967 ®istry,
968 )
969 .await
970 .unwrap();
971
972 assert_eq!(indexer.latest_checkpoint, LATEST_CHECKPOINT);
973
974 indexer
975 .concurrent_pipeline::<A>(A, concurrent_config)
976 .await
977 .unwrap();
978
979 indexer
980 }
981
982 fn pruner_config(retention: u64) -> ConcurrentConfig {
983 ConcurrentConfig {
984 pruner: Some(concurrent::PrunerConfig {
985 retention,
986 ..Default::default()
987 }),
988 ..Default::default()
989 }
990 }
991
992 #[test]
993 fn test_arg_parsing() {
994 #[derive(Parser)]
995 struct Args {
996 #[clap(flatten)]
997 indexer: IndexerArgs,
998 }
999
1000 let args = Args::try_parse_from([
1001 "cmd",
1002 "--first-checkpoint",
1003 "10",
1004 "--last-checkpoint",
1005 "100",
1006 "--pipeline",
1007 "a",
1008 "--pipeline",
1009 "b",
1010 "--task",
1011 "t",
1012 "--reader-interval-ms",
1013 "5000",
1014 ])
1015 .unwrap();
1016
1017 assert_eq!(args.indexer.first_checkpoint, Some(10));
1018 assert_eq!(args.indexer.last_checkpoint, Some(100));
1019 assert_eq!(args.indexer.pipeline, vec!["a", "b"]);
1020 assert_eq!(args.indexer.task.task, Some("t".to_owned()));
1021 assert_eq!(args.indexer.task.reader_interval_ms, Some(5000));
1022 }
1023
1024 #[tokio::test]
1026 async fn test_next_checkpoint_all_pipelines_have_watermarks() {
1027 let registry = Registry::new();
1028 let store = FallibleMockStore::default();
1029
1030 test_pipeline!(A, "concurrent_a");
1031 test_pipeline!(B, "concurrent_b");
1032 test_pipeline!(C, "sequential_c");
1033 test_pipeline!(D, "sequential_d");
1034
1035 let mut conn = store.connect().await.unwrap();
1036
1037 conn.init_watermark(A::NAME, Some(0)).await.unwrap();
1038 set_committer_watermark(&mut conn, A::NAME, 100).await;
1039
1040 conn.init_watermark(B::NAME, Some(0)).await.unwrap();
1041 set_committer_watermark(&mut conn, B::NAME, 10).await;
1042
1043 conn.init_watermark(C::NAME, Some(0)).await.unwrap();
1044 set_committer_watermark(&mut conn, C::NAME, 1).await;
1045
1046 conn.init_watermark(D::NAME, Some(0)).await.unwrap();
1047 set_committer_watermark(&mut conn, D::NAME, 50).await;
1048
1049 let (mut indexer, _temp_dir) =
1050 create_test_indexer(store, IndexerArgs::default(), ®istry, None).await;
1051
1052 add_concurrent(&mut indexer, A).await;
1053 add_concurrent(&mut indexer, B).await;
1054 add_sequential(&mut indexer, C).await;
1055 add_sequential(&mut indexer, D).await;
1056
1057 assert_eq!(indexer.first_checkpoint, None);
1058 assert_eq!(indexer.last_checkpoint, None);
1059 assert_eq!(indexer.latest_checkpoint, 0);
1060 assert_eq!(indexer.next_checkpoint, 2);
1061 assert_eq!(indexer.next_sequential_checkpoint, Some(2));
1062 }
1063
1064 #[tokio::test]
1066 async fn test_next_checkpoint_not_all_pipelines_have_watermarks() {
1067 let registry = Registry::new();
1068 let store = FallibleMockStore::default();
1069
1070 test_pipeline!(A, "concurrent_a");
1071 test_pipeline!(B, "concurrent_b");
1072 test_pipeline!(C, "sequential_c");
1073 test_pipeline!(D, "sequential_d");
1074
1075 let mut conn = store.connect().await.unwrap();
1076 set_committer_watermark(&mut conn, B::NAME, 10).await;
1077 set_committer_watermark(&mut conn, C::NAME, 1).await;
1078
1079 let (mut indexer, _temp_dir) =
1080 create_test_indexer(store, IndexerArgs::default(), ®istry, None).await;
1081
1082 add_concurrent(&mut indexer, A).await;
1083 add_concurrent(&mut indexer, B).await;
1084 add_sequential(&mut indexer, C).await;
1085 add_sequential(&mut indexer, D).await;
1086
1087 assert_eq!(indexer.first_checkpoint, None);
1088 assert_eq!(indexer.last_checkpoint, None);
1089 assert_eq!(indexer.latest_checkpoint, 0);
1090 assert_eq!(indexer.next_checkpoint, 0);
1091 assert_eq!(indexer.next_sequential_checkpoint, Some(0));
1092 }
1093
1094 #[tokio::test]
1097 async fn test_pipelines_reports_enabled_pipelines() {
1098 test_pipeline!(A, "concurrent_a");
1099 test_pipeline!(B, "concurrent_b");
1100 test_pipeline!(C, "sequential_c");
1101 test_pipeline!(D, "sequential_d");
1102
1103 let cases = [
1104 (vec![], vec![A::NAME, B::NAME, C::NAME, D::NAME]),
1105 (vec![A::NAME, C::NAME], vec![A::NAME, C::NAME]),
1106 ];
1107
1108 for (filter, expected) in cases {
1109 let args = IndexerArgs {
1110 pipeline: filter.into_iter().map(String::from).collect(),
1111 ..Default::default()
1112 };
1113 let (mut indexer, _temp_dir) =
1114 create_test_indexer(FallibleMockStore::default(), args, &Registry::new(), None)
1115 .await;
1116
1117 add_concurrent(&mut indexer, A).await;
1118 add_concurrent(&mut indexer, B).await;
1119 add_sequential(&mut indexer, C).await;
1120 add_sequential(&mut indexer, D).await;
1121
1122 let pipelines: BTreeSet<_> = indexer.pipelines().collect();
1123 assert_eq!(pipelines, BTreeSet::from_iter(expected));
1124 }
1125 }
1126
1127 #[tokio::test]
1129 async fn test_next_checkpoint_smallest_is_0() {
1130 let registry = Registry::new();
1131 let store = FallibleMockStore::default();
1132
1133 test_pipeline!(A, "concurrent_a");
1134 test_pipeline!(B, "concurrent_b");
1135 test_pipeline!(C, "sequential_c");
1136 test_pipeline!(D, "sequential_d");
1137
1138 let mut conn = store.connect().await.unwrap();
1139 set_committer_watermark(&mut conn, A::NAME, 100).await;
1140 set_committer_watermark(&mut conn, B::NAME, 10).await;
1141 set_committer_watermark(&mut conn, C::NAME, 1).await;
1142 set_committer_watermark(&mut conn, D::NAME, 0).await;
1143
1144 let (mut indexer, _temp_dir) =
1145 create_test_indexer(store, IndexerArgs::default(), ®istry, None).await;
1146
1147 add_concurrent(&mut indexer, A).await;
1148 add_concurrent(&mut indexer, B).await;
1149 add_sequential(&mut indexer, C).await;
1150 add_sequential(&mut indexer, D).await;
1151
1152 assert_eq!(indexer.next_checkpoint, 1);
1153 }
1154
1155 #[tokio::test]
1158 async fn test_next_checkpoint_first_checkpoint_and_no_watermark() {
1159 let registry = Registry::new();
1160 let store = FallibleMockStore::default();
1161
1162 test_pipeline!(A, "concurrent_a");
1163 test_pipeline!(B, "concurrent_b");
1164 test_pipeline!(C, "sequential_c");
1165 test_pipeline!(D, "sequential_d");
1166
1167 let mut conn = store.connect().await.unwrap();
1168 set_committer_watermark(&mut conn, B::NAME, 50).await;
1169 set_committer_watermark(&mut conn, C::NAME, 10).await;
1170
1171 let indexer_args = IndexerArgs {
1172 first_checkpoint: Some(5),
1173 ..Default::default()
1174 };
1175 let (mut indexer, _temp_dir) =
1176 create_test_indexer(store, indexer_args, ®istry, None).await;
1177
1178 add_concurrent(&mut indexer, A).await;
1179 add_concurrent(&mut indexer, B).await;
1180 add_sequential(&mut indexer, C).await;
1181 add_sequential(&mut indexer, D).await;
1182
1183 assert_eq!(indexer.first_checkpoint, Some(5));
1184 assert_eq!(indexer.last_checkpoint, None);
1185 assert_eq!(indexer.latest_checkpoint, 0);
1186 assert_eq!(indexer.next_checkpoint, 5);
1187 assert_eq!(indexer.next_sequential_checkpoint, Some(5));
1188 }
1189
1190 #[tokio::test]
1193 async fn test_next_checkpoint_ignore_first_checkpoint() {
1194 let registry = Registry::new();
1195 let store = FallibleMockStore::default();
1196
1197 test_pipeline!(B, "concurrent_b");
1198 test_pipeline!(C, "sequential_c");
1199
1200 let mut conn = store.connect().await.unwrap();
1201 set_committer_watermark(&mut conn, B::NAME, 50).await;
1202 set_committer_watermark(&mut conn, C::NAME, 10).await;
1203
1204 let indexer_args = IndexerArgs {
1205 first_checkpoint: Some(5),
1206 ..Default::default()
1207 };
1208 let (mut indexer, _temp_dir) =
1209 create_test_indexer(store, indexer_args, ®istry, None).await;
1210
1211 add_concurrent(&mut indexer, B).await;
1212 add_sequential(&mut indexer, C).await;
1213
1214 assert_eq!(indexer.first_checkpoint, Some(5));
1215 assert_eq!(indexer.last_checkpoint, None);
1216 assert_eq!(indexer.latest_checkpoint, 0);
1217 assert_eq!(indexer.next_checkpoint, 11);
1218 assert_eq!(indexer.next_sequential_checkpoint, Some(11));
1219 }
1220
1221 #[tokio::test]
1225 async fn test_next_checkpoint_large_first_checkpoint() {
1226 let registry = Registry::new();
1227 let store = FallibleMockStore::default();
1228
1229 test_pipeline!(A, "concurrent_a");
1230 test_pipeline!(B, "concurrent_b");
1231 test_pipeline!(C, "sequential_c");
1232
1233 let mut conn = store.connect().await.unwrap();
1234 set_committer_watermark(&mut conn, B::NAME, 50).await;
1235 set_committer_watermark(&mut conn, C::NAME, 10).await;
1236
1237 let indexer_args = IndexerArgs {
1238 first_checkpoint: Some(24),
1239 ..Default::default()
1240 };
1241 let (mut indexer, _temp_dir) =
1242 create_test_indexer(store, indexer_args, ®istry, None).await;
1243
1244 add_concurrent(&mut indexer, A).await;
1245 add_concurrent(&mut indexer, B).await;
1246 add_sequential(&mut indexer, C).await;
1247
1248 assert_eq!(indexer.first_checkpoint, Some(24));
1249 assert_eq!(indexer.last_checkpoint, None);
1250 assert_eq!(indexer.latest_checkpoint, 0);
1251 assert_eq!(indexer.next_checkpoint, 11);
1252 assert_eq!(indexer.next_sequential_checkpoint, Some(11));
1253 }
1254
1255 #[tokio::test]
1257 async fn test_latest_checkpoint_from_watermark() {
1258 let registry = Registry::new();
1259 let store = FallibleMockStore::default();
1260 let temp_dir = init_ingestion_dir(Some(30));
1261 let indexer = Indexer::new(
1262 store,
1263 IndexerArgs::default(),
1264 ClientArgs {
1265 ingestion: IngestionClientArgs {
1266 local_ingestion_path: Some(temp_dir.path().to_owned()),
1267 ..Default::default()
1268 },
1269 ..Default::default()
1270 },
1271 IngestionConfig::default(),
1272 None,
1273 ®istry,
1274 )
1275 .await
1276 .unwrap();
1277
1278 assert_eq!(indexer.latest_checkpoint, 30);
1279 }
1280
1281 #[tokio::test]
1284 async fn test_next_checkpoint_with_pruner_uses_retention() {
1285 let retention = LATEST_CHECKPOINT - 1;
1286 let indexer = test_next_checkpoint(None, None, pruner_config(retention)).await;
1287 assert_eq!(indexer.next_checkpoint, LATEST_CHECKPOINT - retention);
1288 }
1289
1290 #[tokio::test]
1292 async fn test_next_checkpoint_without_pruner_falls_back_to_genesis() {
1293 let indexer = test_next_checkpoint(None, None, ConcurrentConfig::default()).await;
1294 assert_eq!(indexer.next_checkpoint, 0);
1295 }
1296
1297 #[tokio::test]
1299 async fn test_next_checkpoint_watermark_takes_priority_over_pruner() {
1300 let retention = LATEST_CHECKPOINT - 1;
1301 let indexer = test_next_checkpoint(Some(5), None, pruner_config(retention)).await;
1302 assert_eq!(indexer.next_checkpoint, 6);
1303 }
1304
1305 #[tokio::test]
1308 async fn test_next_checkpoint_first_checkpoint_takes_priority_over_pruner() {
1309 let retention = LATEST_CHECKPOINT - 1;
1310 let indexer = test_next_checkpoint(None, Some(2), pruner_config(retention)).await;
1311 assert_eq!(indexer.next_checkpoint, 2);
1312 }
1313
1314 #[tokio::test]
1316 async fn test_next_checkpoint_retention_exceeds_latest_checkpoint() {
1317 let retention = LATEST_CHECKPOINT + 1;
1318 let indexer = test_next_checkpoint(None, None, pruner_config(retention)).await;
1319 assert_eq!(indexer.next_checkpoint, 0);
1320 }
1321
1322 #[tokio::test]
1324 async fn test_indexer_ingestion_existing_watermarks_no_first_checkpoint() {
1325 let registry = Registry::new();
1326 let store = FallibleMockStore::default();
1327
1328 test_pipeline!(A, "concurrent_a");
1329 test_pipeline!(B, "concurrent_b");
1330 test_pipeline!(C, "sequential_c");
1331 test_pipeline!(D, "sequential_d");
1332
1333 let mut conn = store.connect().await.unwrap();
1334 set_committer_watermark(&mut conn, A::NAME, 5).await;
1335 set_committer_watermark(&mut conn, B::NAME, 10).await;
1336 set_committer_watermark(&mut conn, C::NAME, 15).await;
1337 set_committer_watermark(&mut conn, D::NAME, 20).await;
1338
1339 let indexer_args = IndexerArgs {
1340 last_checkpoint: Some(29),
1341 ..Default::default()
1342 };
1343 let (mut indexer, _temp_dir) =
1344 create_test_indexer(store.clone(), indexer_args, ®istry, Some((30, 1))).await;
1345
1346 add_concurrent(&mut indexer, A).await;
1347 add_concurrent(&mut indexer, B).await;
1348 add_sequential(&mut indexer, C).await;
1349 add_sequential(&mut indexer, D).await;
1350
1351 let ingestion_metrics = indexer.ingestion_metrics().clone();
1352 let indexer_metrics = indexer.indexer_metrics().clone();
1353
1354 indexer.run().await.unwrap().join().await.unwrap();
1355
1356 assert_eq!(
1357 ingestion_metrics
1358 .total_ingested_checkpoints
1359 .with_label_values(&["0"])
1360 .get(),
1361 24
1362 );
1363 assert_out_of_order!(indexer_metrics, A::NAME, 0);
1364 assert_out_of_order!(indexer_metrics, B::NAME, 5);
1365 assert_out_of_order!(indexer_metrics, C::NAME, 10);
1366 assert_out_of_order!(indexer_metrics, D::NAME, 15);
1367 }
1368
1369 #[tokio::test]
1371 async fn test_indexer_ingestion_existing_watermarks_ignore_first_checkpoint() {
1372 let registry = Registry::new();
1373 let store = FallibleMockStore::default();
1374
1375 test_pipeline!(A, "concurrent_a");
1376 test_pipeline!(B, "concurrent_b");
1377 test_pipeline!(C, "sequential_c");
1378 test_pipeline!(D, "sequential_d");
1379
1380 let mut conn = store.connect().await.unwrap();
1381 set_committer_watermark(&mut conn, A::NAME, 5).await;
1382 set_committer_watermark(&mut conn, B::NAME, 10).await;
1383 set_committer_watermark(&mut conn, C::NAME, 15).await;
1384 set_committer_watermark(&mut conn, D::NAME, 20).await;
1385
1386 let indexer_args = IndexerArgs {
1387 first_checkpoint: Some(3),
1388 last_checkpoint: Some(29),
1389 ..Default::default()
1390 };
1391 let (mut indexer, _temp_dir) =
1392 create_test_indexer(store.clone(), indexer_args, ®istry, Some((30, 1))).await;
1393
1394 add_concurrent(&mut indexer, A).await;
1395 add_concurrent(&mut indexer, B).await;
1396 add_sequential(&mut indexer, C).await;
1397 add_sequential(&mut indexer, D).await;
1398
1399 let ingestion_metrics = indexer.ingestion_metrics().clone();
1400 let metrics = indexer.indexer_metrics().clone();
1401 indexer.run().await.unwrap().join().await.unwrap();
1402
1403 assert_eq!(
1404 ingestion_metrics
1405 .total_ingested_checkpoints
1406 .with_label_values(&["0"])
1407 .get(),
1408 24
1409 );
1410 assert_out_of_order!(metrics, A::NAME, 0);
1411 assert_out_of_order!(metrics, B::NAME, 5);
1412 assert_out_of_order!(metrics, C::NAME, 10);
1413 assert_out_of_order!(metrics, D::NAME, 15);
1414 }
1415
1416 #[tokio::test]
1418 async fn test_indexer_ingestion_missing_watermarks_no_first_checkpoint() {
1419 let registry = Registry::new();
1420 let store = FallibleMockStore::default();
1421
1422 test_pipeline!(A, "concurrent_a");
1423 test_pipeline!(B, "concurrent_b");
1424 test_pipeline!(C, "sequential_c");
1425 test_pipeline!(D, "sequential_d");
1426
1427 let mut conn = store.connect().await.unwrap();
1428 set_committer_watermark(&mut conn, B::NAME, 10).await;
1429 set_committer_watermark(&mut conn, C::NAME, 15).await;
1430 set_committer_watermark(&mut conn, D::NAME, 20).await;
1431
1432 let indexer_args = IndexerArgs {
1433 last_checkpoint: Some(29),
1434 ..Default::default()
1435 };
1436 let (mut indexer, _temp_dir) =
1437 create_test_indexer(store.clone(), indexer_args, ®istry, Some((30, 1))).await;
1438
1439 add_concurrent(&mut indexer, A).await;
1440 add_concurrent(&mut indexer, B).await;
1441 add_sequential(&mut indexer, C).await;
1442 add_sequential(&mut indexer, D).await;
1443
1444 let ingestion_metrics = indexer.ingestion_metrics().clone();
1445 let metrics = indexer.indexer_metrics().clone();
1446 indexer.run().await.unwrap().join().await.unwrap();
1447
1448 assert_eq!(
1449 ingestion_metrics
1450 .total_ingested_checkpoints
1451 .with_label_values(&["0"])
1452 .get(),
1453 30
1454 );
1455 assert_out_of_order!(metrics, A::NAME, 0);
1456 assert_out_of_order!(metrics, B::NAME, 11);
1457 assert_out_of_order!(metrics, C::NAME, 16);
1458 assert_out_of_order!(metrics, D::NAME, 21);
1459 }
1460
1461 #[tokio::test]
1463 async fn test_indexer_ingestion_use_first_checkpoint() {
1464 let registry = Registry::new();
1465 let store = FallibleMockStore::default();
1466
1467 test_pipeline!(A, "concurrent_a");
1468 test_pipeline!(B, "concurrent_b");
1469 test_pipeline!(C, "sequential_c");
1470 test_pipeline!(D, "sequential_d");
1471
1472 let mut conn = store.connect().await.unwrap();
1473 set_committer_watermark(&mut conn, B::NAME, 10).await;
1474 set_committer_watermark(&mut conn, C::NAME, 15).await;
1475 set_committer_watermark(&mut conn, D::NAME, 20).await;
1476
1477 let indexer_args = IndexerArgs {
1478 first_checkpoint: Some(10),
1479 last_checkpoint: Some(29),
1480 ..Default::default()
1481 };
1482 let (mut indexer, _temp_dir) =
1483 create_test_indexer(store.clone(), indexer_args, ®istry, Some((30, 1))).await;
1484
1485 add_concurrent(&mut indexer, A).await;
1486 add_concurrent(&mut indexer, B).await;
1487 add_sequential(&mut indexer, C).await;
1488 add_sequential(&mut indexer, D).await;
1489
1490 let ingestion_metrics = indexer.ingestion_metrics().clone();
1491 let metrics = indexer.indexer_metrics().clone();
1492 indexer.run().await.unwrap().join().await.unwrap();
1493
1494 assert_eq!(
1495 ingestion_metrics
1496 .total_ingested_checkpoints
1497 .with_label_values(&["0"])
1498 .get(),
1499 20
1500 );
1501 assert_out_of_order!(metrics, A::NAME, 0);
1502 assert_out_of_order!(metrics, B::NAME, 1);
1503 assert_out_of_order!(metrics, C::NAME, 6);
1504 assert_out_of_order!(metrics, D::NAME, 11);
1505 }
1506
1507 #[tokio::test]
1508 async fn test_init_watermark_concurrent_no_first_checkpoint() {
1509 let (committer_watermark, pruner_watermark) = test_init_watermark(None, true).await;
1510 assert_eq!(committer_watermark, None);
1511 assert_eq!(pruner_watermark, None);
1512 }
1513
1514 #[tokio::test]
1515 async fn test_init_watermark_concurrent_first_checkpoint_0() {
1516 let (committer_watermark, pruner_watermark) = test_init_watermark(Some(0), true).await;
1517 assert_eq!(committer_watermark, None);
1518 assert_eq!(pruner_watermark, None);
1519 }
1520
1521 #[tokio::test]
1522 async fn test_init_watermark_concurrent_first_checkpoint_1() {
1523 let (committer_watermark, pruner_watermark) = test_init_watermark(Some(1), true).await;
1524
1525 let committer_watermark = committer_watermark.unwrap();
1526 assert_eq!(committer_watermark.checkpoint_hi_inclusive, 0);
1527
1528 let pruner_watermark = pruner_watermark.unwrap();
1529 assert_eq!(pruner_watermark.reader_lo, 1);
1530 assert_eq!(pruner_watermark.pruner_hi, 1);
1531 }
1532
1533 #[tokio::test]
1534 async fn test_init_watermark_sequential() {
1535 let (committer_watermark, pruner_watermark) = test_init_watermark(Some(1), false).await;
1536
1537 let committer_watermark = committer_watermark.unwrap();
1538 assert_eq!(committer_watermark.checkpoint_hi_inclusive, 0);
1539
1540 let pruner_watermark = pruner_watermark.unwrap();
1541 assert_eq!(pruner_watermark.reader_lo, 1);
1542 assert_eq!(pruner_watermark.pruner_hi, 1);
1543 }
1544
1545 #[tokio::test]
1546 async fn test_multiple_sequential_pipelines_next_checkpoint() {
1547 let registry = Registry::new();
1548 let store = FallibleMockStore::default();
1549
1550 let mut conn = store.connect().await.unwrap();
1551 set_committer_watermark(&mut conn, MockHandler::NAME, 10).await;
1552 set_committer_watermark(&mut conn, SequentialHandler::NAME, 5).await;
1553
1554 let indexer_args = IndexerArgs {
1555 first_checkpoint: None,
1556 last_checkpoint: Some(19),
1557 pipeline: vec![],
1558 ..Default::default()
1559 };
1560 let (mut indexer, _temp_dir) =
1561 create_test_indexer(store.clone(), indexer_args, ®istry, Some((20, 2))).await;
1562
1563 add_sequential(&mut indexer, MockHandler).await;
1565
1566 assert_eq!(
1568 indexer.next_sequential_checkpoint(),
1569 Some(11),
1570 "next_sequential_checkpoint should be 11"
1571 );
1572
1573 add_sequential(&mut indexer, SequentialHandler).await;
1575
1576 assert_eq!(
1578 indexer.next_sequential_checkpoint(),
1579 Some(6),
1580 "next_sequential_checkpoint should still be 6"
1581 );
1582
1583 indexer.run().await.unwrap().join().await.unwrap();
1585
1586 let watermark1 = conn.committer_watermark(MockHandler::NAME).await.unwrap();
1588 let watermark2 = conn
1589 .committer_watermark(SequentialHandler::NAME)
1590 .await
1591 .unwrap();
1592
1593 assert_eq!(watermark1.unwrap().checkpoint_hi_inclusive, 19);
1594 assert_eq!(watermark2.unwrap().checkpoint_hi_inclusive, 19);
1595 }
1596
1597 #[tokio::test]
1601 async fn test_tasked_pipelines_ignore_below_main_reader_lo() {
1602 let registry = Registry::new();
1603 let store = FallibleMockStore::default();
1604
1605 let mut conn = store.connect().await.unwrap();
1608 set_committer_watermark(&mut conn, MockCheckpointSequenceNumberHandler::NAME, 10).await;
1609 conn.set_reader_watermark(MockCheckpointSequenceNumberHandler::NAME, 7)
1610 .await
1611 .unwrap();
1612
1613 let indexer_args = IndexerArgs {
1616 first_checkpoint: Some(0),
1617 last_checkpoint: Some(15),
1618 task: TaskArgs::tasked("task".to_string(), 10),
1619 ..Default::default()
1620 };
1621 let (mut tasked_indexer, _temp_dir) =
1622 create_test_indexer(store.clone(), indexer_args, ®istry, Some((16, 2))).await;
1623
1624 add_concurrent(&mut tasked_indexer, MockCheckpointSequenceNumberHandler).await;
1625
1626 let ingestion_metrics = tasked_indexer.ingestion_metrics().clone();
1627 let metrics = tasked_indexer.indexer_metrics().clone();
1628
1629 tasked_indexer.run().await.unwrap().join().await.unwrap();
1630
1631 assert_eq!(
1632 ingestion_metrics
1633 .total_ingested_checkpoints
1634 .with_label_values(&["0"])
1635 .get(),
1636 16
1637 );
1638 assert_eq!(
1639 metrics
1640 .total_collector_skipped_checkpoints
1641 .get_metric_with_label_values(&[MockCheckpointSequenceNumberHandler::NAME])
1642 .unwrap()
1643 .get(),
1644 7
1645 );
1646 let data = store
1647 .data
1648 .get(MockCheckpointSequenceNumberHandler::NAME)
1649 .unwrap();
1650 assert_eq!(data.len(), 9);
1651 for i in 0..7 {
1652 assert!(data.get(&i).is_none());
1653 }
1654 for i in 7..16 {
1655 assert!(data.get(&i).is_some());
1656 }
1657 }
1658
1659 #[tokio::test]
1663 async fn test_tasked_indexer_skips_excluded_sequential_pipeline() {
1664 let store = FallibleMockStore::default();
1665 let tasked = |pipeline: Vec<&str>| IndexerArgs {
1666 first_checkpoint: Some(0),
1667 last_checkpoint: Some(4),
1668 pipeline: pipeline.into_iter().map(String::from).collect(),
1669 task: TaskArgs::tasked("task".to_string(), 10),
1670 };
1671
1672 let (mut indexer, _temp_dir) = create_test_indexer(
1673 store.clone(),
1674 tasked(vec![MockCheckpointSequenceNumberHandler::NAME]),
1675 &Registry::new(),
1676 Some((5, 1)),
1677 )
1678 .await;
1679 add_sequential(&mut indexer, SequentialHandler).await;
1680 add_concurrent(&mut indexer, MockCheckpointSequenceNumberHandler).await;
1681 indexer.run().await.unwrap().join().await.unwrap();
1682
1683 let tasked_watermark = store.watermark("test@task").unwrap();
1684 assert_eq!(tasked_watermark.checkpoint_hi_inclusive, Some(4));
1685 assert!(
1686 store.watermark(SequentialHandler::NAME).is_none(),
1687 "the excluded sequential pipeline ran"
1688 );
1689
1690 for pipeline in [vec![SequentialHandler::NAME], vec![]] {
1691 let (mut indexer, _temp_dir) =
1692 create_test_indexer(store.clone(), tasked(pipeline), &Registry::new(), None).await;
1693 let err = indexer
1694 .sequential_pipeline(SequentialHandler, SequentialConfig::default())
1695 .await
1696 .unwrap_err();
1697 assert!(
1698 err.to_string()
1699 .contains("Sequential pipelines do not support pipeline tasks"),
1700 "unexpected error: {err:#}"
1701 );
1702 }
1703 }
1704
1705 #[tokio::test]
1707 async fn test_tasked_pipelines_surpass_main_pipeline_committer_hi() {
1708 let registry = Registry::new();
1709 let store = FallibleMockStore::default();
1710
1711 let mut conn = store.connect().await.unwrap();
1712 set_committer_watermark(&mut conn, "test", 10).await;
1713 conn.set_reader_watermark("test", 5).await.unwrap();
1714
1715 let indexer_args = IndexerArgs {
1718 first_checkpoint: Some(9),
1719 last_checkpoint: Some(25),
1720 task: TaskArgs::tasked("task".to_string(), 10),
1721 ..Default::default()
1722 };
1723 let (mut tasked_indexer, _temp_dir) =
1724 create_test_indexer(store.clone(), indexer_args, ®istry, Some((26, 2))).await;
1725
1726 add_concurrent(&mut tasked_indexer, MockCheckpointSequenceNumberHandler).await;
1727
1728 let ingestion_metrics = tasked_indexer.ingestion_metrics().clone();
1729 let metrics = tasked_indexer.indexer_metrics().clone();
1730
1731 tasked_indexer.run().await.unwrap().join().await.unwrap();
1732
1733 assert_eq!(
1734 ingestion_metrics
1735 .total_ingested_checkpoints
1736 .with_label_values(&["0"])
1737 .get(),
1738 17
1739 );
1740 assert_out_of_order!(metrics, "test", 0);
1741 assert_eq!(
1742 metrics
1743 .total_collector_skipped_checkpoints
1744 .get_metric_with_label_values(&[MockCheckpointSequenceNumberHandler::NAME])
1745 .unwrap()
1746 .get(),
1747 0
1748 );
1749
1750 let data = store.data.get("test").unwrap();
1751 assert_eq!(data.len(), 17);
1752 for i in 0..9 {
1753 assert!(data.get(&i).is_none());
1754 }
1755 for i in 9..26 {
1756 assert!(data.get(&i).is_some());
1757 }
1758 let main_pipeline_watermark = store.watermark("test").unwrap();
1759 assert_eq!(main_pipeline_watermark.checkpoint_hi_inclusive, Some(10));
1761 assert_eq!(main_pipeline_watermark.reader_lo, 5);
1762 let tasked_pipeline_watermark = store.watermark("test@task").unwrap();
1763 assert_eq!(tasked_pipeline_watermark.checkpoint_hi_inclusive, Some(25));
1764 assert_eq!(tasked_pipeline_watermark.reader_lo, 9);
1765 }
1766
1767 #[tokio::test]
1770 async fn test_tasked_pipelines_skip_checkpoints_trailing_main_reader_lo() {
1771 let registry = Registry::new();
1772 let store = FallibleMockStore::default();
1773 let mut conn = store.connect().await.unwrap();
1774 set_committer_watermark(&mut conn, ControllableHandler::NAME, 11).await;
1776
1777 let indexer_args = IndexerArgs {
1779 first_checkpoint: Some(0),
1780 last_checkpoint: Some(500),
1781 task: TaskArgs::tasked("task".to_string(), 10 ),
1782 ..Default::default()
1783 };
1784 let (mut tasked_indexer, _temp_dir) =
1785 create_test_indexer(store.clone(), indexer_args, ®istry, Some((501, 2))).await;
1786 let mut allow_process = 10;
1787 let (controllable_handler, process_below) = ControllableHandler::with_limit(allow_process);
1789 let _ = tasked_indexer
1790 .concurrent_pipeline(
1791 controllable_handler,
1792 ConcurrentConfig {
1793 committer: CommitterConfig {
1794 watermark_interval_ms: 10,
1795 ..Default::default()
1796 },
1797 fanout: Some(ConcurrencyConfig::Fixed { value: 501 }),
1800 ..Default::default()
1801 },
1802 )
1803 .await;
1804 let metrics = tasked_indexer.indexer_metrics().clone();
1805
1806 let mut s_indexer = tasked_indexer.run().await.unwrap();
1807
1808 store
1812 .wait_for_watermark(
1813 &pipeline_task::<FallibleMockStore>(ControllableHandler::NAME, Some("task"))
1814 .unwrap(),
1815 10,
1816 Duration::from_secs(10),
1817 )
1818 .await;
1819
1820 conn.set_reader_watermark(ControllableHandler::NAME, 250)
1825 .await
1826 .unwrap();
1827
1828 let reader_lo = metrics
1829 .collector_reader_lo
1830 .with_label_values(&[ControllableHandler::NAME]);
1831
1832 let mut interval = tokio::time::interval(Duration::from_millis(10));
1836 while reader_lo.get() != 250 {
1837 interval.tick().await;
1838 allow_process += 1;
1840 assert!(
1841 allow_process <= 500,
1842 "Released all checkpoints but collector never observed new reader_lo"
1843 );
1844 process_below.send(allow_process).ok();
1845 }
1846
1847 process_below.send(500).ok();
1854
1855 s_indexer.join().await.unwrap();
1856
1857 let data = store.data.get(ControllableHandler::NAME).unwrap();
1858
1859 for chkpt in (allow_process + 1)..250 {
1861 assert!(
1862 data.get(&chkpt).is_none(),
1863 "Checkpoint {chkpt} should have been skipped"
1864 );
1865 }
1866
1867 for chkpt in 250..=500 {
1869 assert!(
1870 data.get(&chkpt).is_some(),
1871 "Checkpoint {chkpt} should have been committed (>= reader_lo)"
1872 );
1873 }
1874
1875 for chkpt in 0..=10 {
1877 assert!(
1878 data.get(&chkpt).is_some(),
1879 "Checkpoint {chkpt} should have been committed (baseline)"
1880 );
1881 }
1882 }
1883
1884 #[tokio::test]
1888 async fn test_multi_cohort_ingestion() {
1889 let registry = Registry::new();
1890 let store = FallibleMockStore::default();
1891
1892 test_pipeline!(A, "near_tip");
1893 test_pipeline!(B, "far_behind");
1894
1895 let temp_dir = multi_cohort_setup(&store, A::NAME, B::NAME).await;
1896
1897 let mut indexer = Indexer::new(
1898 store.clone(),
1899 IndexerArgs {
1900 last_checkpoint: Some(59),
1901 ..Default::default()
1902 },
1903 ClientArgs {
1904 ingestion: IngestionClientArgs {
1905 local_ingestion_path: Some(temp_dir.path().to_owned()),
1906 ..Default::default()
1907 },
1908 ..Default::default()
1909 },
1910 IngestionConfig::default(),
1911 None,
1912 ®istry,
1913 )
1914 .await
1915 .unwrap();
1916
1917 add_concurrent(&mut indexer, A).await;
1918 add_concurrent(&mut indexer, B).await;
1919
1920 let ingestion_metrics = indexer.ingestion_metrics().clone();
1921 let indexer_metrics = indexer.indexer_metrics().clone();
1922 indexer.run().await.unwrap().join().await.unwrap();
1923
1924 assert_eq!(
1928 ingestion_metrics
1929 .total_ingested_checkpoints
1930 .with_label_values(&["0"])
1931 .get(),
1932 0
1933 );
1934 assert_eq!(
1935 ingestion_metrics
1936 .total_ingested_checkpoints
1937 .with_label_values(&["1"])
1938 .get(),
1939 50
1940 );
1941
1942 let received = |name| {
1945 indexer_metrics
1946 .total_handler_checkpoints_received
1947 .get_metric_with_label_values(&[name])
1948 .unwrap()
1949 .get()
1950 };
1951 assert_eq!(received(A::NAME), 0);
1952 assert_eq!(received(B::NAME), 50);
1953
1954 let watermark = store.watermark(B::NAME).unwrap();
1957 assert_eq!(watermark.checkpoint_hi_inclusive, Some(59));
1958 let watermark = store.watermark(A::NAME).unwrap();
1959 assert_eq!(watermark.checkpoint_hi_inclusive, Some(100_100));
1960 }
1961
1962 #[tokio::test]
1965 async fn test_multi_cohort_ingestion_with_clients() {
1966 let registry = Registry::new();
1967 let store = FallibleMockStore::default();
1968
1969 test_pipeline!(A, "near_tip");
1970 test_pipeline!(B, "far_behind");
1971
1972 let temp_dir = multi_cohort_setup(&store, A::NAME, B::NAME).await;
1973
1974 let ingestion_metrics = IngestionMetrics::new(None, ®istry);
1975 let ingestion_client = IngestionClient::new(
1976 IngestionClientArgs {
1977 local_ingestion_path: Some(temp_dir.path().to_owned()),
1978 ..Default::default()
1979 },
1980 ingestion_metrics.clone(),
1981 )
1982 .unwrap();
1983
1984 let mut indexer = Indexer::with_ingestion_clients(
1985 store.clone(),
1986 IndexerArgs {
1987 last_checkpoint: Some(59),
1988 ..Default::default()
1989 },
1990 ingestion_client,
1991 None,
1992 None,
1993 IngestionConfig::default(),
1994 None,
1995 ®istry,
1996 )
1997 .await
1998 .unwrap();
1999
2000 add_concurrent(&mut indexer, A).await;
2001 add_concurrent(&mut indexer, B).await;
2002
2003 let indexer_metrics = indexer.indexer_metrics().clone();
2004 indexer.run().await.unwrap().join().await.unwrap();
2005
2006 assert_eq!(
2009 ingestion_metrics
2010 .total_ingested_checkpoints
2011 .with_label_values(&["0"])
2012 .get(),
2013 0
2014 );
2015 assert_eq!(
2016 ingestion_metrics
2017 .total_ingested_checkpoints
2018 .with_label_values(&["1"])
2019 .get(),
2020 50
2021 );
2022 assert_eq!(
2023 indexer_metrics
2024 .total_handler_checkpoints_received
2025 .get_metric_with_label_values(&[A::NAME])
2026 .unwrap()
2027 .get(),
2028 0
2029 );
2030
2031 let watermark = store.watermark(B::NAME).unwrap();
2032 assert_eq!(watermark.checkpoint_hi_inclusive, Some(59));
2033 }
2034
2035 #[tokio::test]
2040 async fn test_near_tip_cohort_progresses_while_far_behind_stalls() {
2041 const TIP: u64 = 100;
2042
2043 let registry = Registry::new();
2044 let store = FallibleMockStore::default();
2045
2046 test_pipeline!(A, "near_tip");
2047
2048 let (far_behind, _release) = ControllableHandler::with_limit(9);
2052
2053 let temp_dir = init_ingestion_dir(Some(TIP));
2054 let mut conn = store.connect().await.unwrap();
2055 set_committer_watermark(&mut conn, A::NAME, 94).await; set_committer_watermark(&mut conn, ControllableHandler::NAME, 9).await; synthetic_ingestion::generate_ingestion(synthetic_ingestion::Config {
2058 ingestion_dir: temp_dir.path().to_owned(),
2059 starting_checkpoint: 0,
2060 num_checkpoints: TIP + 1, checkpoint_size: 1,
2062 })
2063 .await;
2064
2065 let mut indexer = Indexer::new(
2066 store.clone(),
2067 IndexerArgs {
2068 last_checkpoint: Some(TIP),
2069 ..Default::default()
2070 },
2071 ClientArgs {
2072 ingestion: IngestionClientArgs {
2073 local_ingestion_path: Some(temp_dir.path().to_owned()),
2074 ..Default::default()
2075 },
2076 ..Default::default()
2077 },
2078 IngestionConfig {
2081 min_cohort_boundary: 10,
2082 ..Default::default()
2083 },
2084 None,
2085 ®istry,
2086 )
2087 .await
2088 .unwrap();
2089
2090 indexer
2093 .concurrent_pipeline(
2094 A,
2095 ConcurrentConfig {
2096 committer: CommitterConfig {
2097 watermark_interval_ms: 10,
2098 ..Default::default()
2099 },
2100 ..Default::default()
2101 },
2102 )
2103 .await
2104 .unwrap();
2105 add_concurrent(&mut indexer, far_behind).await;
2107
2108 let service = indexer.run().await.unwrap();
2109
2110 store
2114 .wait_for_watermark(A::NAME, TIP, Duration::from_secs(10))
2115 .await;
2116
2117 assert_eq!(
2120 store
2121 .watermark(ControllableHandler::NAME)
2122 .unwrap()
2123 .checkpoint_hi_inclusive,
2124 Some(9),
2125 );
2126 assert!(store.data.get(ControllableHandler::NAME).is_none());
2127
2128 drop(service);
2130 }
2131}