feat(propulsion sync): Add --requireAll (#262)

src/Propulsion.CosmosStore/CosmosStoreSink.fs

@@ -180,7 +180,7 @@ type CosmosStoreSink =
     /// Starts a <c>Sink</c> that ingests all submitted events into the supplied <c>context</c>
     static member Start
         (   log: ILogger, maxReadAhead, eventsContext, maxConcurrentStreams, stats: CosmosStoreSinkStats,
-            ?purgeInterval, ?wakeForResults, ?idleDelay,
+            ?purgeInterval, ?wakeForResults, ?idleDelay, ?requireAll,
             // Default: 16384
             ?maxEvents,
             // Default: 256KB (limited by maximum size of a CosmosDB stored procedure invocation)
@@ -190,6 +190,6 @@ type CosmosStoreSink =
         let scheduler =
             let dumpStreams logStreamStates _log = logStreamStates Event.storedSize
             Scheduling.Engine(dispatcher, stats, dumpStreams, pendingBufferSize = 5, prioritizeStreamsBy = Event.storedSize,
-                              ?purgeInterval = purgeInterval, ?wakeForResults = wakeForResults, ?idleDelay = idleDelay)
+                              ?purgeInterval = purgeInterval, ?wakeForResults = wakeForResults, ?idleDelay = idleDelay, ?requireAll = requireAll)
         Factory.Start(log, scheduler.Pump, maxReadAhead, scheduler,
                       ingesterStateInterval = defaultArg ingesterStateInterval stats.StateInterval.Period, ?commitInterval = commitInterval)

src/Propulsion.CosmosStore/Propulsion.CosmosStore.fsproj

@@ -5,6 +5,7 @@
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
 <!--    <PackageValidationBaselineVersion>3.0.0-rc.12</PackageValidationBaselineVersion>-->
+<!--    <PackageValidationBaselineVersion>3.0.0-rc.12.1</PackageValidationBaselineVersion>-->
   </PropertyGroup>
 
   <ItemGroup>
@@ -27,7 +28,7 @@
 
   <ItemGroup>
     <ProjectReference Condition=" '$(Configuration)' == 'Debug' " Include="..\Propulsion\Propulsion.fsproj" />
-    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.12, 4.0.0)" />
+    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.13, 4.0.0)" />
   </ItemGroup>
 
 </Project>

src/Propulsion.CosmosStore3/Propulsion.CosmosStore3.fsproj

@@ -4,7 +4,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
-    <PackageValidationBaselineVersion>3.0.0-rc.12</PackageValidationBaselineVersion>
+<!--    <PackageValidationBaselineVersion>3.0.0-rc.12</PackageValidationBaselineVersion>-->
     <DefineConstants>COSMOSV3</DefineConstants>
   </PropertyGroup>
 
@@ -46,7 +46,7 @@
 
   <ItemGroup>
     <ProjectReference Condition=" '$(Configuration)' == 'Debug' " Include="..\Propulsion\Propulsion.fsproj" />
-    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.12, 4.0.0)" />
+    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.13, 4.0.0)" />
   </ItemGroup>
 
 </Project>

src/Propulsion.DynamoStore.Notifier/Propulsion.DynamoStore.Notifier.fsproj

@@ -43,7 +43,7 @@
       <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.4.10" />
       <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
       <ProjectReference Condition=" '$(Configuration)' == 'Debug' " Include="..\Propulsion\Propulsion.fsproj" />
-      <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.12, 4.0.0)" />
+      <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.13, 4.0.0)" />
     </ItemGroup>
 
     <!-- The Notifier dotnet new template extracts the published binaries from the tools folder using logic in https://github.com/jet/propulsion/pull/143 -->

src/Propulsion.EventStore/Propulsion.EventStore.fsproj

@@ -25,7 +25,7 @@
 
   <ItemGroup>
     <ProjectReference Condition=" '$(Configuration)' == 'Debug' " Include="..\Propulsion\Propulsion.fsproj" />
-    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.12, 4.0.0)" />
+    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.13, 4.0.0)" />
   </ItemGroup>
 
 </Project>

src/Propulsion.Feed/Propulsion.Feed.fsproj

@@ -23,7 +23,7 @@
 
     <ItemGroup>
         <ProjectReference Condition=" '$(Configuration)' == 'Debug' " Include="..\Propulsion\Propulsion.fsproj" />
-        <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.12, 4.0.0)" />
+        <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.13, 4.0.0)" />
     </ItemGroup>
 
 </Project>

src/Propulsion.Kafka/Propulsion.Kafka.fsproj

@@ -23,7 +23,7 @@
 
   <ItemGroup>
     <ProjectReference Condition=" '$(Configuration)' == 'Debug' " Include="..\Propulsion\Propulsion.fsproj" />
-    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.12, 4.0.0)" />
+    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.13, 4.0.0)" />
   </ItemGroup>
 
 </Project>

src/Propulsion.MemoryStore/Propulsion.MemoryStore.fsproj

@@ -27,7 +27,7 @@
 
   <ItemGroup>
     <ProjectReference Condition=" '$(Configuration)' == 'Debug' " Include="..\Propulsion\Propulsion.fsproj" />
-    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.12, 4.0.0)" />
+    <PackageReference Condition=" '$(Configuration)' == 'Release' " Include="Propulsion" Version="[3.0.0-rc.13, 4.0.0)" />
   </ItemGroup>
 
 </Project>

tools/Propulsion.Tool/Sync.fs

@@ -11,10 +11,11 @@ type [<NoEquality; NoComparison; RequireSubcommand>] Parameters =
     | [<AltCommandLine "-w"; Unique>]       MaxWriters of int
     | [<AltCommandLine "-Z"; Unique>]       FromTail
     | [<AltCommandLine "-F"; Unique>]       Follow
+    | [<AltCommandLine "-A"; Unique>]       RequireAll
     | [<AltCommandLine "-C"; Unique>]       Categorize
     | [<AltCommandLine "-b"; Unique>]       MaxItems of int
 
-    | [<AltCommandLine "-I";    AltCommandLine "--include-indexes"; Unique>] IncIdx
+    | [<AltCommandLine "-I";    AltCommandLine "--include-system"; Unique>] IncSys
     | [<AltCommandLine "-cat";  AltCommandLine "--include-category">]   IncCat of    regex: string
     | [<AltCommandLine "-ncat"; AltCommandLine "--exclude-category">]   ExcCat of    regex: string
     | [<AltCommandLine "-sn";   AltCommandLine "--include-streamname">] IncStream of regex: string
@@ -32,10 +33,14 @@ type [<NoEquality; NoComparison; RequireSubcommand>] Parameters =
             | MaxWriters _ ->               "maximum number of concurrent streams on which to process at any time. Default: 8 (Cosmos: 16)."
             | FromTail ->                   "(iff fresh projection) - force starting from present Position. Default: Ensure each and every event is projected from the start."
             | Follow ->                     "Stop when the Tail is reached."
+            | RequireAll ->                 "Wait for out of order events to arrive (including waiting for event 0 per stream) before dispatching for any stream. " +
+                                            "NOTE normally a large `MaxReadAhead` and `cosmos -b` is required to avoid starving the scheduler. " +
+                                            "NOTE This mode does not make sense to apply unless the ProcessorName is fresh; if the consumer group name is not fresh (and hence items are excluded from the feed), there will inevitably be missing events, and processing will stall. " +
+                                            "Default: assume events arrive from the changefeed (and/or the input JSON file) without any gaps or out of order deliveries for any stream."
             | Categorize ->                 "Gather handler latency stats by category"
             | MaxItems _ ->                 "Controls checkpointing granularity by adjusting the batch size being loaded from the feed. Default: Unlimited"
 
-            | IncIdx ->                     "Include Index streams. Default: Exclude Index Streams, identified by a $ prefix."
+            | IncSys ->                     "Include System streams. Default: Exclude Index Streams, identified by a $ prefix."
             | IncCat _ ->                   "Allow Stream Category. Multiple values are combined with OR. Default: include all, subject to Category Deny and Stream Deny rules."
             | ExcCat _ ->                   "Deny  Stream Category. Specified values/regexes are applied after the Category Allow rule(s)."
             | IncStream _ ->                "Allow Stream Name. Multiple values are combined with OR. Default: Allow all streams that pass the category Allow test, Fail the Category and Stream deny tests."
@@ -51,7 +56,7 @@ and Arguments(c, p: ParseResults<Parameters>) =
     member val Filters = Propulsion.StreamFilter(
                                             allowCats = p.GetResults IncCat, denyCats = p.GetResults ExcCat,
                                             allowSns = p.GetResults IncStream, denySns = p.GetResults ExcStream,
-                                            incIndexes = p.Contains IncIdx,
+                                            includeSystem = p.Contains IncSys,
                                             allowEts = p.GetResults IncEvent, denyEts = p.GetResults ExcEvent)
     member val Categorize =                 p.Contains Categorize
     member val Command =
@@ -217,7 +222,7 @@ let run appName (c: Args.Configuration, p: ParseResults<Parameters>) = async {
         | Some x, _ -> x
         | None, Json _ -> System.Guid.NewGuid() |> _.ToString("N")
         | None, _ -> p.Raise "ConsumerGroupName is mandatory, unless consuming from a JSON file"
-    let startFromTail, follow, maxItems = p.Contains FromTail, p.Contains Follow, p.TryGetResult MaxItems
+    let startFromTail, follow, requireAll, maxItems = p.Contains FromTail, p.Contains Follow, p.Contains RequireAll, p.TryGetResult MaxItems
     let producer =
         match a.Command with
         | SubCommand.Kafka a ->
@@ -240,15 +245,19 @@ let run appName (c: Args.Configuration, p: ParseResults<Parameters>) = async {
                 match producer with
                 | None -> ()
                 | Some producer ->
-                    let json = Propulsion.Codec.NewtonsoftJson.RenderedSpan.ofStreamSpan stream events |> Propulsion.Codec.NewtonsoftJson.Serdes.Serialize
+                    let json = Propulsion.Codec.NewtonsoftJson.RenderedSpan.ofStreamSpan stream events
+                               |> Propulsion.Codec.NewtonsoftJson.Serdes.Serialize
                     do! producer.ProduceAsync(FsCodec.StreamName.toString stream, json) |> Async.Ignore
                 return Propulsion.Sinks.StreamResult.AllProcessed, Outcome.render_ stream ham spam 0 }
-            Propulsion.Sinks.Factory.StartConcurrent(Log.Logger, maxReadAhead, maxConcurrentProcessors, handle a.Filters.EventFilter, stats)
+            Propulsion.Sinks.Factory.StartConcurrent(Log.Logger, maxReadAhead, maxConcurrentProcessors, handle a.Filters.EventFilter, stats,
+                                                     requireAll = requireAll)
         | SubCommand.Sync sa ->
             let eventsContext = sa.ConnectEvents() |> Async.RunSynchronously
-            let stats = Propulsion.CosmosStore.CosmosStoreSinkStats(Log.Logger, statsInterval, stateInterval, logExternalStats = dumpStoreStats, Categorize = a.Categorize)
+            let stats = Propulsion.CosmosStore.CosmosStoreSinkStats(Log.Logger, statsInterval, stateInterval,
+                                                                    logExternalStats = dumpStoreStats, Categorize = a.Categorize)
             Propulsion.CosmosStore.CosmosStoreSink.Start(Metrics.log, maxReadAhead, eventsContext, maxConcurrentProcessors, stats,
-                                                         purgeInterval = TimeSpan.hours 1, maxBytes = sa.MaxBytes)
+                                                         maxBytes = sa.MaxBytes, requireAll = requireAll,
+                                                         ?purgeInterval = if requireAll then None else Some (TimeSpan.hours 1))
     let source =
         match a.Command.Source with
         | Cosmos sa ->