Cosmos Store

Directory.Build.props

@@ -20,8 +20,9 @@
     <EnableSourceLink Condition=" '$(OS)' != 'Windows_NT' AND '$(MSBuildRuntimeType)' != 'Core' ">false</EnableSourceLink>
 
     <!-- suppress false positive warning FS2003 about invalid version of AssemblyInformationalVersionAttribute -->
-    <!-- Supress NU5105 triggered by trailing dotted elements such as .43 and .2 in e.g.: pr.43-rc1.2: The package version '<X>' uses SemVer 2.0.0 or components of SemVer 1.0.0 that are not supported on legacy clients. Change the package version to a SemVer 1.0.0 string. If the version contains a release label it must start with a letter. This message can be ignored if the package is not intended for older clients. -->
-    <NoWarn>$(NoWarn);FS2003;NU5105</NoWarn>
+    <!-- supress NU5105 triggered by trailing dotted elements such as .43 and .2 in e.g.: pr.43-rc1.2: The package version '<X>' uses SemVer 2.0.0 or components of SemVer 1.0.0 that are not supported on legacy clients. Change the package version to a SemVer 1.0.0 string. If the version contains a release label it must start with a letter. This message can be ignored if the package is not intended for older clients. -->
+    <!-- supress FS0988 triggeredby dotnet sdk insisting on entrypoints for test libs; yes, could hide a problem in an actual EXE, but we'll survive that ;): Main module of program is empty: nothing will happen when it is run -->
+    <NoWarn>$(NoWarn);FS2003;NU5105;FS0988</NoWarn>
   </PropertyGroup>
 
   <!-- Workaround for https://github.com/xunit/xunit/issues/1357 -->

Equinox.sln

@@ -46,6 +46,10 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Codec", "src\Equino
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Tool", "tools\Equinox.Tool\Equinox.Tool.fsproj", "{C8992C1C-6DC5-42CD-A3D7-1C5663433FED}"
 EndProject
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos", "src\Equinox.Cosmos\Equinox.Cosmos.fsproj", "{54EA6187-9F9F-4D67-B602-163D011E43E6}"
+EndProject
+Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos.Integration", "tests\Equinox.Cosmos.Integration\Equinox.Cosmos.Integration.fsproj", "{DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}"
+EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "TodoBackend", "samples\TodoBackend\TodoBackend.fsproj", "{EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Samples", "Samples", "{8F3EB30C-8BA3-4CC0-8361-0EA47C19ABB9}"
@@ -110,6 +114,14 @@ Global
 		{C8992C1C-6DC5-42CD-A3D7-1C5663433FED}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C8992C1C-6DC5-42CD-A3D7-1C5663433FED}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C8992C1C-6DC5-42CD-A3D7-1C5663433FED}.Release|Any CPU.Build.0 = Release|Any CPU
+		{54EA6187-9F9F-4D67-B602-163D011E43E6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{54EA6187-9F9F-4D67-B602-163D011E43E6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{54EA6187-9F9F-4D67-B602-163D011E43E6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{54EA6187-9F9F-4D67-B602-163D011E43E6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DE0FEBF0-72DC-4D4A-BBA7-788D875D6B4B}.Release|Any CPU.Build.0 = Release|Any CPU
 		{EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EC2EC658-3D85-44F3-AD2F-52AFCAFF8871}.Release|Any CPU.ActiveCfg = Release|Any CPU

README.md

@@ -7,7 +7,7 @@ Equinox provides a unified programming model for event sourced processing agains
 Current supported backends are:
 
 - [EventStore](https://eventstore.org/) - this codebase itself has been in production since 2017 (commit history reflects usage), with elements dating back to 2016.
-- [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db/) (See [`cosmos` branch](https://github.com/jet/equinox/tree/cosmos) - will converge with `master` very shortly).
+- [Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db) - contains code dating back to 2016, however [the storage model](https://github.com/jet/equinox/wiki/Cosmos-Storage-Model) was arrived at based on intensive benchmarking squash-merged in [#42](https://github.com/jet/equinox/pull/42).
 - (For integration test purposes only) Volatile in-memory store.
 
 The underlying patterns have their roots in the [DDD-CQRS-ES](https://groups.google.com/forum/#!forum/dddcqrs) community, and the hard work and generosity of countless folks there presenting, explaining, writing and hacking over the years. It would be unfair to single out even a small number of people despite the immense credit that is due.
@@ -26,14 +26,14 @@ _If you're looking to learn more about and/or discuss Event Sourcing and it's my
 - Logging is both high performance and pluggable (using [Serilog](https://github.com/serilog/serilog) to your hosting context (we feed log info to Splunk and the metrics embedded in the LogEvent Properties to Prometheus; see relevant tests for examples)
 - Extracted from working software; currently used for all data storage within Jet's API gateway and Cart processing.
 - Significant test coverage for core facilities, and per Storage system.
-- **`Equinox.EventStore` Transactionally-consistent Rolling Snapshots**: Command processing can be optimized by employing in-stream 'compaction' events in service of the following ends:
+- **`Equinox.EventStore` In-stream Rolling Snapshots**: Command processing can be optimized by employing 'compaction' events in service of the following ends:
   - no additional roundtrips to the store needed at either the Load or Sync points in the flow
   - support, (via `UnionContractEncoder`) for the maintenance of multiple co-existing compaction schemas in a given stream (A snapshot isa Event) 
   - compaction events typically do not get deleted (consistent with how EventStore works), although it is safe to do so in concept
   - NB while this works well, and can deliver excellent performance (especially when allied with the Cache), [it's not a panacea, as noted in this excellent EventStore article on the topic](https://eventstore.org/docs/event-sourcing-basics/rolling-snapshots/index.html)
 - **`Equinox.Cosmos` 'Tip with Unfolds' schema**: In contrast to `Equinox.EventStore`'s `Access.RollingSnapshots`, when using `Equinox.Cosmos`, optimized command processing is managed via the `Tip`; a document per stream with a well-known identity enabling syncs via point-reads by virtue of the fact that the document maintains:
   a) the present Position of the stream - i.e. the index at which the next events will be appended
-  b) compressed [_unfolds_]((https://github.com/jet/equinox/wiki/Cosmos-Storage-Model)
+  b) (compressed) [_unfolds_](https://github.com/jet/equinox/wiki/Cosmos-Storage-Model)
   c) (optionally) events since those unfolded events ([presently removed](https://github.com/jet/equinox/pull/58), but [should return](https://github.com/jet/equinox/wiki/Roadmap))
 
   This yields many of the benefits of the in-stream Rolling Snapshots approach while reducing latency, RU provisioning requirement, and Request Charges:-
@@ -48,42 +48,38 @@ _If you're looking to learn more about and/or discuss Event Sourcing and it's my
 The Equinox components within this repository are delivered as a series of multi-targeted Nuget packages targeting `net461` (F# 3.1+) and `netstandard2.0` (F# 4.5+) profiles; each of the constituent elements is designed to be easily swappable as dictated by the task at hand. Each of the components can be inlined or customized easily:-
 
 - `Equinox.Handler` (Nuget: `Equinox`, depends on `Serilog` (but no specific Serilog sinks, i.e. you can forward to `NLog` etc)): Store-agnostic decision flow runner that manages the optimistic concurrency protocol
-- `Equinox.Codec` (Nuget: `Equinox.Codec`, depends on `TypeShape`, (optionally) `Newtonsoft.Json >= 11.0.2` but can support any serializer): [a scheme for the serializing Events modelled as an F# Discriminated Union with the following capabilities](https://eiriktsarpalis.wordpress.com/2018/10/30/a-contract-pattern-for-schemaless-datastores/):
+- `Equinox.Codec` (Nuget: `Equinox.Codec`, depends on `TypeShape`, `Newtonsoft.Json` (`>= 10.0.3` on `net461`, `>= 11.0.2` on `netstandard2.0` but can support any serializer): [a scheme for the serializing Events modelled as an F# Discriminated Union with the following capabilities](https://eiriktsarpalis.wordpress.com/2018/10/30/a-contract-pattern-for-schemaless-datastores/):
   - independent of any specific serializer
-  - allows tagging of Discriminated Union cases in a versionable manner with low-dependency `DataMember(Name=` tags using [TypeShape](https://github.com/eiriktsarpalis/TypeShape)'s [`UnionContractEncoder`](https://github.com/eiriktsarpalis/TypeShape/blob/master/tests/TypeShape.Tests/UnionContractTests.fs)
-- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `DocumentDb.Client`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`): Production-strength Azure CosmosDb Adapter with integrated transactionally-consistent snapshotting, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements.
-- `Equinox.EventStore` (Nuget: `Equinox.EventStore`, depends on `EventStore.Client[Api.NetCore] >= 4`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`): Production-strength [EventStore](http://geteventstore.com) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements
+  - allows tagging of F# Discriminated Union cases in a versionable manner with low-dependency `DataMember(Name=` tags using [TypeShape](https://github.com/eiriktsarpalis/TypeShape)'s [`UnionContractEncoder`](https://github.com/eiriktsarpalis/TypeShape/blob/master/tests/TypeShape.Tests/UnionContractTests.fs)
+- `Equinox.Tool` (Nuget: `dotnet tool install Equinox.Tool -g`): Tool incorporating a benchmark scenario runner, facilitating running representative load tests composed of transactions in `samples/Store` and `samples/TodoBackend` against any nominated store; this allows perf tuning and measurement in terms of both latency and transaction charge aspects.
 - `Equinox.MemoryStore` (Nuget: `Equinox.MemoryStore`): In-memory store for integration testing/performance baselining/providing out-of-the-box zero dependency storage for examples.
+- `Equinox.EventStore` (Nuget: `Equinox.EventStore`, depends on `EventStore.Client[Api.NetCore] >= 4`, `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`): Production-strength [EventStore](https://eventstore.org/) Adapter instrumented to the degree necessitated by Jet's production monitoring requirements
+- `Equinox.Cosmos` (Nuget: `Equinox.Cosmos`, depends on `System.Runtime.Caching`, `FSharp.Control.AsyncSeq`,  `Microsoft.Azure.DocumentDb[.Core]`): Production-strength Azure CosmosDb Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RU costs, instrumented to the degree necessitated by Jet's production monitoring requirements.
 - `samples/Store` (in this repo): Example domain types reflecting examples of how one applies Equinox to a diverse set of stream-based models
 - `samples/TodoBackend` (in this repo): Standard https://todobackend.com compliant backend
-- `Equinox.Tool` (Nuget: `dotnet tool install Equinox.Tool -g`): Tool incorporating a benchmark scenario runner, facilitating running representative load tests composed of transactions in `samples/Store` and `samples/TodoBackend` against any nominated store; this allows perf tuning and measurement in terms of both latency and transaction charge aspects.
 
 ## Versioning
 
 ## About Versioning
 
-The repo is versioned based on [SemVer 2.0](https://semver.org/spec/v2.0.0.html) using the tiny-but-mighty [MinVer](https://github.com/adamralph/minver) from @adamralph. [See here](https://github.com/adamralph/minver#how-it-works) for more information on how it works.
+The repo is versioned based on [SemVer 2.0](https://semver.org/spec/v2.0.0.html) using the tiny-but-mighty [MinVer](https://github.com/adamralph/minver) from [@adamralph](https://github.com/adamralph). [See here](https://github.com/adamralph/minver#how-it-works) for more information on how it works.
 
 ## CONTRIBUTING
 
 Please raise GitHub issues for any questions so others can benefit from the discussion.
 
-We are getting very close to that point and are extremely excited by that. But we're not there yet; this is intentionally a soft launch.
-
-For now, the core focus of work here will be on converging the `cosmos` branch, which will bring changes, clarifications, simplifications and features, that all need to be integrated into the production systems built on it, before we can consider broader-based additive changes and/or significantly increasing the API surface area.
-
 The aim in the medium term (and the hope from the inception of this work) is to run Equinox as a proper Open Source project at the point where there is enough time for maintainers to do that properly.
 
-Unfortunately, in the interim, the barrier for contributions will unfortunately be inordinately high in the short term:
+We are getting very close to that point and are extremely excited by that. But we're not there yet; this is intentionally a soft launch.
 
-- bugfixes with good test coverage are always welcome - PRs yield MyGet-hosted NuGets and in general we'll seek to move them to NuGet prerelease and then NuGet release packages with relatively short timelines.
+Unfortunately, in the interim, the barrier for contributions will unfortunately be inordinately high:
+- bugfixes with good test coverage are always welcome - PRs yield [MyGet-hosted NuGets](https://www.myget.org/F/jet/api/v3/index.json); in general we'll seek to move them to NuGet prerelease and then NuGet release packages with relatively short timelines.
 - minor improvements / tweaks, subject to discussing in a GitHub issue first to see if it fits, but no promises at this time, even if the ideas are fantastic and necessary :sob:
 - tests, examples and scenarios are always welcome; Equinox is intended to address a very broad base of usage patterns; Please note that the emphasis will always be (in order)
   1. providing advice on how to achieve your aims without changing Equinox
   2. how to open up an appropriate extension point in Equinox
   3. (when all else fails), add to the complexity of the system by adding API surface area or logic.
-- we will likely punt on non-IO perf improvements until such point as Cosmos support is converged into `master`
-- Naming is hard; there is definitely room for improvement. There likely will be a set of controlled deprecations, switching to names, and then removing the old ones. However, PRs other than for discussion purposes probably don't make sense right now.
+- Naming is hard; there is definitely room for improvement. There may at some point be a set of controlled deprecations, switching to names, and then removing the old ones. However it should be noted that widespread renaming is not on the cards due to the number of downstream systems that would be affected.
 
 ## BUILDING
 
@@ -178,7 +174,6 @@ The CLI can drive the Store and TodoBackend samples in the `samples/Web` ASP.NET
     eqx run -t saveforlater -f 200 web
 
 ### run CosmosDb benchmark (when provisioned)
-
     $env:EQUINOX_COSMOS_CONNECTION="AccountEndpoint=https://....;AccountKey=....=;"
     $env:EQUINOX_COSMOS_DATABASE="equinox-test"
     $env:EQUINOX_COSMOS_COLLECTION="equinox-test"

build.proj

@@ -17,6 +17,7 @@
   <Target Name="Pack">
     <Exec Command="dotnet pack src/Equinox $(Cfg) $(PackOptions)" />
     <Exec Command="dotnet pack src/Equinox.Codec $(Cfg) $(PackOptions)" />
+    <Exec Command="dotnet pack src/Equinox.Cosmos $(Cfg) $(PackOptions)" />
     <Exec Command="dotnet pack src/Equinox.EventStore $(Cfg) $(PackOptions)" />
     <Exec Command="dotnet pack src/Equinox.MemoryStore $(Cfg) $(PackOptions)" />
     <Exec Command='dotnet publish tools/Equinox.Tool $(Cfg) -f net461 -o "$(RepoDir)/bin/equinox-tool/net461" ' />
@@ -27,10 +28,11 @@
   <Target Name="VSTest" Condition=" '$(IsOSX)' != 'true' " >
     <Exec Command="dotnet test tests/Equinox.MemoryStore.Integration $(Cfg) $(TestOptions)" />
     <Exec Command="dotnet test tests/Equinox.EventStore.Integration $(Cfg) $(TestOptions)" />
+    <Exec Command="dotnet test tests/Equinox.Cosmos.Integration $(Cfg) $(TestOptions)" />
     <Exec Command="dotnet test samples/Store/Integration $(Cfg) $(TestOptions)" />
     <Exec Command="dotnet test samples/Store/Domain.Tests $(Cfg) $(TestOptions)" />
   </Target>
 
   <Target Name="Build" DependsOnTargets="VSTest;Pack" />
 
-</Project>
+</Project>

build.ps1

@@ -2,6 +2,13 @@ param(
 	[string] $verbosity="m",
 	[Alias("s")][switch][bool] $skipStores=$false,
 	[Alias("se")][switch][bool] $skipEs=$skipStores,
+	[Alias("sc")][switch][bool] $skipCosmos=$skipStores,
+	[Alias("cs")][string] $cosmosServer=$env:EQUINOX_COSMOS_CONNECTION,
+	[Alias("cd")][string] $cosmosDatabase=$env:EQUINOX_COSMOS_DATABASE,
+	[Alias("cc")][string] $cosmosCollection=$env:EQUINOX_COSMOS_COLLECTION,
+	[Alias("scp")][switch][bool] $skipProvisionCosmos=$skipCosmos -or -not $cosmosServer -or -not $cosmosDatabase -or -not $cosmosCollection,
+	[Alias("ca")][switch][bool] $cosmosProvisionAux,
+	[Alias("scd")][switch][bool] $skipDeprovisionCosmos=$skipProvisionCosmos,
 	[string] $additionalMsBuildArgs="-t:Build"
 )
 
@@ -13,10 +20,36 @@ function warn ($msg) { Write-Host "$msg" -BackgroundColor DarkGreen }
 $env:EQUINOX_INTEGRATION_SKIP_EVENTSTORE=[string]$skipEs
 if ($skipEs) { warn "Skipping EventStore tests" }
 
+function cliCosmos($arghs) {
+	Write-Host "dotnet run tools/Equinox.Tool -- $arghs cosmos -s <REDACTED> -d $cosmosDatabase -c $cosmosCollection"
+	dotnet run -p tools/Equinox.Tool -f netcoreapp2.1 -- @arghs cosmos -s $cosmosServer -d $cosmosDatabase -c $cosmosCollection
+}
+
+if ($skipCosmos) {
+	warn "Skipping Cosmos tests as requested"
+} elseif ($skipProvisionCosmos) {
+	warn "Skipping Provisioning Cosmos"
+} else {
+	warn "Provisioning cosmos (without stored procedure)..."
+	# -P: inhibit creation of stored proc (everything in the repo should work without it due to auto-provisioning)
+	cliCosmos @("init", "-ru", "400", "-P")
+	$deprovisionCosmos=$true
+	if ($cosmosProvisionAux) {
+		warn "Provisioning cosmos aux collection for projector..."
+		cliCosmos @("initAux", "-ru", "400")
+	}
+}
+$env:EQUINOX_INTEGRATION_SKIP_COSMOS=[string]$skipCosmos
+
 warn "RUNNING: dotnet msbuild $args"
 . dotnet msbuild build.proj @args
 
 if( $LASTEXITCODE -ne 0) {
 	warn "open msbuild.log for error info or rebuild with -v n/d/diag for more detail, or open msbuild.binlog using https://github.com/KirillOsenkov/MSBuildStructuredLog/releases/download/v2.0.40/MSBuildStructuredLogSetup.exe"
 	exit $LASTEXITCODE
 }
+
+if (-not $skipDeprovisionCosmos) {
+	warn "Deprovisioning Cosmos"
+	throw "Deprovisioning step not implemented yet - please deallocate your resources using the Azure Portal"
+}

samples/Infrastructure/Infrastructure.fsproj

@@ -11,10 +11,12 @@
   <ItemGroup>
     <Compile Include="Storage.fs" />
     <Compile Include="Services.fs" />
+    <Compile Include="Log.fs" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\src\Equinox\Equinox.fsproj" />
+    <ProjectReference Include="..\..\src\Equinox.Cosmos\Equinox.Cosmos.fsproj" />
     <ProjectReference Include="..\..\src\Equinox.EventStore\Equinox.EventStore.fsproj" />
     <ProjectReference Include="..\..\src\Equinox.MemoryStore\Equinox.MemoryStore.fsproj" />
     <ProjectReference Include="..\Store\Backend\Backend.fsproj" />