Add ability to seed the database using static file-based resources

src/api/Synapse.Api.Application/Configuration/ApiServerOptions.cs

@@ -43,4 +43,9 @@ public ApiServerOptions()
     /// </summary>
     public virtual ApiServerCloudEventOptions CloudEvents { get; set; } = new();
 
+    /// <summary>
+    /// Gets/sets the options used to configure the seeding, if any, of Synapse's database
+    /// </summary>
+    public virtual DatabaseSeedingOptions Seeding { get; set; } = new();
+
 }

src/api/Synapse.Api.Application/Configuration/DatabaseSeedingOptions.cs

@@ -0,0 +1,78 @@
+// Copyright © 2024-Present The Synapse Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"),
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace Synapse.Api.Application.Configuration;
+
+/// <summary>
+/// Represents the options used to configure the seeding, if any, of Synapse's database
+/// </summary>
+public class DatabaseSeedingOptions
+{
+
+    /// <summary>
+    /// Gets the path to the directory from which to load the static resources used to seed the database
+    /// </summary>
+    public static readonly string DefaultDirectory = Path.Combine(AppContext.BaseDirectory, "data", "seed");
+    /// <summary>
+    /// Gets the default GLOB pattern used to match the static resource files to use to seed the database
+    /// </summary>
+    public const string DefaultFilePattern = "*.*";
+
+    /// <summary>
+    /// Initializes a new <see cref="DatabaseSeedingOptions"/>
+    /// </summary>
+    public DatabaseSeedingOptions()
+    {
+        var env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.Reset);
+        if (!string.IsNullOrWhiteSpace(env))
+        {
+            if (!bool.TryParse(env, out var reset)) throw new Exception($"Failed to parse the value specified as '{SynapseDefaults.EnvironmentVariables.Database.Seeding.Reset}' environment variable into a boolean");
+            this.Reset = reset;
+        }
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.Directory);
+        if (!string.IsNullOrWhiteSpace(env))
+        {
+            this.Directory = env;
+            if (!System.IO.Directory.Exists(this.Directory)) System.IO.Directory.CreateDirectory(this.Directory);
+        }
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.Overwrite);
+        if (!string.IsNullOrWhiteSpace(env))
+        {
+            if (!bool.TryParse(env, out var overwrite)) throw new Exception($"Failed to parse the value specified as '{SynapseDefaults.EnvironmentVariables.Database.Seeding.Overwrite}' environment variable into a boolean");
+            this.Overwrite = overwrite;
+        }
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Database.Seeding.FilePattern);
+        if (!string.IsNullOrWhiteSpace(env)) this.FilePattern = env;
+    }
+
+    /// <summary>
+    /// Gets/sets a boolean indicating whether or not to reset the database upon starting up the API server
+    /// </summary>
+    public virtual bool Reset { get; set; }
+
+    /// <summary>
+    /// Gets/sets the directory from which to load the static resources used to seed the database
+    /// </summary>
+    public virtual string Directory { get; set; } = DefaultDirectory;
+
+    /// <summary>
+    /// Gets/sets a boolean indicating whether or not to overwrite existing resources
+    /// </summary>
+    public virtual bool Overwrite { get; set; }
+
+    /// <summary>
+    /// Gets/sets the GLOB pattern used to match the static resource files to use to seed the database
+    /// </summary>
+    public virtual string FilePattern { get; set; } = DefaultFilePattern;
+
+}

src/api/Synapse.Api.Application/Services/WorkflowDatabaseInitializer.cs

@@ -0,0 +1,154 @@
+// Copyright © 2024-Present The Synapse Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"),
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using ServerlessWorkflow.Sdk.IO;
+using ServerlessWorkflow.Sdk.Models;
+using Synapse.Api.Application.Configuration;
+using Synapse.Resources;
+using System.Diagnostics;
+
+namespace Synapse.Api.Application.Services;
+
+/// <summary>
+/// Defines the fundamentals of a service used to initialize the Synapse workflow database
+/// </summary>
+/// <param name="serviceProvider">The current <see cref="IServiceProvider"/></param>
+/// <param name="logger">The service used to perform logging</param>
+/// <param name="workflowDefinitionReader">The service used to read <see cref="WorkflowDefinition"/>s</param>
+/// <param name="options">The service used to access the current <see cref="ApiServerOptions"/></param>
+public class WorkflowDatabaseInitializer(IServiceProvider serviceProvider, ILogger<WorkflowDatabaseInitializer> logger, IWorkflowDefinitionReader workflowDefinitionReader, IOptions<ApiServerOptions> options)
+    : IHostedService
+{
+
+    /// <summary>
+    /// Gets the current <see cref="IServiceProvider"/>
+    /// </summary>
+    protected IServiceProvider ServiceProvider { get; } = serviceProvider;
+
+    /// <summary>
+    /// Gets the service used to perform logging
+    /// </summary>
+    protected ILogger Logger { get; } = logger;
+
+    /// <summary>
+    /// Gets the service used to read <see cref="WorkflowDefinition"/>s
+    /// </summary>
+    protected IWorkflowDefinitionReader WorkflowDefinitionReader { get; } = workflowDefinitionReader; 
+
+    /// <summary>
+    /// Gets the current <see cref="ApiServerOptions"/>
+    /// </summary>
+    protected ApiServerOptions Options { get; } = options.Value;
+
+    /// <inheritdoc/>
+    public virtual async Task StartAsync(CancellationToken cancellationToken)
+    {
+        using var scope = this.ServiceProvider.CreateScope();
+        var resources = scope.ServiceProvider.GetRequiredService<IResourceRepository>();
+        var stopwatch = new Stopwatch();
+        if (this.Options.Seeding.Reset)
+        {
+            this.Logger.LogInformation("Starting resetting database...");
+            stopwatch.Start();
+            await foreach (var correlation in resources.GetAllAsync<Correlation>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Correlation>(correlation.GetName(), correlation.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
+            await foreach (var correlator in resources.GetAllAsync<Correlator>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Correlator>(correlator.GetName(), correlator.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
+            await foreach (var ns in resources.GetAllAsync<Namespace>(cancellationToken: cancellationToken).Where(ns => ns.GetName() != Namespace.DefaultNamespaceName)) await resources.RemoveAsync<Namespace>(ns.GetName(), ns.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
+            await foreach (var @operator in resources.GetAllAsync<Operator>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Operator>(@operator.GetName(), @operator.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
+            await foreach (var serviceAccount in resources.GetAllAsync<ServiceAccount>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<ServiceAccount>(serviceAccount.GetName(), serviceAccount.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
+            await foreach (var workflow in resources.GetAllAsync<Workflow>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<Workflow>(workflow.GetName(), workflow.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
+            await foreach (var workflowInstance in resources.GetAllAsync<WorkflowInstance>(cancellationToken: cancellationToken).ConfigureAwait(false)) await resources.RemoveAsync<WorkflowInstance>(workflowInstance.GetName(), workflowInstance.GetNamespace(), false, cancellationToken).ConfigureAwait(false);
+            stopwatch.Stop();
+            this.Logger.LogInformation("Database reset completed in {ms} milliseconds", stopwatch.Elapsed.TotalMilliseconds);
+        }
+        var directory = new DirectoryInfo(this.Options.Seeding.Directory);
+        if (!directory.Exists)
+        {
+            this.Logger.LogWarning("The directory '{directory}' does not exist or cannot be found. Skipping static resource import", directory.FullName);
+            return;
+        }
+        this.Logger.LogInformation("Starting importing static resources from directory '{directory}'...", directory.FullName);
+        var files = directory.GetFiles(this.Options.Seeding.FilePattern, SearchOption.AllDirectories).Where(f => f.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) || f.FullName.EndsWith(".yml", StringComparison.OrdinalIgnoreCase) || f.FullName.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase));
+        if (!files.Any())
+        {
+            this.Logger.LogWarning("No static resource files matching search pattern '{pattern}' found in directory '{directory}'. Skipping import.", this.Options.Seeding.FilePattern, directory.FullName);
+            return;
+        }
+        var count = 0;
+        stopwatch.Restart();
+        foreach (var file in files)
+        {
+            try
+            {
+                using var stream = file.OpenRead();
+                var workflowDefinition = await this.WorkflowDefinitionReader.ReadAsync(stream, new() { BaseDirectory = file.Directory!.FullName }, cancellationToken).ConfigureAwait(false);
+                var workflow = await resources.GetAsync<Workflow>(workflowDefinition.Document.Name, workflowDefinition.Document.Namespace, cancellationToken).ConfigureAwait(false);
+                if (workflow == null)
+                {
+                    workflow = new()
+                    {
+                        Metadata = new()
+                        {
+                            Namespace = workflowDefinition.Document.Namespace,
+                            Name = workflowDefinition.Document.Name
+                        },
+                        Spec = new()
+                        {
+                            Versions = [workflowDefinition]
+                        }
+                    };
+                    if (await resources.GetAsync<Namespace>(workflow.GetNamespace()!, cancellationToken: cancellationToken).ConfigureAwait(false) == null)
+                    {
+                        await resources.AddAsync(new Namespace() { Metadata = new() { Name = workflow.GetNamespace()! } }, false, cancellationToken).ConfigureAwait(false);
+                        this.Logger.LogInformation("Successfully created namespace '{namespace}'", workflow.GetNamespace());
+                    }
+                    await resources.AddAsync(workflow, false, cancellationToken).ConfigureAwait(false);
+                }
+                else
+                {
+                    var version = workflow.Spec.Versions.Get(workflowDefinition.Document.Version);
+                    if (version != null)
+                    {
+                        if (this.Options.Seeding.Overwrite)
+                        {
+                            workflow.Spec.Versions.Remove(version);
+                            workflow.Spec.Versions.Add(workflowDefinition);
+                            await resources.ReplaceAsync(workflow, false, cancellationToken).ConfigureAwait(false);
+                        }
+                        else
+                        {
+                            this.Logger.LogInformation("Skipped the import of workflow '{workflow}' from file '{file}' because it already exists", $"{workflowDefinition.Document.Name}.{workflowDefinition.Document.Namespace}:{workflowDefinition.Document.Version}", file.FullName);
+                            continue;
+                        }
+                    }
+                }
+                this.Logger.LogInformation("Successfully imported workflow '{workflow}' from file '{file}'", $"{workflowDefinition.Document.Name}.{workflowDefinition.Document.Namespace}:{workflowDefinition.Document.Version}", file.FullName);
+                count++;
+            }
+            catch(Exception ex)
+            {
+                this.Logger.LogError("An error occurred while reading a workflow definition from file '{file}': {ex}", file.FullName, ex);
+                continue;
+            }
+        }
+        stopwatch.Stop();
+        this.Logger.LogInformation("Completed importing {count} static resources in {ms} milliseconds", count, stopwatch.Elapsed.TotalMilliseconds);
+    }
+
+    /// <inheritdoc/>
+    public virtual Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+
+}

src/api/Synapse.Api.Application/Synapse.Api.Application.csproj

@@ -44,7 +44,7 @@
   <ItemGroup>
     <PackageReference Include="IdentityServer4" Version="4.1.2" NoWarn="NU1902" />
     <PackageReference Include="IdentityServer4.Storage" Version="4.1.2" NoWarn="NU1902" />
-    <PackageReference Include="Polly" Version="8.4.1" />
+    <PackageReference Include="Polly" Version="8.4.2" />
   </ItemGroup>
 
   <ItemGroup>

src/api/Synapse.Api.Client.Http/Synapse.Api.Client.Http.csproj

@@ -43,7 +43,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.8" />
-	<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.15" />
+	<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.16" />
     <PackageReference Include="System.Reactive" Version="6.0.1" />
   </ItemGroup>
 

src/api/Synapse.Api.Http/Synapse.Api.Http.csproj

@@ -45,7 +45,7 @@
   <ItemGroup>
     <PackageReference Include="Neuroglia.Mediation.AspNetCore" Version="4.15.6" />
     <PackageReference Include="Neuroglia.Security.AspNetCore" Version="4.15.6" />
-	<PackageReference Include="Swashbuckle.AspNetCore.SwaggerGen" Version="6.7.3" />
+	<PackageReference Include="Swashbuckle.AspNetCore.SwaggerGen" Version="6.8.1" />
   </ItemGroup>
 
   <ItemGroup>

src/api/Synapse.Api.Server/Program.cs

@@ -25,6 +25,7 @@
 builder.Services.AddSynapse(builder.Configuration);
 builder.Services.AddSynapseApi();
 builder.Services.AddSynapseHttpApi(authority);
+builder.Services.AddHostedService<WorkflowDatabaseInitializer>();
 
 var authentication = builder.Services.AddAuthentication(FallbackPolicySchemeDefaults.AuthenticationScheme);
 authentication.AddScheme<StaticBearerAuthenticationOptions, StaticBearerAuthenticationHandler>(StaticBearerDefaults.AuthenticationScheme, options =>

src/api/Synapse.Api.Server/Synapse.Api.Server.csproj

@@ -34,7 +34,7 @@
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="8.0.8" />
     <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="8.0.8" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
-	<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="6.7.3" />
+	<PackageReference Include="Swashbuckle.AspNetCore.SwaggerUI" Version="6.8.1" />
   </ItemGroup>
 
   <ItemGroup>

src/cli/Synapse.Cli/Synapse.Cli.csproj

@@ -33,7 +33,7 @@
 	<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
 	<PackageReference Include="moment.net" Version="1.3.4" />
 	<PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0" />
-	<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.15" />
+	<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.16" />
 	<PackageReference Include="Spectre.Console" Version="0.49.1" />
     <PackageReference Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>

src/core/Synapse.Core.Infrastructure/Extensions/IServiceCollectionExtensions.cs

@@ -45,8 +45,6 @@ public static IServiceCollection AddSynapse(this IServiceCollection services, IC
     {
         services.AddHttpClient();
         services.AddSerialization();
-        services.AddJsonSerializer();
-        services.AddYamlDotNetSerializer();
         services.AddMediator();
         services.AddScoped<IUserInfoProvider, UserInfoProvider>();
         services.AddServerlessWorkflowIO();

src/core/Synapse.Core.Infrastructure/Synapse.Core.Infrastructure.csproj

@@ -44,13 +44,13 @@
 
   <ItemGroup>
 	<PackageReference Include="IdentityModel" Version="7.0.0" />
-	<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.0.2" />
+	<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.1.1" />
 	<PackageReference Include="Neuroglia.Data.Expressions.Abstractions" Version="4.15.6" />
 	<PackageReference Include="Neuroglia.Data.Infrastructure.Redis" Version="4.15.6" />
 	<PackageReference Include="Neuroglia.Data.Infrastructure.ResourceOriented.Redis" Version="4.15.6" />
 	<PackageReference Include="Neuroglia.Mediation" Version="4.15.6" />
 	<PackageReference Include="Neuroglia.Plugins" Version="4.15.6" />
-	<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.15" />
+	<PackageReference Include="ServerlessWorkflow.Sdk.IO" Version="1.0.0-alpha2.16" />
   </ItemGroup>
 
   <ItemGroup>

src/core/Synapse.Core/Synapse.Core.csproj

@@ -63,11 +63,11 @@
 
   <ItemGroup>
 	<PackageReference Include="Docker.DotNet" Version="3.125.15" />
-	<PackageReference Include="KubernetesClient" Version="14.0.12" />
+	<PackageReference Include="KubernetesClient" Version="15.0.1" />
 	<PackageReference Include="Neuroglia.Data.Infrastructure.ResourceOriented" Version="4.15.6" />
 	<PackageReference Include="Neuroglia.Eventing.CloudEvents" Version="4.15.6" />
     <PackageReference Include="Semver" Version="2.3.0" />
-    <PackageReference Include="ServerlessWorkflow.Sdk" Version="1.0.0-alpha2.15" />
+    <PackageReference Include="ServerlessWorkflow.Sdk" Version="1.0.0-alpha2.16" />
   </ItemGroup>
 
 </Project>