refactored sql storage

Author: rofr

docs/storage/sql-storage/index.md

@@ -5,67 +5,57 @@ layout: submenu
 ## {{page.title}}
 
 
-By default, journal files and snapshots are written to the file system. Storage is configurable and supports writing the journal to a relational database taking advantage of existing infrastructure and operations. Also, the journal can be queried/manipulated using regular SQL.
+Storage is configurable. By default, journal files and snapshots are written to the file system. Using Sql Storage, The journal can be stored in a relational database, allowing you to take advantage of existing infrastructure and operations. Also, the journal can be queried/manipulated using regular SQL.
 
-The first step is to set the storage type:
+Sql Storage uses the DbProviderFactories of NET.
+The built-in providers are MsSqlProvider and OleDbProvider.
+
+Sql storage is flexible, you can supply custom statements for initializing, reading and writing entries.
+
+The default table has these columns:
+
+Name | Type | Description
+---- | ---- | -----
+Id | ulong | The unique sequential id of the journal entry
+Created | DateTime | When the command was executed
+Type | String | The type name of the command executed
+Data | Binary or string | The serialized command  
+
+To enable sql storage set the storage type for journaling to Sql:
 
 ```csharp
 var config = new EngineConfiguration();
 config.JournalStorage = StorageType.Sql;
+var engine = Engine.For<MyDb>(config);
+```
+
+The default settings assume a connection string entry in the application configuration file named 'origo':
+
+```xml
+<connectionString name="origo"
+  connectionString="Data Source=.;Initial Catalog=freedb;Integrated Security=True"
+  providerName="System.Data.SqlClient"/>
+```
+
+The providerName must be one the following supported providers or a custom provider. See Custom Providers below.
+
+* System.Data.SqlClient
+* System.Data.OleDbClient
+* System.Data.OdbcClient
+
+ Here are the default settings, which can be assigned new values. The `ConnectionString` property can be assigned either a connection string name in the application configuration file or an actual connection string.
+
+```csharp
+config.SqlSettings.TableName = "OrigoJournal";
+config.ConnectionString = "origo";
+config.SqlSettings.ProviderName = "System.Data.SqlClient";
+var engine = Engine.For<MyDb>(config);
+```
+
+## Register a custom provider
+Providers derive from SqlProvider and supply the vendor specific sql statements for reading and writing the journal. Custom providers need to be registered using the name and a constructor function taking a SqlSettings as input:
+```csharp
+SqlProvider.Register("MyProviderName", settings => new MyProvider(settings));
 ```
 
-## The happy path
-
-1. Set the JournalStorage to StorageType.Sql
-2. S
-
-## Relevant types
-* SqlSettings
-* SqlProvider
-* MsSqlProvider
-* SqlStorage
-
-The provider creates a single table named CommandJournal with the following columns:
-
-Name | Description
------|------------
-Id | The unique sequential id of the command
-CommandName | The type name of the command executed
-Created | The point in time when the command was executed
-Entry | The serialized `JournalEntry<Command>` object
-
-## Supported SQL databases
-Release 0.1.0 was tested on Sql Server 2008 R2 developer edition and should work with 2005, 2008, 2012 and SqlCE.
-
-## Configuring SqlStorage
-1. Add OrigoDb.Modules.SqlStorage to your project. Grab it on the downloads page or using nuget.
-2. Add a connectionstring to your app config file pointing to an existing database
-{% highlight xml %}
-<connectionStrings>
-  <add name="connectionName"
-    connectionString="Data Source=.;Initial Catalog=freedb;Integrated Security=True"
-    providerName="System.Data.SqlClient" />
-</connectionStrings>
-{% endhighlight %}
-
-3. Pass an instance of `SqlEngineConfiguration` when creating or loading your database
-
-{% highlight csharp %}
-var config = new SqlEngineConfiguration("connectionName");
-config.SnapshotLocation = @"c:\\temp";
-var engine = Engine.LoadOrCreate<MyModel>(config);
-{% endhighlight %}
-
-Alternatively, you can set `Location` to a connection string directly. If so, you must also set the `LocationType` and `ProviderName` properties:
-
-{% highlight csharp %}
-var config = new SqlEngineConfiguration();
-config.Location = "Data Source=.;Initial Catalog=freedb;Integrated Security=True";
-config.LocationType = LocationType.ConnectionString;
-config.ProviderName = "System.Data.SqlClient";
-config.SnapshotLocation = @"c:\\temp";
-var engine = Engine.LoadOrCreate<MyModel>(config);
-{% endhighlight %}
-
-## Converting existing journal
-Use the StorageUtility to copy an existing journal from file to sql or sql to file.
+Additionally, the provider name must be recognized by DbProviderFactories, see MSDN documentation.

src/OrigoDB.Core.UnitTests/SqlStorageTest.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Configuration;
+using System.Data;
+using System.Data.Common;
 using System.Diagnostics;
 using System.IO;
 using System.Runtime.Serialization.Formatters.Binary;
@@ -13,23 +15,40 @@ namespace OrigoDB.Core.Test
     [TestFixture]
     public class SqlStorageTest
     {
+
+        [Test]
+        public void DisplayProviders()
+        {
+            var table = DbProviderFactories.GetFactoryClasses();
+            foreach (DataRow row in table.Rows)
+            {
+                foreach (DataColumn column in table.Columns)
+                {
+                    Console.WriteLine(column.ColumnName + ":" + row[column]);
+                }
+                Console.WriteLine("-------------------------------------------------");
+            }
+        }
+
         [Test, Ignore]
-        public void MsSqlProviderSmokeTest()
+        public void MsSqlCommandStoreWriteReadEntries()
         {
-            var settings = new SqlSettings();
+            var config = new EngineConfiguration();
+            var settings = config.SqlSettings;
             settings.ConnectionString = "Data Source=.;Initial Catalog=fish;Integrated Security=True";
             settings.ProviderName = "System.Data.SqlClient";
-            var provider = SqlProvider.Create(settings);
-            provider.Initialize();
+            settings.TableName = "[test-" + Guid.NewGuid() + "]";
+            var commandStore = new SqlCommandStore(config);
+            commandStore.Initialize();
             var formatter = new BinaryFormatter();
-            var writer = new SqlJournalWriter(formatter, provider);
+            var writer = new SqlJournalWriter(formatter, commandStore);
             writer.Write(JournalEntry.Create(1UL, DateTime.Now, new ModelCreated(typeof (TestModel))));
             writer.Write(JournalEntry.Create(2UL, DateTime.Now.AddSeconds(1), new AppendNumberCommand(42)));
             writer.Write(JournalEntry.Create(3UL, DateTime.Now.AddSeconds(2), new AppendNumberCommand(64)));
 
-            foreach (var entry in provider.ReadJournalEntries(1, bytes => formatter.FromByteArray<object>(bytes)))
+            foreach (var entry in commandStore.GetJournalEntriesFrom(1))
             {
-                Trace.WriteLine(entry.GetItem());
+                Trace.WriteLine(entry);
             }
             writer.Dispose();
         }
@@ -39,7 +58,7 @@ public void MsSqlProviderIntegrationTest()
             var config = new EngineConfiguration();
             config.JournalStorage = StorageType.Sql;
             config.SqlSettings.ConnectionString = "Data Source=.;Initial Catalog=fish;Integrated Security=True;";
-            config.SqlSettings.ProviderName = MsSqlProvider.Name;
+            config.SqlSettings.ProviderName = "System.Data.SqlClient";
             var engine = Engine.For<TestModel>(config);
             int initial = engine.Execute(new DelegateQuery<TestModel, int>(m => m.CommandsExecuted));
             engine.Execute(new TestCommandWithoutResult());

src/OrigoDB.Core/Configuration/SqlSettings.cs

@@ -1,11 +1,12 @@
 using System.Configuration;
+using OrigoDB.Core.Storage.Sql;
 
 namespace OrigoDB.Core
 {
 
     /// <summary>
     /// Configuration settings when using a backing sql store.
-    /// Exposed via EngineConfiguration.SqlSettings property.
+    /// Exposed via EngineConfiguration.Sql property.
     /// </summary>
     public class SqlSettings
     {
@@ -31,6 +32,11 @@ public class SqlSettings
         /// </summary>
         public string ConnectionString { get; set; }
 
+        /// <summary>
+        /// Custom SQL Statements, leave null to use the default, provider specific statements
+        /// </summary>
+        public SqlStatements Statements { get; set; }
+
         /// <summary>
         /// Lookup ConnectionStringSetting in application configuration file using ConnectionString as key.
         /// If it exists, assign ConnectionString and ProviderName properties.

src/OrigoDB.Core/OrigoDB.Core.csproj

@@ -127,10 +127,11 @@
     <Compile Include="Storage\Rollover\MaxBytesRolloverStrategy.cs" />
     <Compile Include="Storage\Rollover\MaxEntriesRolloverStrategy.cs" />
     <Compile Include="Storage\Rollover\ScheduledRolloverStrategy.cs" />
+    <Compile Include="Storage\Sql\OleDbStatements.cs" />
     <Compile Include="Storage\Sql\SqlCommandStore.cs" />
     <Compile Include="Storage\Sql\SqlJournalWriter.cs" />
-    <Compile Include="Storage\Sql\MsSqlProvider.cs" />
-    <Compile Include="Storage\Sql\SqlProvider.cs" />
+    <Compile Include="Storage\Sql\MsSqlStatements.cs" />
+    <Compile Include="Storage\Sql\SqlStatements.cs" />
     <Compile Include="Utilities\Utils.Converters.cs" />
     <Compile Include="Storage\Journaling\JournalAppender.cs" />
     <Compile Include="Storage\Journaling\ModelCreated.cs" />

src/OrigoDB.Core/Storage/Journaling/JournalEntry.cs

@@ -58,6 +58,11 @@ internal static JournalEntry Create(ulong id, DateTime created, object item)
             if (item is Command) return new JournalEntry<Command>(id, (Command) item, created);
             throw new ArgumentOutOfRangeException("unrecognized journal entry item :" + item);
         }
+
+        public override string ToString()
+        {
+            return String.Format("[{0} - {1} - {2}", Id, Created, GetItem().GetType());
+        }
     }
 
 

src/OrigoDB.Core/Storage/Sql/MsSqlProvider.cs

@@ -0,0 +1,27 @@
+using System.Text;
+
+namespace OrigoDB.Core.Storage.Sql
+{
+    public class MsSqlStatements : SqlStatements
+    {
+        public MsSqlStatements()
+        {
+            ReadEntries = "SELECT Id, Created, Data FROM {0} WHERE Id >= @id ORDER BY Id";
+            InitStore = BuildInitStatement();
+            AppendEntry = "INSERT INTO {0} VALUES (@Id, @Created, @Type, @Data);";
+        }
+
+        private string BuildInitStatement()
+        {
+            var sb = new StringBuilder();
+            sb.Append("IF NOT EXISTS ( SELECT * FROM sys.tables WHERE name = '{0}')\n");
+            sb.Append("CREATE TABLE {0}\n");
+            sb.AppendLine("(");
+            sb.AppendLine("   Id bigint not null primary key,");
+            sb.AppendLine("   Created datetime not null,");
+            sb.AppendLine("   Type varchar(1024) not null,");
+            sb.AppendLine("   Data varbinary(max) not null);");
+            return sb.ToString();
+        }
+    }
+}

src/OrigoDB.Core/Storage/Sql/MsSqlStatements.cs

@@ -0,0 +1,26 @@
+using System.Text;
+
+namespace OrigoDB.Core.Storage.Sql
+{
+    public class OleDbStatements : SqlStatements
+    {
+        public OleDbStatements()
+        {
+            ReadEntries = "SELECT Id, Created, Data FROM {0} WHERE Id >= ? ORDER BY ID";
+            InitStore = BuildInitStatement();
+            AppendEntry = "INSERT INTO {0} VALUES (?,?,?,?);";
+        }
+        private string BuildInitStatement()
+        {
+            var sb = new StringBuilder();
+            sb.Append("IF NOT EXISTS ( SELECT * FROM INFORMATION.TABLES WHERE TABLE_NAME = '{0}')\n");
+            sb.Append("CREATE TABLE {0}\n");
+            sb.AppendLine("(");
+            sb.AppendLine("   Id bigint not null primary key,");
+            sb.AppendLine("   Created datetime not null,");
+            sb.AppendLine("   Type varchar(1024) not null,");
+            sb.AppendLine("   Data varbinary(max) not null);");
+            return sb.ToString();
+        }
+    }
+}