Merge branch 'v5.4' of github.com:ravendb/ravendb into v6.0

# Conflicts:
#	src/Raven.Server/Documents/Indexes/IndexStorage.cs
#	src/Raven.Studio/typescript/common/appUrl.ts
#	src/Raven.Studio/typescript/viewmodels/resources/widgets/ongoingTasksWidget.ts

src/Raven.Client/Documents/Operations/ETL/SQL/SqlProviderParser.cs

@@ -9,6 +9,7 @@ public static SqlProvider GetSupportedProvider(string factoryName)
             switch (factoryName)
             {
                 case "System.Data.SqlClient":
+                case "Microsoft.Data.SqlClient":
                     return SqlProvider.SqlClient;
                 case "Npgsql":
                     return SqlProvider.Npgsql;

src/Raven.Server/Documents/ETL/Providers/SQL/RelationalWriters/DbProviderFactories.cs

@@ -1,6 +1,6 @@
 using System;
 using System.Data.Common;
-using System.Data.SqlClient;
+using Microsoft.Data.SqlClient;
 using Npgsql;
 using Oracle.ManagedDataAccess.Client;
 using Raven.Client.Documents.Operations.ETL.SQL;

src/Raven.Server/Documents/ETL/Providers/SQL/RelationalWriters/RelationalDatabaseWriter.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Data;
 using System.Data.Common;
-using System.Data.SqlClient;
+using Microsoft.Data.SqlClient;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -50,6 +50,16 @@ public RelationalDatabaseWriter(SqlEtl etl, DocumentDatabase database)
             _commandBuilder = _providerFactory.InitializeCommandBuilder();
             _connection = _providerFactory.CreateConnection();
             var connectionString = etl.Configuration.Connection.ConnectionString;
+
+            if (_etl.Configuration.Connection.FactoryName == "System.Data.SqlClient" &&
+                SqlConnectionStringParser.GetConnectionStringValue(connectionString, new string[] { "Encrypt" }) == null)
+            {
+                // We want to ensure compatibility between deprecated System.Data.SqlClient (where Encrypt parameter defaults to false)
+                // and Microsoft.Data.SqlClient which defaults it to SqlConnectionEncryptOption.Mandatory. If no option is provided,
+                // we set it to SqlConnectionEncryptOption.Optional (equivalent of false).
+                connectionString = SqlConnectionStringUtil.GetConnectionStringWithOptionalEncrypt(connectionString);
+            }
+
             _connection.ConnectionString = connectionString;
 
             OpenConnection(database, etl.Configuration.Name, etl.Configuration.ConnectionStringName);

src/Raven.Server/Documents/ETL/Providers/SQL/RelationalWriters/RelationalDatabaseWriterBase.cs

@@ -9,6 +9,7 @@ public class RelationalDatabaseWriterBase
         private static readonly string[] SqlServerFactoryNames =
         {
             "System.Data.SqlClient",
+            "Microsoft.Data.SqlClient",
             "System.Data.SqlServerCe.4.0",
             "MySql.Data.MySqlClient",
             "System.Data.SqlServerCe.3.5"

src/Raven.Server/Documents/ETL/Providers/SQL/RelationalWriters/RelationalDatabaseWriterSimulator.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Data.Common;
-using System.Data.SqlClient;
+using Microsoft.Data.SqlClient;
 using System.Linq;
 using System.Text;
 using System.Threading;

src/Raven.Server/Documents/ETL/Providers/SQL/RelationalWriters/SqlConnectionStringUtil.cs

@@ -0,0 +1,16 @@
+namespace Raven.Server.Documents.ETL.Providers.SQL.RelationalWriters;
+
+public static class SqlConnectionStringUtil
+{
+    public static string GetConnectionStringWithOptionalEncrypt(string connectionString)
+    {
+        connectionString = connectionString.Trim();
+
+        if (connectionString.EndsWith(';') == false)
+            connectionString += ";";
+
+        connectionString += "Encrypt=Optional";
+
+        return connectionString;
+    }
+}

src/Raven.Server/Documents/Indexes/Index.cs

@@ -3184,7 +3184,7 @@ internal HashSet<string> GetReferencedCollectionNames()
         {
             // multiple maps can reference the same collections, we wants to return distinct names only
             return GetReferencedCollections()?
-                .SelectMany(p =>  p.Value?.Select(z => z.Name))
+                .SelectMany(p => p.Value?.Select(z => z.Name))
                 .ToHashSet();
         }
 
@@ -4231,7 +4231,8 @@ private int MinimumSizeForCalculateIndexEtagLength(QueryMetadata q)
             var length = sizeof(long) * 4 * Collections.Count + // last document etag, last tombstone etag and last mapped etags per collection
                          sizeof(int) + // definition hash
                          1 + // isStale
-                         1; // index state
+                         1 + // index state
+                         sizeof(long); // created timestamp (binary)
 
             if (q == null)
                 return length;
@@ -4273,6 +4274,8 @@ private unsafe void CalculateIndexEtagInternal(byte* indexEtagBytes, bool isStal
             *indexEtagBytes = isStale ? (byte)0 : (byte)1;
             indexEtagBytes += sizeof(byte);
             *indexEtagBytes = (byte)indexState;
+            indexEtagBytes += sizeof(byte);
+            *(long*)indexEtagBytes = _indexStorage.CreatedTimestampAsBinary;
         }
 
         public long GetIndexEtag(QueryOperationContext context, QueryMetadata q)
@@ -4324,7 +4327,7 @@ public Dictionary<TombstoneDeletionBlockageSource, HashSet<string>> GetDisabledS
         {
             var dict = new Dictionary<TombstoneDeletionBlockageSource, HashSet<string>>();
 
-            if (Status is not (IndexRunningStatus.Disabled or IndexRunningStatus.Paused)) 
+            if (Status is not (IndexRunningStatus.Disabled or IndexRunningStatus.Paused))
                 return dict;
 
             var source = new TombstoneDeletionBlockageSource(ITombstoneAware.TombstoneDeletionBlockerType.Index, Name);

src/Raven.Server/Documents/Indexes/IndexStorage.cs

@@ -72,6 +72,8 @@ private Constants()
 
         public readonly CompareExchangeReferences ReferencesForCompareExchange;
 
+        public long CreatedTimestampAsBinary;
+
         public IndexStorage(Index index, TransactionContextPool contextPool, DocumentDatabase database)
         {
             _index = index;
@@ -123,12 +125,15 @@ private unsafe void CreateSchema(DocumentDatabase documentDatabase)
                 using (Slice.External(context.Allocator, (byte*)&sourceTypeInt, sizeof(int), out Slice tmpSlice))
                     statsTree.Add(IndexSchema.SourceTypeSlice, tmpSlice);
 
-                if (statsTree.Read(IndexSchema.CreatedTimestampSlice) == null)
+                var createdTimestampResult = statsTree.Read(IndexSchema.CreatedTimestampSlice);
+                if (createdTimestampResult == null)
                 {
-                    var binaryDate = SystemTime.UtcNow.ToBinary();
+                    var binaryDate = CreatedTimestampAsBinary = SystemTime.UtcNow.ToBinary();
                     using (Slice.External(context.Allocator, (byte*)&binaryDate, sizeof(long), out Slice tmpSlice))
                         statsTree.Add(IndexSchema.CreatedTimestampSlice, tmpSlice);
                 }
+                else
+                    CreatedTimestampAsBinary = createdTimestampResult.Reader.ReadLittleEndianInt64();
 
                 using (Slice.From(context.Allocator, documentDatabase.DbBase64Id, out var dbId))
                     statsTree.Add(IndexSchema.DatabaseIdSlice, dbId);
@@ -180,11 +185,11 @@ void PersistSearchEngine(Tree configurationTree)
                 {
                     string configurationKey = nameof(SearchEngineType);
                     string configurationName = _index.Type.IsAuto() ? RavenConfiguration.GetKey(x => x.Indexing.AutoIndexingEngineType) : RavenConfiguration.GetKey(x => x.Indexing.StaticIndexingEngineType);
-                   
+
                     SearchEngineType defaultEngineType =
                         _index.Type.IsAuto() ? _index.Configuration.AutoIndexingEngineType : _index.Configuration.StaticIndexingEngineType;
-                    
-                    if(defaultEngineType == SearchEngineType.None)
+
+                    if (defaultEngineType == SearchEngineType.None)
                         throw new InvalidDataException($"Default search engine is {SearchEngineType.None}. Please set {configurationName}.");
                     var result = configurationTree.Read(configurationKey);
                     if (result != null)
@@ -402,7 +407,7 @@ public IndexStats ReadStats(RavenTransaction tx)
 
             var stats = new IndexStats
             {
-                CreatedTimestamp = DateTime.FromBinary(statsTree.Read(IndexSchema.CreatedTimestampSlice).Reader.ReadLittleEndianInt64()),
+                CreatedTimestamp = DateTime.FromBinary(CreatedTimestampAsBinary),
                 ErrorsCount = (int)(table?.NumberOfEntries ?? 0)
             };
 
@@ -424,8 +429,8 @@ public IndexStats ReadStats(RavenTransaction tx)
             {
                 var entriesCountSize = entriesCountReader.Value.Length;
                 //backward compatibility https://github.com/ravendb/ravendb/commit/5c53b01ee2b4fad8f3ef410f3e4976144d72c023
-                entriesCount = entriesCountSize == sizeof(long) 
-                    ? entriesCountReader.Value.ReadLittleEndianInt64() 
+                entriesCount = entriesCountSize == sizeof(long)
+                    ? entriesCountReader.Value.ReadLittleEndianInt64()
                     : entriesCountReader.Value.ReadLittleEndianInt32();
             }
 
@@ -918,7 +923,7 @@ public unsafe IndexFailureInformation UpdateStats(DateTime indexingTime, Indexin
 
                 if (stats.EntriesCount != null) // available only when tx was committed
                     statsTree.Add(IndexSchema.EntriesCount, stats.EntriesCount.Value);
-               
+
                 var binaryDate = indexingTime.ToBinary();
                 using (Slice.External(context.Allocator, (byte*)&binaryDate, sizeof(long), out Slice binaryDateslice))
                     statsTree.Add(IndexSchema.LastIndexingTimeSlice, binaryDateslice);
@@ -1014,11 +1019,11 @@ public static SearchEngineType ReadSearchEngineType(string name, StorageEnvironm
                 {
                     throw new InvalidOperationException($"Index '{name}' does not contain valid {nameof(SearchEngineType)} property. It contains: {result.Reader.ToStringValue()}.");
                 }
-               
+
                 return persistedSearchEngineType;
             }
-        }        
-        
+        }
+
         public static IndexSourceType ReadIndexSourceType(string name, StorageEnvironment environment)
         {
             using (var tx = environment.ReadTransaction())
@@ -1078,8 +1083,8 @@ internal HashSet<string> ReadIndexTimeFields()
             var searchEngineType = ReadSearchEngineType(_index.Name, _environment);
             using (var tx = _environment.ReadTransaction())
             {
-                fields = searchEngineType == SearchEngineType.Corax 
-                    ? Corax.Utils.TimeFields.ReadTimeFieldsNames(tx) 
+                fields = searchEngineType == SearchEngineType.Corax
+                    ? Corax.Utils.TimeFields.ReadTimeFieldsNames(tx)
                     : ReadLuceneTimeFields(tx);
             }
 
@@ -1102,7 +1107,7 @@ HashSet<string> ReadLuceneTimeFields(Transaction tx)
                         }
                     }
                 }
-                
+
                 return container;
             }
         }
@@ -1181,7 +1186,7 @@ internal sealed class IndexSchema
             public static readonly Slice TimeSlice;
 
             public static readonly Slice SearchEngineType;
-            
+
             static IndexSchema()
             {
                 using (StorageEnvironment.GetStaticContext(out var ctx))

src/Raven.Server/Raven.Server.csproj

@@ -179,6 +179,14 @@
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.10.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.10.0" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.7.0" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.7.0" />
+    <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="MySqlConnector" Version="2.3.7" />
     <PackageReference Include="NCrontab.Advanced" Version="1.3.28" />
     <PackageReference Include="Npgsql" Version="8.0.3" />
@@ -196,6 +204,7 @@
     </PackageReference>
     <PackageReference Include="Snappier" Version="1.1.6" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
+    <PackageReference Include="System.Collections.Immutable" Version="8.0.0" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
     <PackageReference Include="System.Net.Http" Version="4.3.4" />
     <PackageReference Include="System.Reflection.Metadata" Version="8.0.0" />

src/Raven.Server/SqlMigration/MsSQL/MsSqlDatabaseMigrator.Migration.cs

@@ -1,12 +1,14 @@
-using Raven.Server.SqlMigration.Schema;
+using Raven.Client.Documents.Operations.ETL.SQL;
+using Raven.Server.Documents.ETL.Providers.SQL.RelationalWriters;
+using Raven.Server.SqlMigration.Schema;
 
 namespace Raven.Server.SqlMigration.MsSQL
 {
     internal partial class MsSqlDatabaseMigrator : GenericDatabaseMigrator
     {
-        protected override string FactoryName => "System.Data.SqlClient";
+        protected override string FactoryName => "Microsoft.Data.SqlClient";
 
-        public MsSqlDatabaseMigrator(string connectionString) : base(connectionString)
+        public MsSqlDatabaseMigrator(string connectionString) : base(OverrideConnectionStringIfNeeded(connectionString))
         {
         }
 
@@ -78,5 +80,13 @@ protected override string GetSelectAllQueryForTable(string tableSchema, string t
         {
             return "select * from " + QuoteTable(tableSchema, tableName);
         }
+
+        private static string OverrideConnectionStringIfNeeded(string connectionString)
+        {
+            if (SqlConnectionStringParser.GetConnectionStringValue(connectionString, new string[] { "Encrypt" }) == null)
+                connectionString = SqlConnectionStringUtil.GetConnectionStringWithOptionalEncrypt(connectionString);
+
+            return connectionString;
+        }
     }
 }

src/Raven.Studio/typescript/models/database/settings/connectionStringSqlEtlModel.ts

@@ -9,11 +9,12 @@ import sqlConnectionStringSyntax = require("common/helpers/database/sqlConnectio
 class connectionStringSqlEtlModel extends connectionStringModel {
 
     static sqlProviders: Array<valueAndLabelItem<string, string> & { tooltip: string }> = [
-        { value: "System.Data.SqlClient", label: "Microsoft SQL Server", tooltip: sqlConnectionStringSyntax.mssqlSyntax },
+        { value: "Microsoft.Data.SqlClient", label: "Microsoft SQL Server", tooltip: sqlConnectionStringSyntax.mssqlSyntax },
         { value: "MySqlConnector.MySqlConnectorFactory", label: "MySQL Server", tooltip: sqlConnectionStringSyntax.mysqlSyntax },
         { value: "Npgsql", label: "PostgreSQL", tooltip: sqlConnectionStringSyntax.npgsqlSyntax },
         { value: "Oracle.ManagedDataAccess.Client", label: "Oracle Database", tooltip: sqlConnectionStringSyntax.oracleSyntax },
         { value: "MySql.Data.MySqlClient", label: "DEPRECATED: MySQL Server", tooltip: sqlConnectionStringSyntax.mysqlSyntax },
+        { value: "System.Data.SqlClient", label: "DEPRECATED: Microsoft SQL Server", tooltip: sqlConnectionStringSyntax.mssqlSyntax },
     ];
 
     connectionString = ko.observable<string>();

src/Raven.Studio/typescript/models/database/tasks/ongoingTaskSqlEtlEditModel.ts

@@ -127,15 +127,13 @@ class ongoingTaskSqlEtlEditModel extends ongoingTaskEditModel {
             Disabled: this.taskState() === "Disabled",
             MentorNode: this.manualChooseMentor() ? this.mentorNode() : undefined,
             PinToMentorNode: this.pinMentorNode(),
-            FactoryName: "System.Data.SqlClient",
             ForceQueryRecompile: this.forceRecompileQuery(),
             ParameterizeDeletes: this.parameterizedDeletes(),
             CommandTimeout: this.commandTimeout() || null,
             QuoteTables: this.tableQuotation(),
             Transforms: this.transformationScripts().map(x => x.toDto()),
             SqlTables: this.sqlTables().map(x => x.toDto())
-
-        } as Raven.Client.Documents.Operations.ETL.SQL.SqlEtlConfiguration;
+        };
     }
 
     static empty(): ongoingTaskSqlEtlEditModel {

src/Raven.Studio/typescript/models/database/tasks/sql/sqlMigration.ts

@@ -161,7 +161,7 @@ class sqlMigration {
     labelForProvider(type: Raven.Server.SqlMigration.MigrationProvider) {
         switch (type) {
             case "MsSQL":
-                return "Microsoft SQL Server (System.Data.SqlClient)";
+                return "Microsoft SQL Server (Microsoft.Data.SqlClient)";
             case "MySQL_MySql_Data":
                 return "DEPRECATED: MySQL Server (MySql.Data.MySqlClient)";
             case "MySQL_MySqlConnector":
@@ -305,7 +305,7 @@ class sqlMigration {
             case "MySQL_MySqlConnector":
                 return "MySqlConnector.MySqlConnectorFactory";
             case "MsSQL":
-                return "System.Data.SqlClient";
+                return "Microsoft.Data.SqlClient";
             case "NpgSQL":
                 return "Npgsql";
             case "Oracle":

src/Raven.Studio/typescript/viewmodels/resources/widgets/ongoingTasksWidget.ts

@@ -3,7 +3,6 @@ import websocketBasedWidget = require("viewmodels/resources/widgets/websocketBas
 import virtualGridController = require("widgets/virtualGrid/virtualGridController");
 import virtualColumn = require("widgets/virtualGrid/columns/virtualColumn");
 import iconsPlusTextColumn = require("widgets/virtualGrid/columns/iconsPlusTextColumn");
-import textColumn = require("widgets/virtualGrid/columns/textColumn");
 import genUtils = require("common/generalUtils");
 import clusterDashboardWebSocketClient = require("common/clusterDashboardWebSocketClient");
 import multiNodeTagsColumn = require("widgets/virtualGrid/columns/multiNodeTagsColumn");
@@ -196,15 +195,25 @@ class ongoingTasksWidget extends websocketBasedWidget<Raven.Server.Dashboard.Clu
     private getTaskCountClass(item: taskItem): string {
         if (!item.isTitleItem()) {
             return ""
-    }
+        }
 
         if (item.taskCount()) {
             return "text-bold";
-    }
+        }
 
         return "text-muted small";
+    }
+
+    private getOngoingTasksUrlForNode(databaseName: string, nodeTag: string): string {
+        const node = clusterTopologyManager.default.getClusterNodeByTag(nodeTag);
+
+        if (!node) {
+            return "#"
         }
 
+        return node.serverUrl() + "/studio/index.html" + appUrl.forOngoingTasks(databaseName);
+    }
+
     private prepareColumns(): virtualColumn[] {
         const grid = this.gridController();
         return [
@@ -224,7 +233,7 @@ class ongoingTasksWidget extends websocketBasedWidget<Raven.Server.Dashboard.Clu
                 headerTitle: "Nodes running the tasks"
             })
         ];
-        }
+    }
 
     reducePerDatabase(itemsArray: rawTaskItem[]): taskItem[] {
         const output: taskItem[] = [];