Serialize dates with min 3 fractional digits when using SourceSerializer in integration tests (#4175)

This commit adds a new JsonConverter to serialize DateTime/DateTimeOffset with a minimum of 3 fractional digits when running integration tests using the test SourceSerializer. Related to a bug in Elasticsearch <7.1.0 in the XML comment of the JsonConverter.

Always emit the random values used when running a test/integrate build, not just if they were explicitly set.

(cherry picked from commit 57a6266)

Author: russcam

src/Tests/Tests.Core/Client/Serializers/TestSourceSerializerBase.cs

@@ -23,6 +23,7 @@ protected override JsonSerializerSettings CreateJsonSerializerSettings() =>
 		protected override IEnumerable<JsonConverter> CreateJsonConverters()
 		{
 			yield return new SourceOnlyUsingBuiltInConverter();
+			yield return new Domain.JsonConverters.DateTimeConverter();
 		}
 
 		protected override void ModifyContractResolver(ConnectionSettingsAwareContractResolver resolver) =>

src/Tests/Tests.Core/Xunit/NestXunitRunOptions.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using Elastic.Xunit;
 using Tests.Configuration;
@@ -91,18 +92,18 @@ private static string ReproduceCommandLine(ConcurrentBag<Tuple<string, string>>
 			bool runningIntegrations
 		)
 		{
-			var sb = new StringBuilder("build.bat ")
-				.Append($"seed:{config.Seed} ");
+			var sb = new StringBuilder(".\\build.bat ")
+				.Append("seed:").Append(config.Seed).Append(" ");
 
-			AppendExplictConfig(nameof(RandomConfiguration.SourceSerializer), sb);
-			AppendExplictConfig(nameof(RandomConfiguration.TypedKeys), sb);
-			AppendExplictConfig(nameof(RandomConfiguration.HttpCompression), sb);
+			AppendConfig(nameof(RandomConfiguration.SourceSerializer), config.Random.SourceSerializer, sb);
+			AppendConfig(nameof(RandomConfiguration.TypedKeys), config.Random.TypedKeys, sb);
+			AppendConfig(nameof(RandomConfiguration.HttpCompression), config.Random.HttpCompression, sb);
 
 			if (runningIntegrations)
 				sb.Append("integrate ")
 					.Append(TestConfiguration.Instance.ElasticsearchVersion);
-
-			else sb.Append("test");
+			else
+				sb.Append("test");
 
 			if (runningIntegrations && failedCollections.Count > 0)
 			{
@@ -135,20 +136,9 @@ bool runningIntegrations
 		}
 
 		/// <summary>
-		/// Append random overwrite to reproduce line only if one was provided explicitly
+		/// Append random values used
 		/// </summary>
-		private static void AppendExplictConfig(string key, StringBuilder sb)
-		{
-			if (!TryGetExplicitRandomConfig(key, out var b)) return;
-
-			sb.Append($"random:{key}{(b ? "" : ":false")} ");
-		}
-
-		private static bool TryGetExplicitRandomConfig(string key, out bool value)
-		{
-			value = false;
-			var v = Environment.GetEnvironmentVariable($"NEST_RANDOM_{key.ToUpper()}");
-			return !string.IsNullOrWhiteSpace(v) && bool.TryParse(v, out value);
-		}
+		private static void AppendConfig(string key, bool value, StringBuilder sb) =>
+			sb.Append($"random:{key.ToLowerInvariant()}{(value ? "" : ":false")} ");
 	}
 }

src/Tests/Tests.Domain/JsonConverters/DateTimeConverter.cs

@@ -0,0 +1,87 @@
+using System;
+using System.Globalization;
+using System.Text;
+using Newtonsoft.Json;
+
+namespace Tests.Domain.JsonConverters
+{
+	/// <summary>
+	/// DateTime/DateTimeOffset converter that always serializes values with a minimum of three sub second fractions.
+	/// This is to fix a bug in Elastisearch < 7.1.0: https://github.com/elastic/elasticsearch/pull/41871
+	/// </summary>
+	public class DateTimeConverter : Newtonsoft.Json.Converters.IsoDateTimeConverter
+	{
+		public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+		{
+			if (value == null)
+			{
+				writer.WriteNull();
+				return;
+			}
+
+			const string format = "yyyy'-'MM'-'dd'T'HH':'mm':'ss.FFFFFFF";
+			var builder = new StringBuilder(33);
+
+			if (value is DateTime dateTime)
+			{
+				if ((DateTimeStyles & DateTimeStyles.AdjustToUniversal) == DateTimeStyles.AdjustToUniversal
+					|| (DateTimeStyles & DateTimeStyles.AssumeUniversal) == DateTimeStyles.AssumeUniversal)
+				{
+					dateTime = dateTime.ToUniversalTime();
+				}
+
+				builder.Append(dateTime.ToString(format, CultureInfo.InvariantCulture));
+			}
+			else if (value is DateTimeOffset dateTimeOffset)
+			{
+				if ((DateTimeStyles & DateTimeStyles.AdjustToUniversal) == DateTimeStyles.AdjustToUniversal
+					|| (DateTimeStyles & DateTimeStyles.AssumeUniversal) == DateTimeStyles.AssumeUniversal)
+				{
+					dateTimeOffset = dateTimeOffset.ToUniversalTime();
+				}
+
+				builder.Append(dateTimeOffset.ToString(format, CultureInfo.InvariantCulture));
+				dateTime = dateTimeOffset.DateTime;
+			}
+			else
+				throw new JsonSerializationException(
+					$"Unexpected value when converting date. Expected DateTime or DateTimeOffset, got {value.GetType()}.");
+
+			if (builder.Length > 20 && builder.Length < 23)
+			{
+				var diff = 23 - builder.Length;
+				for (var i = 0; i < diff; i++)
+					builder.Append('0');
+			}
+
+			switch (dateTime.Kind)
+			{
+				case DateTimeKind.Local:
+					var offset = TimeZoneInfo.Local.GetUtcOffset(dateTime);
+					if (offset >= TimeSpan.Zero)
+						builder.Append('+');
+					else
+					{
+						builder.Append('-');
+						offset = offset.Negate();
+					}
+
+					AppendTwoDigitNumber(builder, offset.Hours);
+					builder.Append(':');
+					AppendTwoDigitNumber(builder, offset.Minutes);
+					break;
+				case DateTimeKind.Utc:
+					builder.Append('Z');
+					break;
+			}
+
+			writer.WriteValue(builder.ToString());
+		}
+
+		private static void AppendTwoDigitNumber(StringBuilder result, int val)
+		{
+			result.Append((char)('0' + (val / 10)));
+			result.Append((char)('0' + (val % 10)));
+		}
+	}
+}

src/Tests/Tests/Cluster/TaskManagement/GetTask/GetTaskApiTests.cs

@@ -138,7 +138,7 @@ protected override void IntegrationSetup(IElasticClient client, CallUniqueValues
 			var targetIndex = "tasks-lists-completed-get";
 			var bulkResponse = client.IndexMany(Project.Generator.Generate(500), sourceIndex);
 			if (!bulkResponse.IsValid)
-				throw new Exception("failure in setting up integration");
+				throw new Exception($"failure in setting up integration for {nameof(GetTaskApiCompletedTaskTests)}. {bulkResponse.DebugInformation}");
 
 			var createIndex = client.Indices.Create(targetIndex, i => i
 				.Settings(settings => settings.Analysis(DefaultSeeder.ProjectAnalysisSettings))