Add the ability to record / playback multiple versions (Azure#29246)

* wip for testing the ability to record / playback multiple versions

* address feedback and normalize version types

* update location of test project
move clientTestBase* tests to new test project
few minor fixes on clienttestfixtureattribute

* move recordedtestbase and testenvironmenttests

* create shared source for test framework helpers and move mgmtrecordedtestbasetests

* move recordings

* update after merge

* update namespaces

* fix a couple issues

* regen ResourceGroupResource tests for multi-version

* remove unused file

* address comments

* initial draft on readme changes

* address consistency between two flags and collapse into one

* minor update on readme

common/ManagementTestShared/Redesign/ManagementRecordedTestBase.cs

@@ -32,8 +32,11 @@ public abstract class ManagementRecordedTestBase<TEnvironment> : RecordedTestBas
 
         private ArmClient _cleanupClient;
         private WaitUntil _waitForCleanup;
+        private ResourceType _resourceType;
+        private string _apiVersion;
 
-        protected ManagementRecordedTestBase(bool isAsync, RecordedTestMode? mode = default) : base(isAsync, mode)
+        protected ManagementRecordedTestBase(bool isAsync, RecordedTestMode? mode = default)
+            : base(isAsync, mode)
         {
             AdditionalInterceptors = new[] { new ManagementInterceptor(this) };
 
@@ -42,6 +45,13 @@ protected ManagementRecordedTestBase(bool isAsync, RecordedTestMode? mode = defa
             Initialize();
         }
 
+        protected ManagementRecordedTestBase(bool isAsync, ResourceType resourceType, string apiVersion, RecordedTestMode? mode = default)
+            : this(isAsync, mode)
+        {
+            _resourceType = resourceType;
+            _apiVersion = apiVersion;
+        }
+
         private void Initialize()
         {
             _waitForCleanup = Mode == RecordedTestMode.Live ? WaitUntil.Completed : WaitUntil.Started;
@@ -67,6 +77,8 @@ protected ArmClient GetArmClient(ArmClientOptions clientOptions = default, strin
             options.Environment = GetEnvironment(TestEnvironment.ResourceManagerUrl);
             options.AddPolicy(ResourceGroupCleanupPolicy, HttpPipelinePosition.PerCall);
             options.AddPolicy(ManagementGroupCleanupPolicy, HttpPipelinePosition.PerCall);
+            if (_apiVersion is not null)
+                options.SetApiVersion(_resourceType, _apiVersion);
 
             return InstrumentClient(new ArmClient(
                 TestEnvironment.Credential,

common/ManagementTestShared/Redesign/MockTestBase.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 using Azure.Core.TestFramework;
-using Azure.ResourceManager.Resources;
 using System;
 using System.Net.Sockets;
 using System.Threading.Tasks;

eng/Directory.Build.Common.props

@@ -20,6 +20,7 @@
     <IsTestSupportProject Condition="'$(IsTestSupportProject)' == '' and '$(IsTestProject)' != 'true' and ($(MSBuildProjectDirectory.Contains('/tests/')) or $(MSBuildProjectDirectory.Contains('\tests\')))">true</IsTestSupportProject>
     <IsShippingLibrary Condition="'$(IsShippingLibrary)' == '' and '$(IsTestProject)' != 'true' and '$(IsTestSupportProject)' != 'true' and '$(IsPerfProject)' != 'true' and '$(IsSamplesProject)' != 'true' and '$(IsStressProject)' != 'true'">true</IsShippingLibrary>
     <IsShippingClientLibrary Condition="'$(IsClientLibrary)' == 'true' and '$(IsShippingLibrary)' == 'true'">true</IsShippingClientLibrary>
+    <TestFrameworkSupportFiles>$(MSBuildThisFileDirectory)/../sdk/core/Azure.Core.TestFramework/src/Shared/*.cs</TestFrameworkSupportFiles>
 
     <IncludeOperationsSharedSource Condition="'$(IncludeOperationsSharedSource)' == '' and '$(IsMgmtLibrary)' == 'true' and '$(IsTestProject)' != 'true' and '$(IsPerfProject)' != 'true'">true</IncludeOperationsSharedSource>
   </PropertyGroup>

sdk/core/Azure.Core.TestFramework/README.md

@@ -334,13 +334,28 @@ public abstract class BlobTestBase : StorageTestBase
 }
 ```
 
+The `ServiceVersion` must be either an Enum that is convertible to an Int32 or a string in the format of a date with an optional preview qualifier `yyyy-MM-dd[-preview]`.
+The list passed into `ClientTestFixture` must be homogenous.
+
+By default these versions will only apply to live tests.  There is an overloaded constructor which adds a flag `recordAllVersions` to apply these versions to record and playback as well.
+If this flag is set to true you will now get a version qualifier string added to the file name.
+
 Add a `ServiceVersion` parameter to the test class constructor and use the provided service version to create the `ClientOptions` instance.
 
 ```C#
 public BlobClientOptions GetOptions() =>
     new BlobClientOptions(_serviceVersion) { /* ... */ };
 ```
 
+For Management plane setting this in the client options is handled by default in the `ManagementRecordedTestBase` class by calling the new constructor which takes in the ResourceType and apiVersion to use.
+
+```C#
+        public ResourceGroupOperationsTests(bool isAsync, string apiVersion)
+            : base(isAsync, ResourceGroupResource.ResourceType, apiVersion)
+        {
+        }
+```
+
 To control what service versions a test will run against, use the `ServiceVersion` attribute by setting it's `Min` or `Max` properties (inclusive).
 
 ```C#

sdk/core/Azure.Core.TestFramework/src/Azure.Core.TestFramework.csproj

@@ -23,9 +23,10 @@
   </ItemGroup>
   <!-- Import Azure.Core shared source -->
   <ItemGroup>
-    <Compile Include="$(AzureCoreSharedSources)ConnectionString.cs" LinkBase="Shared" />
-    <Compile Include="$(AzureCoreSharedSources)DictionaryHeaders.cs" LinkBase="Shared" />
-    <Compile Include="$(AzureCoreSharedSources)EventSourceEventFormatting.cs" LinkBase="Shared" />
+    <Compile Remove="Shared\*.cs" />
+    <Compile Include="$(AzureCoreSharedSources)ConnectionString.cs" LinkBase="Shared\Core" />
+    <Compile Include="$(AzureCoreSharedSources)DictionaryHeaders.cs" LinkBase="Shared\Core" />
+    <Compile Include="$(AzureCoreSharedSources)EventSourceEventFormatting.cs" LinkBase="Shared\Core" />
   </ItemGroup>
 
   <ItemGroup>

sdk/core/Azure.Core.TestFramework/src/ClientTestFixtureAttribute.cs

@@ -5,6 +5,7 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Reflection;
 using NUnit.Framework;
@@ -15,13 +16,51 @@ namespace Azure.Core.TestFramework
 {
     public class ClientTestFixtureAttribute : NUnitAttribute, IFixtureBuilder2, IPreFilter
     {
+        public const string VersionQualifierProperty = "VersionQualifier";
+
+        private readonly struct TestVersion : IComparable
+        {
+            private readonly IComparable _comparable;
+            public object Object { get; }
+            public TestVersion(object obj)
+            {
+                Object = obj;
+                if (obj.GetType().IsEnum)
+                {
+                    _comparable = Convert.ToInt32(obj);
+                }
+                else if (obj is string)
+                {
+                    _comparable = new VersionString(obj as string);
+                }
+                else
+                {
+                    throw new InvalidOperationException("The items in the serviceVersions array must be an enum convertable to Int32 or a string in date format with an optional preview string");
+                }
+            }
+            public TestVersion(IComparable comparable, object obj)
+            {
+                _comparable = comparable;
+                Object = obj;
+            }
+
+            public int CompareTo(object obj)
+            {
+                if (obj is not TestVersion other)
+                    return 1;
+
+                return _comparable.CompareTo(other._comparable);
+            }
+        }
+
         public static readonly string SyncOnlyKey = "SyncOnly";
         public static readonly string RecordingDirectorySuffixKey = "RecordingDirectory";
 
         private readonly object[] _additionalParameters;
         private readonly object[] _serviceVersions;
-        private int? _actualPlaybackServiceVersion;
-        private int[] _actualLiveServiceVersions;
+        private readonly bool _recordAllVersions;
+        private object _actualPlaybackServiceVersion;
+        private object[] _actualLiveServiceVersions;
 
         /// <summary>
         /// Specifies which service version is run during recording/playback runs.
@@ -37,18 +76,33 @@ public class ClientTestFixtureAttribute : NUnitAttribute, IFixtureBuilder2, IPre
         /// Initializes an instance of the <see cref="ClientTestFixtureAttribute"/> accepting additional fixture parameters.
         /// </summary>
         /// <param name="serviceVersions">The set of service versions that will be passed to the test suite.</param>
-        public ClientTestFixtureAttribute(params object[] serviceVersions) : this(serviceVersions: serviceVersions, default)
+        public ClientTestFixtureAttribute(params object[] serviceVersions) : this(serviceVersions: serviceVersions, additionalParameters: default)
+        { }
+
+        /// <summary>
+        /// Initializes an instance of the <see cref="ClientTestFixtureAttribute"/> accepting additional fixture parameters.
+        /// </summary>
+        /// <param name="serviceVersions">The set of service versions that will be passed to the test suite.</param>
+        public ClientTestFixtureAttribute(bool recordAllVersions, params object[] serviceVersions) : this(serviceVersions: serviceVersions, additionalParameters: default, recordAllVersions)
         { }
 
         /// <summary>
         /// Initializes an instance of the <see cref="ClientTestFixtureAttribute"/> accepting additional fixture parameters.
         /// </summary>
         /// <param name="serviceVersions">The set of service versions that will be passed to the test suite.</param>
         /// <param name="additionalParameters">An array of additional parameters that will be passed to the test suite.</param>
-        public ClientTestFixtureAttribute(object[] serviceVersions, object[] additionalParameters)
+        /// <param name="recordAllVersions">True if you want all versions in serviceVersions to be recorded and used for playback, false otherwise.</param>
+        public ClientTestFixtureAttribute(object[] serviceVersions, object[] additionalParameters, bool recordAllVersions = false)
         {
             _additionalParameters = additionalParameters ?? new object[] { };
-            _serviceVersions = serviceVersions ?? new object[] { };
+            _serviceVersions = serviceVersions ?? Array.Empty<object>();
+            if (_serviceVersions.Length > 0)
+            {
+                var first = _serviceVersions[0];
+                if (!_serviceVersions.All(x => x.GetType() == first.GetType()))
+                    throw new InvalidOperationException("All service versions must be the same type and must be an enum convertable to Int32 or a string in date format with an optional preview string");
+            }
+            _recordAllVersions = recordAllVersions;
         }
 
         public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo)
@@ -58,22 +112,22 @@ public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo)
 
         public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo, IPreFilter filter)
         {
-            var latestVersion = _serviceVersions.Any() ? _serviceVersions.Max(Convert.ToInt32) : (int?)null;
-            _actualPlaybackServiceVersion = RecordingServiceVersion != null ? Convert.ToInt32(RecordingServiceVersion) : latestVersion;
+            object latestVersion = GetMax(_serviceVersions);
+            _actualPlaybackServiceVersion = RecordingServiceVersion ?? latestVersion;
 
-            int[] liveVersions = (LiveServiceVersions ?? _serviceVersions).Select(Convert.ToInt32).ToArray();
+            object[] liveVersions = (LiveServiceVersions ?? _serviceVersions).ToArray();
 
             if (liveVersions.Any())
             {
                 if (TestEnvironment.GlobalTestOnlyLatestVersion)
                 {
-                    _actualLiveServiceVersions = new[] { liveVersions.Max() };
+                    _actualLiveServiceVersions = new[] { GetMax(liveVersions) };
                 }
                 else if (TestEnvironment.GlobalTestServiceVersions is { Length: > 0 } globalTestServiceVersions &&
-                         _serviceVersions is { Length: > 0 })
+                            _serviceVersions is { Length: > 0 })
                 {
                     var enumType = _serviceVersions[0].GetType();
-                    var selectedVersions = new List<int>();
+                    var selectedVersions = new List<object>();
 
                     foreach (var versionString in globalTestServiceVersions)
                     {
@@ -111,13 +165,20 @@ public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo, IPreFilter filter)
             }
         }
 
+        internal static object GetMax(object[] array)
+        {
+            if (array == null || array.Length == 0)
+                return null;
+            return array.Max(v => new TestVersion(v)).Object;
+        }
+
         private List<(TestFixtureAttribute Suite, bool IsAsync, object ServiceVersion, object Parameter)> GeneratePermutations(bool includeSync, bool includeAsync)
         {
             var result = new List<(TestFixtureAttribute Suite, bool IsAsync, object ServiceVersion, object Parameter)>();
 
             void AddResult(object serviceVersion, object parameter)
             {
-                var parameters = new List<object>() ;
+                var parameters = new List<object>();
                 parameters.Add(true);
                 if (serviceVersion != null)
                 {
@@ -161,20 +222,20 @@ private void Process(TestSuite testSuite, object serviceVersion, bool isAsync, o
                 testSuite.Properties.Set(RecordingDirectorySuffixKey, parameter.ToString());
             }
 
-            var serviceVersionNumber = Convert.ToInt32(serviceVersion);
-            ApplyLimits(serviceVersionNumber, testSuite);
+            if (serviceVersion != null)
+                ApplyLimits(serviceVersion, testSuite);
 
-            ProcessTestList(testSuite, serviceVersion, isAsync, parameter, serviceVersionNumber);
+            ProcessTestList(testSuite, serviceVersion, isAsync, parameter);
         }
 
-        private void ProcessTestList(TestSuite testSuite, object serviceVersion, bool isAsync, object parameter, int serviceVersionNumber)
+        private void ProcessTestList(TestSuite testSuite, object serviceVersion, bool isAsync, object parameter)
         {
             List<Test> testsDoDelete = null;
             foreach (Test test in testSuite.Tests)
             {
                 if (test is ParameterizedMethodSuite parameterizedMethodSuite)
                 {
-                    ProcessTestList(parameterizedMethodSuite, serviceVersion, isAsync, parameter, serviceVersionNumber);
+                    ProcessTestList(parameterizedMethodSuite, serviceVersion, isAsync, parameter);
                     if (parameterizedMethodSuite.Tests.Count == 0)
                     {
                         testsDoDelete ??= new List<Test>();
@@ -183,7 +244,7 @@ private void ProcessTestList(TestSuite testSuite, object serviceVersion, bool is
                 }
                 else
                 {
-                    if (!ProcessTest(serviceVersion, isAsync, serviceVersionNumber, parameter, test))
+                    if (!ProcessTest(serviceVersion, isAsync, parameter, test))
                     {
                         testsDoDelete ??= new List<Test>();
                         testsDoDelete.Add(test);
@@ -201,7 +262,7 @@ private void ProcessTestList(TestSuite testSuite, object serviceVersion, bool is
             }
         }
 
-        private bool ProcessTest(object serviceVersion, bool isAsync, int serviceVersionNumber, object parameter, Test test)
+        private bool ProcessTest(object serviceVersion, bool isAsync, object parameter, Test test)
         {
             if (parameter != null)
             {
@@ -232,20 +293,20 @@ private bool ProcessTest(object serviceVersion, bool isAsync, int serviceVersion
             }
             else
             {
-                if (serviceVersionNumber != _actualPlaybackServiceVersion)
+                if (!_recordAllVersions && !serviceVersion.Equals(_actualPlaybackServiceVersion))
                 {
                     test.Properties.Add("_SkipRecordings", $"Test is ignored when not running live because the service version {serviceVersion} is not {_actualPlaybackServiceVersion}.");
                     runsRecorded = false;
                 }
 
                 if (_actualLiveServiceVersions != null &&
-                    !_actualLiveServiceVersions.Contains(serviceVersionNumber))
+                    !_actualLiveServiceVersions.Contains(serviceVersion))
                 {
                     test.Properties.Set("_SkipLive",
-                        $"Test ignored when running live service version {serviceVersion} is not one of {string.Join(", " , _actualLiveServiceVersions)}.");
+                        $"Test ignored when running live service version {serviceVersion} is not one of {string.Join(", ", _actualLiveServiceVersions)}.");
                     runsLive = false;
                 }
-                var passesVersionLimits = ApplyLimits(serviceVersionNumber, test);
+                var passesVersionLimits = ApplyLimits(serviceVersion, test);
                 runsLive &= passesVersionLimits;
                 runsRecorded &= passesVersionLimits;
             }
@@ -260,24 +321,28 @@ private bool ProcessTest(object serviceVersion, bool isAsync, int serviceVersion
                 test.Properties.Set("RunsLive", "These tests would run in Live mode.");
             }
 
+            if (_recordAllVersions)
+                test.Properties.Set(VersionQualifierProperty, serviceVersion.ToString());
+
             return runsRecorded || runsLive;
         }
 
-        private static bool ApplyLimits(int serviceVersionNumber, Test test)
+        private static bool ApplyLimits(object serviceVersionNumber, Test test)
         {
             var minServiceVersion = test.GetCustomAttributes<ServiceVersionAttribute>(true);
+            TestVersion testVersion = new TestVersion(serviceVersionNumber);
             foreach (ServiceVersionAttribute serviceVersionAttribute in minServiceVersion)
             {
                 if (serviceVersionAttribute.Min != null &&
-                    Convert.ToInt32(serviceVersionAttribute.Min) > serviceVersionNumber)
+                    testVersion.CompareTo(new TestVersion(serviceVersionAttribute.Min)) < 0)
                 {
                     test.RunState = RunState.Ignored;
                     test.Properties.Set("_SKIPREASON", $"Test ignored because it's minimum service version is set to {serviceVersionAttribute.Min}");
                     return false;
                 }
 
-                if (serviceVersionAttribute.Max != null &
-                    Convert.ToInt32(serviceVersionAttribute.Max) < serviceVersionNumber)
+                if (serviceVersionAttribute.Max != null &&
+                    testVersion.CompareTo(new TestVersion(serviceVersionAttribute.Max)) > 0)
                 {
                     test.RunState = RunState.Ignored;
                     test.Properties.Set("_SKIPREASON", $"Test ignored because it's maximum service version is set to {serviceVersionAttribute.Max}");

sdk/core/Azure.Core.TestFramework/src/Properties/AssemblyInfo.cs

@@ -5,3 +5,4 @@
 using Castle.Core.Internal;
 
 [assembly: InternalsVisibleTo(InternalsVisible.ToDynamicProxyGenAssembly2)]
+[assembly: InternalsVisibleTo("Azure.Core.TestFramework.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100d15ddcb29688295338af4b7686603fe614abd555e09efba8fb88ee09e1f7b1ccaeed2e8f823fa9eef3fdd60217fc012ea67d2479751a0b8c087a4185541b851bd8b16f8d91b840e51b1cb0ba6fe647997e57429265e85ef62d565db50a69ae1647d54d7bd855e4db3d8a91510e5bcbd0edfbbecaa20a7bd9ae74593daa7b11b4")]

sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs

@@ -177,6 +177,11 @@ public string ReplacementHost
         {
         };
 
+        /// <summary>
+        /// Creats a new instance of <see cref="RecordedTestBase"/>.
+        /// </summary>
+        /// <param name="isAsync">True if this instance is testing the async API variants false otherwise.</param>
+        /// <param name="mode">Indicates which <see cref="RecordedTestMode" /> this instance should run under.</param>
         protected RecordedTestBase(bool isAsync, RecordedTestMode? mode = null) : base(isAsync)
         {
             Mode = mode ?? TestEnvironment.GlobalTestMode;
@@ -207,7 +212,12 @@ protected internal string GetSessionFilePath()
 
             string name = new string(testAdapter.Name.Select(c => s_invalidChars.Contains(c) ? '%' : c).ToArray());
 
-            string fileName = name + (IsAsync ? "Async" : string.Empty) + ".json";
+            string versionQualifier = testAdapter.Properties.Get(ClientTestFixtureAttribute.VersionQualifierProperty) as string;
+
+            string async = IsAsync ? "Async" : string.Empty;
+            string version = versionQualifier is null ? string.Empty : $"[{versionQualifier}]";
+
+            string fileName = $"{name}{version}{async}.json";
 
             return Path.Combine(
                 GetSessionFileDirectory(),