Load CLR profiler assembly per AppDomain (DataDog#510)

Product changes:
- Introduce new map data structures in the profiler to keep track of which app domain's the profiler has been loaded into and which ones have run the startup IL code, and use them to determine if it is safe to instrument a caller method based on its AppDomain.
- Edit MSI to install net45 assets into GAC. This is necessary because we still run into "security grant" issue when the assembly to instrument is domain-neutral and our profiler assembly is not.

Testing changes:
- Refactor AppDomain.Instance program to easily run subprograms to test how integrations are instrumented while run in different AppDomain's. The original subprogram is `SqlServerNestedProgram`. Add a new `ElasticsearchNestedProgram` subprogram with wrapper sample `Samples.Elasticsearch.MultipleAppDomains`, but it is not running in CI because we haven't figured out how to run Elasticsearch in Windows CI on Azure DevOps

Notes:
- As noted with the MSI, we still must have the profiler in the GAC in order to load as a domain-neutral assembly and avoid the "security grant" issue.

Datadog.Trace.sln

@@ -249,6 +249,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EntityFramework6x.MdTokenLo
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.WcfClient", "samples\Samples.WcfClient\Samples.WcfClient.csproj", "{21D420EB-06D0-489D-A71C-C748BB46B8EF}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.Elasticsearch.MultipleAppDomains", "samples\Samples.Elasticsearch.MultipleAppDomains\Samples.Elasticsearch.MultipleAppDomains.csproj", "{48283691-0D4B-4ABD-B75B-EDFC682E1547}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -799,6 +801,16 @@ Global
 		{21D420EB-06D0-489D-A71C-C748BB46B8EF}.Release|x64.Build.0 = Release|x64
 		{21D420EB-06D0-489D-A71C-C748BB46B8EF}.Release|x86.ActiveCfg = Release|x86
 		{21D420EB-06D0-489D-A71C-C748BB46B8EF}.Release|x86.Build.0 = Release|x86
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Debug|Any CPU.ActiveCfg = Debug|x86
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Debug|x64.ActiveCfg = Debug|x64
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Debug|x64.Build.0 = Debug|x64
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Debug|x86.ActiveCfg = Debug|x86
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Debug|x86.Build.0 = Debug|x86
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Release|Any CPU.ActiveCfg = Release|x86
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Release|x64.ActiveCfg = Release|x64
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Release|x64.Build.0 = Release|x64
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Release|x86.ActiveCfg = Release|x86
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547}.Release|x86.Build.0 = Release|x86
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -856,6 +868,7 @@ Global
 		{DF923186-5397-4E15-A95B-F15C8327ED53} = {550AE553-2BBB-4021-B55A-137EF31A6B1F}
 		{E1706893-D3A5-43B9-9036-AEF49DB9600B} = {550AE553-2BBB-4021-B55A-137EF31A6B1F}
 		{21D420EB-06D0-489D-A71C-C748BB46B8EF} = {AA6F5582-3B71-49AC-AA39-8F7815AC46BE}
+		{48283691-0D4B-4ABD-B75B-EDFC682E1547} = {AA6F5582-3B71-49AC-AA39-8F7815AC46BE}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {160A1D00-1F5B-40F8-A155-621B4459D78F}

deploy/Datadog.Trace.ClrProfiler.WindowsInstaller/Files.Managed.Net45.wxs

@@ -8,32 +8,32 @@
       <Component Win64="$(var.Win64)">
         <File Id="net45_Datadog.Trace.ClrProfiler.Managed.dll"
               Source="$(var.ManagedDllPath)\net45\Datadog.Trace.ClrProfiler.Managed.dll"
-              KeyPath="yes" Checksum="yes"/>
+              KeyPath="yes" Checksum="yes" Assembly=".net"/>
       </Component>
       <Component Win64="$(var.Win64)">
         <File Id="net45_Datadog.Trace.dll"
               Source="$(var.ManagedDllPath)\net45\Datadog.Trace.dll"
-              KeyPath="yes" Checksum="yes"/>
+              KeyPath="yes" Checksum="yes" Assembly=".net"/>
       </Component>
       <Component Win64="$(var.Win64)">
         <File Id="net45_MsgPack.dll"
               Source="$(var.ManagedDllPath)\net45\MsgPack.dll"
-              KeyPath="yes" Checksum="yes"/>
+              KeyPath="yes" Checksum="yes" Assembly=".net"/>
       </Component>
       <Component Win64="$(var.Win64)">
         <File Id="net45_Newtonsoft.Json.dll"
               Source="$(var.ManagedDllPath)\net45\Newtonsoft.Json.dll"
-              KeyPath="yes" Checksum="yes"/>
+              KeyPath="yes" Checksum="yes" Assembly=".net"/>
       </Component>
       <Component Win64="$(var.Win64)">
         <File Id="net45_Sigil.dll"
               Source="$(var.ManagedDllPath)\net45\Sigil.dll"
-              KeyPath="yes" Checksum="yes"/>
+              KeyPath="yes" Checksum="yes" Assembly=".net"/>
       </Component>
       <Component Win64="$(var.Win64)">
         <File Id="net45_System.Runtime.InteropServices.RuntimeInformation.dll"
               Source="$(var.ManagedDllPath)\net45\System.Runtime.InteropServices.RuntimeInformation.dll"
-              KeyPath="yes" Checksum="yes"/>
+              KeyPath="yes" Checksum="yes" Assembly=".net"/>
       </Component>
     </ComponentGroup>
   </Fragment>

reproduction-dependencies/AppDomain.Instance/AppDomain.Instance.csproj

@@ -1,11 +1,21 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <OutputType>Library</OutputType>
+    <OutputType>Exe</OutputType>
     <TargetFrameworks>net45</TargetFrameworks>
     <Platforms>x64;x86</Platforms>
     <PlatformTarget>$(Platform)</PlatformTarget>
     <IsPackable>false</IsPackable>
+    <LoadManagedProfilerFromProfilerDirectory>true</LoadManagedProfilerFromProfilerDirectory>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Elasticsearch.Net" Version="6.1.0" />
+    <PackageReference Include="NEST" Version="6.1.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="System.Net.Http" />
+  </ItemGroup>
+
 </Project>

reproduction-dependencies/AppDomain.Instance/AppDomainInstanceProgram.cs

@@ -11,160 +11,55 @@ namespace AppDomain.Instance
 {
     public class AppDomainInstanceProgram : MarshalByRefObject
     {
-        public NestedProgram WorkerProgram { get; set; }
+        public static void Main(string[] args)
+        {
+            new AppDomainInstanceProgram().Run(args);
+        }
 
-        public int Main(string[] args)
+        public int Run(string[] args)
         {
             Console.WriteLine("Starting AppDomain Instance Test");
 
             string appDomainName = "crash-dummy";
+            string programName = string.Empty;
             int index = 1;
 
             if (args?.Length > 0)
             {
                 appDomainName = args[0];
                 index = int.Parse(args[1]);
+                programName = args[2];
             }
-            try
-            {
-                var instance = new NestedProgram()
-                {
-                    AppDomainName = appDomainName,
-                    AppDomainIndex = index
-                };
-
-                WorkerProgram = instance;
 
-                // Act like we're doing some continuing work
-                while (true)
-                {
-                    Thread.Sleep(500);
-                    instance.MakeSomeCall();
-
-                    if (instance.TotalCallCount > 3)
-                    {
-                        // Meh, call it quits
-                        break;
-                    }
-                }
-            }
-            catch (Exception ex)
+            NestedProgram instance;
+            if (programName.Equals("SqlServer", StringComparison.OrdinalIgnoreCase))
             {
-                Console.WriteLine($"We have encountered an exception in this instance: {appDomainName} : {ex.Message}");
-                Console.Error.WriteLine(ex);
-                return -10;
+                instance = new SqlServerNestedProgram();
             }
-
-            return 0;
-        }
-
-        public class NestedProgram : MarshalByRefObject
-        {
-            public object CallLock { get; } = new object();
-            public string AppDomainName { get; set; }
-            public int AppDomainIndex { get; set; }
-            public int TotalCallCount { get; set; }
-            public int CurrentCallCount { get; set; }
-            public bool DenyAllCalls { get; set; }
-
-            private readonly string _connectionString = @"Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Connection Timeout=30";
-            private readonly string _jokeTable = "GreatJoke";
-
-            public NestedProgram()
+            else if (programName.Equals("Elasticsearch", StringComparison.OrdinalIgnoreCase))
             {
-                InitializeJokeTable();
+                instance = new ElasticsearchNestedProgram();
             }
-
-            public void MakeSomeCall()
+            else
             {
-                if (DenyAllCalls)
-                {
-                    return;
-                }
-
-                lock (CallLock)
-                {
-                    try
-                    {
-                        CurrentCallCount++;
-
-                        var httpClient = new HttpClient();
-                        httpClient.DefaultRequestHeaders
-                              .Accept
-                              .Add(new MediaTypeWithQualityHeaderValue("application/json"));
-                        Console.WriteLine($"App {AppDomainIndex} - Starting client.GetAsync");
-                        var responseTask = httpClient.GetAsync("https://icanhazdadjoke.com/");
-                        responseTask.Wait(1000);
-                        if (responseTask.IsCompleted)
-                        {
-                            var jokeReaderTask = responseTask.Result.Content.ReadAsStringAsync();
-                            jokeReaderTask.Wait();
-                            var joke = jokeReaderTask.Result;
-
-                            StoreJoke(joke);
-                            var lastStoredJoke = GetLastJoke();
-
-                            Console.WriteLine($"Joke: {lastStoredJoke}");
-                        }
-
-                        Console.WriteLine($"App {AppDomainIndex} - Finished client.GetAsync");
-                    }
-                    finally
-                    {
-                        lock (CallLock)
-                        {
-                            CurrentCallCount--;
-                        }
-
-                        TotalCallCount++;
-                    }
-                }
+                Console.WriteLine($"programName {programName} not recognized. Exiting with error code -10.");
+                return -10;
             }
 
-            public string GetLastJoke()
+            try
             {
-                using (var connection = (DbConnection)new SqlConnection(_connectionString))
-                using (var command = connection.CreateCommand())
-                {
-                    Console.WriteLine($"Reading last joke from SQL for instance #{AppDomainIndex}");
-                    command.CommandText = $"SELECT TOP 1 Text FROM {_jokeTable} ORDER BY Id DESC;";
-                    connection.Open();
-                    var reader = command.ExecuteReader();
-                    reader.Read();
-                    var result = reader[0];
-                    return result.ToString();
-                }
+                instance.AppDomainName = appDomainName;
+                instance.AppDomainIndex = index;
+                instance.Run();
             }
-
-            public void StoreJoke(string joke)
+            catch (Exception ex)
             {
-                joke = joke.Replace("'", "`"); // horrible sanitization :)
-                using (var connection = new SqlConnection(_connectionString))
-                using (var command = new SqlCommand())
-                {
-                    Console.WriteLine($"Inserting joke into SQL for instance #{AppDomainIndex}");
-                    connection.Open();
-                    command.Connection = connection;
-                    command.CommandText = $"INSERT INTO {_jokeTable} (Text) VALUES ('{joke}')";
-                    command.ExecuteNonQuery();
-                    connection.Close();
-                }
+                Console.WriteLine($"We have encountered an exception in this instance: {appDomainName} : {ex.Message}");
+                Console.Error.WriteLine(ex);
+                return -10;
             }
 
-            private void InitializeJokeTable()
-            {
-                using (var connection = new SqlConnection(_connectionString))
-                using (var command = new SqlCommand())
-                {
-                    connection.Open();
-                    command.Connection = connection;
-                    command.CommandText = $"IF NOT EXISTS( SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{_jokeTable}') " +
-                                          $"CREATE TABLE {_jokeTable} (Id int identity(1,1),Text VARCHAR(500))";
-                    command.ExecuteNonQuery();
-                    connection.Close();
-                    Console.WriteLine($"Created joke table for instance #{AppDomainIndex}");
-                }
-            }
+            return 0;
         }
     }
 }