Track dependency instrumented with HttpDesktop DiagnosticSource in

DiagnosticSource Response event (#509)

Src/DependencyCollector/Net46.Tests/DependencyTrackingTelemetryModuleTestNet46.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using System.Diagnostics;
+    using System.IO;
     using System.Net;
 
     using Microsoft.ApplicationInsights.Channel;
@@ -54,9 +55,13 @@ public void TestDependencyCollectionNoParentActivity()
                 module.ProfileQueryEndpoint = FakeProfileApiEndpoint;
                 module.Initialize(config);
 
-                var url = new Uri("http://bing.com");
+                var url = new Uri("https://www.bing.com/");
                 HttpWebRequest request = WebRequest.CreateHttp(url);
-                request.GetResponse();
+                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
+                using (var reader = new StreamReader(response.GetResponseStream()))
+                {
+                    reader.ReadToEnd();
+                }
 
                 Assert.IsNotNull(sentTelemetry);
                 var item = (DependencyTelemetry)sentTelemetry;
@@ -118,11 +123,16 @@ public void TestDependencyCollectionWithParentActivity()
                 module.ProfileQueryEndpoint = FakeProfileApiEndpoint;
                 module.Initialize(config);
 
-                var url = new Uri("http://bing.com");
+                var url = new Uri("https://www.bing.com/");
                 HttpWebRequest request = WebRequest.CreateHttp(url);
 
                 var parent = new Activity("parent").AddBaggage("k", "v").SetParentId("|guid.").Start();
-                request.GetResponse();
+                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
+                using (var reader = new StreamReader(response.GetResponseStream()))
+                {
+                    reader.ReadToEnd();
+                }
+
                 parent.Stop();
 
                 Assert.IsNotNull(sentTelemetry);

Src/DependencyCollector/Shared.Tests/DependencyCollector.Shared.Tests.projitems

@@ -14,6 +14,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)HttpDependenciesParsingTelemetryInitializerTest.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\ClientServerDependencyTrackerTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\DependencyTargetNameHelperTest.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Implementation\DesktopDiagnosticSourceHttpProcessingTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\HttpParsers\AzureBlobHttpParserTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\HttpParsers\AzureServiceBusHttpParserTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\HttpParsers\DocumentDbHttpParserTests.cs" />

Src/DependencyCollector/Shared/DependencyCollector.Shared.projitems

@@ -16,6 +16,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\AppMapCorrelationEventSource.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\ClientServerDependencyTracker.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\DependencyTargetNameHelper.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Implementation\DesktopDiagnosticSourceHttpProcessing.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\HttpParsers\AzureBlobHttpParser.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\HttpParsers\AzureQueueHttpParser.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Implementation\HttpParsers\AzureIotHubHttpParser.cs" />

Src/DependencyCollector/Shared/DependencyTrackingTelemetryModule.cs

@@ -221,9 +221,23 @@ protected virtual void Dispose(bool disposing)
         /// </summary>
         private void InitializeForFrameworkEventSource()
         {
+#if NET45
+            DesktopDiagnosticSourceHttpProcessing desktopHttpProcessing = new DesktopDiagnosticSourceHttpProcessing(
+                this.telemetryConfiguration, 
+                DependencyTableStore.Instance.WebRequestConditionalHolder,
+                this.SetComponentCorrelationHttpHeaders,
+                this.ExcludeComponentCorrelationHttpHeadersOnDomains,
+                this.EffectiveProfileQueryEndpoint);
+            this.httpDesktopDiagnosticSourceListener = new HttpDesktopDiagnosticSourceListener(desktopHttpProcessing);
+#endif
+
 #if !NET40
-            FrameworkHttpProcessing frameworkHttpProcessing = new FrameworkHttpProcessing(this.telemetryConfiguration, DependencyTableStore.Instance.WebRequestCacheHolder, this.SetComponentCorrelationHttpHeaders, this.ExcludeComponentCorrelationHttpHeadersOnDomains, this.EffectiveProfileQueryEndpoint);
-            this.httpDesktopDiagnosticSourceListener = new HttpDesktopDiagnosticSourceListener(frameworkHttpProcessing);
+            FrameworkHttpProcessing frameworkHttpProcessing = new FrameworkHttpProcessing(
+                this.telemetryConfiguration,
+                DependencyTableStore.Instance.WebRequestCacheHolder, 
+                this.SetComponentCorrelationHttpHeaders, 
+                this.ExcludeComponentCorrelationHttpHeadersOnDomains, 
+                this.EffectiveProfileQueryEndpoint);
 
             // In 4.5 EventListener has a race condition issue in constructor so we retry to create listeners
             this.httpEventListener = RetryPolicy.Retry<InvalidOperationException, TelemetryConfiguration, FrameworkHttpEventListener>(

Src/DependencyCollector/Shared/Implementation/ClientServerDependencyTracker.cs

@@ -106,7 +106,7 @@ internal static Tuple<DependencyTelemetry, bool> GetTupleForWebDependencies(WebR
 
             Tuple<DependencyTelemetry, bool> telemetryTuple = null;
 
-            if (DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
+            if (DependencyTableStore.Instance.IsDesktopHttpDiagnosticSourceActivated || DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
             {
                 telemetryTuple = DependencyTableStore.Instance.WebRequestConditionalHolder.Get(webRequest);
             }
@@ -139,7 +139,7 @@ internal static void AddTupleForWebDependencies(WebRequest webRequest, Dependenc
             }
 
             var telemetryTuple = new Tuple<DependencyTelemetry, bool>(telemetry, isCustomCreated);
-            if (DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
+            if (DependencyTableStore.Instance.IsDesktopHttpDiagnosticSourceActivated || DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
             {
                 DependencyTableStore.Instance.WebRequestConditionalHolder.Store(webRequest, telemetryTuple);
             }
@@ -165,7 +165,7 @@ internal static Tuple<DependencyTelemetry, bool> GetTupleForSqlDependencies(SqlC
 
             Tuple<DependencyTelemetry, bool> telemetryTuple = null;
 
-            if (DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
+            if (DependencyTableStore.Instance.IsDesktopHttpDiagnosticSourceActivated || DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
             {
                 telemetryTuple = DependencyTableStore.Instance.SqlRequestConditionalHolder.Get(sqlRequest);
             }
@@ -198,7 +198,7 @@ internal static void AddTupleForSqlDependencies(SqlCommand sqlRequest, Dependenc
             }
 
             var telemetryTuple = new Tuple<DependencyTelemetry, bool>(telemetry, isCustomCreated);
-            if (DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
+            if (DependencyTableStore.Instance.IsDesktopHttpDiagnosticSourceActivated || DependencyTableStore.Instance.IsProfilerActivated || PretendProfilerIsAttached)
             {
                 DependencyTableStore.Instance.SqlRequestConditionalHolder.Store(sqlRequest, telemetryTuple);
             }

Src/DependencyCollector/Shared/Implementation/DependencyTableStore.cs

@@ -12,6 +12,7 @@ internal class DependencyTableStore : IDisposable
         internal ObjectInstanceBasedOperationHolder SqlRequestConditionalHolder;
 
         internal bool IsProfilerActivated = false;
+        internal bool IsDesktopHttpDiagnosticSourceActivated = false;
         private static DependencyTableStore instance;
 
         private DependencyTableStore() 

Src/DependencyCollector/Shared/Implementation/DesktopDiagnosticSourceHttpProcessing.cs

@@ -0,0 +1,81 @@
+#if NET45
+namespace Microsoft.ApplicationInsights.DependencyCollector.Implementation
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Net;
+    using Microsoft.ApplicationInsights.DataContracts;
+    using Microsoft.ApplicationInsights.DependencyCollector.Implementation.Operation;
+    using Microsoft.ApplicationInsights.Extensibility;
+    using Microsoft.ApplicationInsights.Web.Implementation;
+
+    /// <summary>
+    /// Concrete class with all processing logic to generate RDD data from the callbacks received from HttpDesktopDiagnosticSourceListener.
+    /// </summary>
+    internal sealed class DesktopDiagnosticSourceHttpProcessing : HttpProcessing
+    {
+        internal ObjectInstanceBasedOperationHolder TelemetryTable;
+
+        internal DesktopDiagnosticSourceHttpProcessing(TelemetryConfiguration configuration, ObjectInstanceBasedOperationHolder telemetryTupleHolder, bool setCorrelationHeaders, ICollection<string> correlationDomainExclusionList, string appIdEndpoint)
+            : base(configuration, SdkVersionUtils.GetSdkVersion("rdd" + RddSource.FrameworkAndDiagnostic + ":"), null, setCorrelationHeaders, correlationDomainExclusionList, appIdEndpoint)
+        {
+            if (telemetryTupleHolder == null)
+            {
+                throw new ArgumentNullException("telemetryTupleHolder");
+            }
+
+            this.TelemetryTable = telemetryTupleHolder;
+        }
+
+        /// <summary>
+        /// On request send callback from Http diagnostic source.
+        /// </summary>
+        /// <param name="request">The WebRequest object.</param>
+        public void OnRequestSend(WebRequest request)
+        {
+            this.OnBegin(request, true);
+        }
+
+        /// <summary>
+        /// On request send callback from Http diagnostic source.
+        /// </summary>
+        /// <param name="request">The WebRequest object.</param>
+        /// <param name="response">The WebResponse object.</param>
+        public void OnResponseReceive(WebRequest request, HttpWebResponse response)
+        {
+            this.OnEnd(null, request, response);
+        }
+
+        /// <summary>
+        /// Implemented by the derived class for adding the tuple to its specific cache.
+        /// </summary>
+        /// <param name="webRequest">The request which acts the key.</param>
+        /// <param name="telemetry">The dependency telemetry for the tuple.</param>
+        /// <param name="isCustomCreated">Boolean value that tells if the current telemetry item is being added by the customer or not.</param>
+        protected override void AddTupleForWebDependencies(WebRequest webRequest, DependencyTelemetry telemetry, bool isCustomCreated)
+        {
+            var telemetryTuple = new Tuple<DependencyTelemetry, bool>(telemetry, isCustomCreated);
+            this.TelemetryTable.Store(webRequest, telemetryTuple);
+        }
+
+        /// <summary>
+        /// Implemented by the derived class for getting the tuple from its specific cache.
+        /// </summary>
+        /// <param name="webRequest">The request which acts as the key.</param>
+        /// <returns>The tuple for the given request.</returns>
+        protected override Tuple<DependencyTelemetry, bool> GetTupleForWebDependencies(WebRequest webRequest)
+        {
+            return this.TelemetryTable.Get(webRequest);
+        }
+
+        /// <summary>
+        /// Implemented by the derived class for removing the tuple from its specific cache.
+        /// </summary>
+        /// <param name="webRequest">The request which acts as the key.</param>
+        protected override void RemoveTupleForWebDependencies(WebRequest webRequest)
+        {
+            this.TelemetryTable.Remove(webRequest);
+        }
+    }
+}
+#endif

Src/DependencyCollector/Shared/Implementation/FrameworkHttpProcessing.cs

@@ -12,15 +12,7 @@ namespace Microsoft.ApplicationInsights.DependencyCollector.Implementation
     using Microsoft.ApplicationInsights.Web.Implementation;
 
     /// <summary>
-    /// Concrete class with all processing logic to generate RDD data from the callbacks
-    /// received from framework events for HTTP. This class receives and uses two types of events, which
-    /// are EventSource from FrameworkHttpEventListener, and DiagnosticSource from
-    /// HttpDiagnosticSourceListener. The challenge is, the diagnostic source events have the WebRequest
-    /// object which we need for header injection, but the events can be fired multiple times. So you won't
-    /// know if it's the first request, or if it's the last response. The event source events fire at the right
-    /// locations and just once, but they don't have the rich information. This class coordinates both events,
-    /// store information in DependencyTelemetry properly, and fire the telemetry only on the EventSource
-    /// response receive to guarantee it's done just once and at the right time.
+    /// Concrete class with all processing logic to generate RDD data from the callbacks received from FrameworkHttpEventListener.
     /// </summary>
     internal sealed class FrameworkHttpProcessing : HttpProcessing
     {
@@ -49,6 +41,12 @@ public void OnBeginHttpCallback(long id, string resourceName)
             try
             {
                 DependencyCollectorEventSource.Log.BeginCallbackCalled(id, resourceName);
+                if (DependencyTableStore.Instance.IsDesktopHttpDiagnosticSourceActivated)
+                {
+                    // request is handled by Desktop DiagnosticSource Listener
+                    DependencyCollectorEventSource.Log.TrackingAnExistingTelemetryItemVerbose();
+                    return;
+                }
 
                 if (string.IsNullOrEmpty(resourceName))
                 {
@@ -111,7 +109,6 @@ public void OnBeginHttpCallback(long id, string resourceName)
         public void OnEndHttpCallback(long id, bool? success, bool synchronous, int? statusCode)
         {
             DependencyCollectorEventSource.Log.EndCallbackCalled(id.ToString(CultureInfo.InvariantCulture));
-
             var telemetryTuple = this.TelemetryTable.Get(id);
 
             if (telemetryTuple == null)
@@ -125,13 +122,6 @@ public void OnEndHttpCallback(long id, bool? success, bool synchronous, int? sta
                 this.TelemetryTable.Remove(id);
                 DependencyTelemetry telemetry = telemetryTuple.Item1;
 
-                // If this telemetry was processed via the DiagnosticSource path, we should record that fact in the
-                // SdkVersion field
-                if (this.HasTouchedByDiagnosticSource(telemetry))
-                {
-                    telemetry.Context.GetInternalContext().SdkVersion = SdkVersionUtils.GetSdkVersion("rdd" + RddSource.FrameworkAndDiagnostic + ":");
-                }
-
                 if (statusCode.HasValue)
                 {
                     // We calculate success on the base of http code and do not use the 'success' method argument
@@ -152,44 +142,6 @@ public void OnEndHttpCallback(long id, bool? success, bool synchronous, int? sta
             }
         }
 
-        /// <summary>
-        /// On request send callback from Http diagnostic source.
-        /// </summary>
-        /// <param name="request">The WebRequest object.</param>
-        public void OnRequestSend(WebRequest request)
-        {
-            // At this point, we need to determine if this is the first time we are examining this request.
-            // There are 3 possibilities
-            // 1. This is the very first time
-            // 2. This is the first time via OnRequestSend, but it's been processed by OnBeginHttpCallback already
-            // 3. This is not the first time it's processed by OnRequestSend.
-            // We need to determine which case. If the telemetry object is not found, then it's case 1. If the
-            // telemetry object exists, but it's never processed via DiagnosticSource, then it's case 2.
-            // Otherwise, it's case 3. In both case 1 and 2, we need OnBegin to add all properties.
-            Tuple<DependencyTelemetry, bool> tuple = this.GetTupleForWebDependencies(request);
-            DependencyTelemetry telemetry = tuple?.Item1;
-
-            if (this.HasTouchedByDiagnosticSource(telemetry))
-            {
-                // This is case 3, so make sure we skip update if it already exists.
-                this.OnBegin(request, true /*skipIfNotNew*/);
-            }
-            else
-            {            
-                this.OnBegin(request, false /*skipIfNotNew*/);
-            }
-        }
-
-        /// <summary>
-        /// On request send callback from Http diagnostic source.
-        /// </summary>
-        /// <param name="request">The WebRequest object.</param>
-        /// <param name="response">The WebResponse object.</param>
-        public void OnResponseReceive(WebRequest request, HttpWebResponse response)
-        {
-            this.OnEnd(null, request, response, false);
-        }
-
         /// <summary>
         /// Implemented by the derived class for adding the tuple to its specific cache.
         /// </summary>
@@ -220,23 +172,6 @@ protected override void RemoveTupleForWebDependencies(WebRequest webRequest)
         {
             this.TelemetryTable.Remove(ClientServerDependencyTracker.GetIdForRequestObject(webRequest));
         }
-
-        /// <summary>
-        /// Detects if the telemetry object has been processed via the DiagnosticSource path.
-        /// </summary>
-        /// <param name="telemetry">The DependencyTelemetry object to examine.</param>
-        private bool HasTouchedByDiagnosticSource(DependencyTelemetry telemetry)
-        {
-            // If it was ever processed via the DiagnosticSource path, then telemetry.Name
-            // must have the HTTP method name at the front, so first character is not a '/'.
-            string name = telemetry?.Name;
-            if (!string.IsNullOrEmpty(name) && !name.StartsWith("/", StringComparison.OrdinalIgnoreCase))
-            {
-                return true;
-            }
-
-            return false;
-        }
     }
 }
 #endif

Src/DependencyCollector/Shared/Implementation/HttpDesktopDiagnosticSourceListener.cs

@@ -1,4 +1,4 @@
-#if !NET40
+#if NET45
 namespace Microsoft.ApplicationInsights.DependencyCollector.Implementation
 {
     using System;
@@ -10,14 +10,14 @@ namespace Microsoft.ApplicationInsights.DependencyCollector.Implementation
     /// </summary>
     internal class HttpDesktopDiagnosticSourceListener : IObserver<KeyValuePair<string, object>>, IDisposable
     {
-        private readonly FrameworkHttpProcessing httpProcessingFramework;
+        private readonly DesktopDiagnosticSourceHttpProcessing httpProcessingFramework;
         private readonly HttpDesktopDiagnosticSourceSubscriber subscribeHelper;
         private readonly PropertyFetcher requestFetcherRequestEvent;
         private readonly PropertyFetcher requestFetcherResponseEvent;
         private readonly PropertyFetcher responseFetcher;
         private bool disposed = false;
 
-        internal HttpDesktopDiagnosticSourceListener(FrameworkHttpProcessing httpProcessing)
+        internal HttpDesktopDiagnosticSourceListener(DesktopDiagnosticSourceHttpProcessing httpProcessing)
         {
             this.httpProcessingFramework = httpProcessing;
             this.subscribeHelper = new HttpDesktopDiagnosticSourceSubscriber(this);

Src/DependencyCollector/Shared/Implementation/HttpDesktopDiagnosticSourceSubscriber.cs

@@ -48,6 +48,7 @@ public void OnNext(DiagnosticListener value)
                 if (value.Name == "System.Net.Http.Desktop")
                 {
                     this.sourceSubscription = value.Subscribe(this.parent, (Predicate<string>)null);
+                    DependencyTableStore.Instance.IsDesktopHttpDiagnosticSourceActivated = true;
                 }
             }
         }