Add metrics for datacollector.exe - provides information about profil…

…ers (#2705)

* Initial version of metrics for DC profiling

* Tests

* Push fixes to make linux build

* provide telemetry opted in flag

* Small refactoring

* Change name

* Upgrade CC comp

* Move to guids

* Revert lang change

* Fixing descriptions

* Last changes

Co-authored-by: Jakub Chocholowicz <jachocho@microsoft.com>

scripts/build/TestPlatform.Dependencies.props

@@ -32,7 +32,7 @@
     <JsonNetVersion>9.0.1</JsonNetVersion>
     <MoqVersion>4.7.63</MoqVersion>
     <TestPlatformExternalsVersion>16.9.0-preview-4267359</TestPlatformExternalsVersion>
-    <CodeCoverageExternalsVersion>16.9.0-beta.21064.1</CodeCoverageExternalsVersion>
+    <CodeCoverageExternalsVersion>16.9.0-beta.21072.1</CodeCoverageExternalsVersion>
     <MicrosoftFakesVersion>16.9.0-beta.20628.1</MicrosoftFakesVersion>
 
     <MicrosoftBuildPackageVersion>16.0.461</MicrosoftBuildPackageVersion>

src/Microsoft.TestPlatform.Common/DataCollection/AfterTestRunEndResult.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollection
+{
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using System.Collections.Generic;
+    using System.Collections.ObjectModel;
+    using System.Runtime.Serialization;
+
+    /// <summary>
+    /// Payload object that is used to exchange data between datacollector process and runner process.
+    /// </summary>
+    [DataContract]
+    public class AfterTestRunEndResult
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="AfterTestRunEndResult"/> class.
+        /// </summary>
+        /// <param name="attachmentSets">
+        /// The collection of attachment sets.
+        /// </param>
+        /// <param name="metrics">
+        /// The metrics.
+        /// </param>
+        public AfterTestRunEndResult(Collection<AttachmentSet> attachmentSets, IDictionary<string, object> metrics)
+        {
+            this.AttachmentSets = attachmentSets;
+            this.Metrics = metrics;
+        }
+
+        [DataMember]
+        public Collection<AttachmentSet> AttachmentSets { get; private set; }
+
+        [DataMember]
+        public IDictionary<string, object> Metrics { get; private set; }
+    }
+}

src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionManager.cs

@@ -8,12 +8,12 @@ namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector
     using System.Collections.ObjectModel;
     using System.Globalization;
     using System.Linq;
-
     using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces;
     using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework;
     using Microsoft.VisualStudio.TestPlatform.Common.Logging;
     using Microsoft.VisualStudio.TestPlatform.Common.Utilities;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
@@ -61,13 +61,18 @@ internal class DataCollectionManager : IDataCollectionManager
         /// </summary>
         private DataCollectorExtensionManager dataCollectorExtensionManager;
 
+        /// <summary>
+        /// Request data
+        /// </summary>
+        private IDataCollectionTelemetryManager dataCollectionTelemetryManager;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="DataCollectionManager"/> class.
         /// </summary>
         /// <param name="messageSink">
         /// The message Sink.
         /// </param>
-        internal DataCollectionManager(IMessageSink messageSink) : this(new DataCollectionAttachmentManager(), messageSink)
+        internal DataCollectionManager(IMessageSink messageSink, IRequestData requestData) : this(new DataCollectionAttachmentManager(), messageSink, new DataCollectionTelemetryManager(requestData))
         {
         }
 
@@ -83,13 +88,14 @@ internal DataCollectionManager(IMessageSink messageSink) : this(new DataCollecti
         /// <remarks>
         /// The constructor is not public because the factory method should be used to get instances of this class.
         /// </remarks>
-        protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink)
+        protected DataCollectionManager(IDataCollectionAttachmentManager datacollectionAttachmentManager, IMessageSink messageSink, IDataCollectionTelemetryManager dataCollectionTelemetryManager)
         {
             this.attachmentManager = datacollectionAttachmentManager;
             this.messageSink = messageSink;
             this.events = new TestPlatformDataCollectionEvents();
             this.dataCollectorExtensionManager = null;
             this.RunDataCollectors = new Dictionary<Type, DataCollectorInformation>();
+            this.dataCollectionTelemetryManager = dataCollectionTelemetryManager;
         }
 
         /// <summary>
@@ -128,15 +134,15 @@ private DataCollectorExtensionManager DataCollectorExtensionManager
         /// <returns>
         /// The <see cref="DataCollectionManager"/>.
         /// </returns>
-        public static DataCollectionManager Create(IMessageSink messageSink)
+        public static DataCollectionManager Create(IMessageSink messageSink, IRequestData requestData)
         {
             if (Instance == null)
             {
                 lock (syncObject)
                 {
                     if (Instance == null)
                     {
-                        Instance = new DataCollectionManager(messageSink);
+                        Instance = new DataCollectionManager(messageSink, requestData);
                     }
                 }
             }
@@ -673,6 +679,8 @@ private void AddCollectorEnvironmentVariables(
                                 dataCollectionWrapper.Logger.LogError(this.dataCollectionEnvironmentContext.SessionDataCollectionContext, message);
                             }                            
                         }
+
+                        dataCollectionTelemetryManager.RecordEnvironmentVariableConflict(dataCollectionWrapper, namevaluepair.Key, namevaluepair.Value, alreadyRequestedVariable.Value);
                     }
                     else
                     {
@@ -685,6 +693,8 @@ private void AddCollectorEnvironmentVariables(
                         dataCollectorEnvironmentVariables.Add(
                             namevaluepair.Key,
                             new DataCollectionEnvironmentVariable(namevaluepair, collectorFriendlyName));
+
+                        dataCollectionTelemetryManager.RecordEnvironmentVariableAddition(dataCollectionWrapper, namevaluepair.Key, namevaluepair.Value);
                     }
                 }
             }

src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionTelemetryManager.cs

@@ -0,0 +1,102 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces;
+using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
+using System;
+using System.Linq;
+
+namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector
+{
+    /// <summary>
+    /// Stores and provides telemetry information for data collection.
+    /// </summary>
+    internal class DataCollectionTelemetryManager : IDataCollectionTelemetryManager
+    {
+        private const string CorProfilerVariable = "COR_PROFILER";
+        private const string CoreClrProfilerVariable = "CORECLR_PROFILER";
+        private const string ClrIeInstrumentationMethodConfigurationPrefix32Variable = "MicrosoftInstrumentationEngine_ConfigPath32_";
+        private const string ClrIeInstrumentationMethodConfigurationPrefix64Variable = "MicrosoftInstrumentationEngine_ConfigPath64_";
+
+        private static readonly Guid ClrIeProfilerGuid = Guid.Parse("{324f817a-7420-4e6d-b3c1-143fbed6d855}");
+        private const string OverwrittenProfilerName = "overwritten";
+
+        private readonly IRequestData requestData;
+
+        internal DataCollectionTelemetryManager(IRequestData requestData)
+        {
+            this.requestData = requestData;
+        }
+
+        /// <inheritdoc/>
+        public void RecordEnvironmentVariableAddition(DataCollectorInformation dataCollectorInformation, string name, string value)
+        {
+            RecordProfilerMetricForNewVariable(CorProfilerVariable, TelemetryDataConstants.DataCollectorsCorProfiler, dataCollectorInformation, name, value);
+            RecordProfilerMetricForNewVariable(CoreClrProfilerVariable, TelemetryDataConstants.DataCollectorsCoreClrProfiler, dataCollectorInformation, name, value);
+        }
+
+        /// <inheritdoc/>
+        public void RecordEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string value, string existingValue)
+        {
+            RecordProfilerMetricForConflictedVariable(CorProfilerVariable, TelemetryDataConstants.DataCollectorsCorProfiler, dataCollectorInformation, name, value, existingValue);
+            RecordProfilerMetricForConflictedVariable(CoreClrProfilerVariable, TelemetryDataConstants.DataCollectorsCoreClrProfiler, dataCollectorInformation, name, value, existingValue);
+        }
+
+        private void RecordProfilerMetricForNewVariable(string profilerVariable, string telemetryPrefix, DataCollectorInformation dataCollectorInformation, string name, string value)
+        {
+            if (!string.Equals(profilerVariable, name, StringComparison.Ordinal))
+            {
+                return;
+            }
+
+            requestData.MetricsCollection.Add(GetTelemetryKey(telemetryPrefix, dataCollectorInformation), GetProfilerGuid(value).ToString());
+        }
+
+        private void RecordProfilerMetricForConflictedVariable(string profilerVariable, string telemetryPrefix, DataCollectorInformation dataCollectorInformation, string name, string value, string existingValue)
+        {
+            // If data collector is requesting same profiler record it same as new
+            if (string.Equals(value, existingValue, StringComparison.OrdinalIgnoreCase))
+            {
+                RecordProfilerMetricForNewVariable(profilerVariable, telemetryPrefix, dataCollectorInformation, name, value);
+                return;
+            }
+
+            if (!string.Equals(profilerVariable, name, StringComparison.Ordinal))
+            {
+                return;
+            }
+
+            var existingProfilerGuid = GetProfilerGuid(existingValue);
+
+            if (ClrIeProfilerGuid == existingProfilerGuid)
+            {
+                if (dataCollectorInformation.TestExecutionEnvironmentVariables != null &&
+                    dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix32Variable)) &&
+                    dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix64Variable)))
+                {
+                    requestData.MetricsCollection.Add(GetTelemetryKey(telemetryPrefix, dataCollectorInformation), ClrIeProfilerGuid.ToString());
+                    return;
+                }
+            }
+
+            requestData.MetricsCollection.Add(GetTelemetryKey(telemetryPrefix, dataCollectorInformation), $"{existingProfilerGuid}({OverwrittenProfilerName}:{GetProfilerGuid(value)})");
+        }
+
+        private static Guid GetProfilerGuid(string profilerGuid)
+        {
+            Guid guid;
+            if (Guid.TryParse(profilerGuid, out guid))
+            {
+                return guid;
+            }
+
+            return Guid.Empty;
+        }
+
+        private static string GetTelemetryKey(string telemetryPrefix, DataCollectorInformation dataCollectorInformation)
+        {
+            return string.Format("{0}.{1}", telemetryPrefix, dataCollectorInformation.DataCollectorConfig?.TypeUri?.ToString());
+        }
+    }
+}

src/Microsoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionTelemetryManager.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces
+{
+    /// <summary>
+    /// The IDataCollectionTelemetryManager Interface.
+    /// </summary>
+    internal interface IDataCollectionTelemetryManager
+    {
+        /// <summary>
+        /// Record telemetry regarding environment variable added.
+        /// </summary>
+        /// <param name="dataCollectorInformation">
+        /// Data collector information which requested environment variable.
+        /// </param>
+        /// <param name="name">
+        /// Environment variable name.
+        /// </param>
+        /// <param name="value">
+        /// Environment variable value.
+        /// </param>
+        void RecordEnvironmentVariableAddition(DataCollectorInformation dataCollectorInformation, string name, string value);
+
+        /// <summary>
+        /// Record telemetry regarding environment variable is conflicting.
+        /// </summary>
+        /// <param name="dataCollectorInformation">
+        /// Data collector information which requested environment variable.
+        /// </param>
+        /// <param name="name">
+        /// Environment variable name.
+        /// </param>
+        /// <param name="value">
+        /// Environment variable value.
+        /// </param>
+        /// <param name="existingValue">
+        /// Environment variable value that was requested previously.
+        /// </param>
+        void RecordEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string value, string existingValue);
+    }
+}

src/Microsoft.TestPlatform.Common/RequestData.cs

@@ -4,7 +4,7 @@
 namespace Microsoft.VisualStudio.TestPlatform.Common
 {
     using System;
-
+    using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
 
     /// <inheritdoc />
@@ -23,6 +23,15 @@ public class RequestData : IRequestData
         /// </summary>
         private ProtocolConfig protocolConfig;
 
+        /// <summary>
+        /// The default constructor for request data.
+        /// </summary>
+        public RequestData()
+        {
+            this.MetricsCollection = new NoOpMetricsCollection();
+            this.IsTelemetryOptedIn = false;
+        }
+
         /// <summary>
         /// Gets or sets the metrics collection.
         /// </summary>

src/Microsoft.TestPlatform.Common/Telemetry/TelemetryDataConstants.cs

@@ -26,6 +26,10 @@ public static class TelemetryDataConstants
 
         public static string DataCollectorsEnabled = "VS.TestRun.DataCollectorsEnabled";
 
+        internal const string DataCollectorsCorProfiler = "VS.TestPlatform.DataCollector.CorProfiler";
+
+        internal const string DataCollectorsCoreClrProfiler = "VS.TestPlatform.DataCollector.CoreClrProfiler";
+
         public static string RunState = "VS.TestRun.RunState";
 
         public static string NumberOfSourcesSentForRun = "VS.TestRun.NumberOfSources";