Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add metrics for datacollector.exe - provides information about profilers #2705

Merged
merged 12 commits into from
Jan 25, 2021
Merged

Add metrics for datacollector.exe - provides information about profilers #2705

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
a4039b6
Initial version of metrics for DC profiling
Jan 20, 2021
acb01d4
Tests
Jan 20, 2021
68c5434
Push fixes to make linux build
Jan 21, 2021
525d674
provide telemetry opted in flag
Jan 22, 2021
984805e
Small refactoring
Jan 22, 2021
71b2e42
Change name
Jan 22, 2021
549488e
Merge remote-tracking branch 'upstream/master' into dev/jachocho/tele…
Jan 22, 2021
6a446aa
Upgrade CC comp
Jan 22, 2021
14ff291
Move to guids
Jan 25, 2021
33d386c
Revert lang change
Jan 25, 2021
8e5a82a
Fixing descriptions
Jan 25, 2021
393d8b7
Last changes
Jan 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Tests
  • Loading branch information
Jakub Chocholowicz committed Jan 20, 2021
commit acb01d4964644c950e3452a5368e833c368c0448
25 changes: 16 additions & 9 deletions src/Microsoft.TestPlatform.Common/DataCollection/DataCollectionTelemetryManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@

namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector
{
/// <summary>
/// Stores and provides telemetry information for data collection.
/// </summary>
internal class DataCollectionTelemetryManager : IDataCollectionTelemetryManager
AbhitejJohn marked this conversation as resolved.
Show resolved Hide resolved
{
private const string CorProfilerVariable = "COR_PROFILER";
Expand All @@ -17,7 +20,7 @@ internal class DataCollectionTelemetryManager : IDataCollectionTelemetryManager

private const string VanguardProfilerGuid = "{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}";
private const string ClrIeProfilerGuid = "{324F817A-7420-4E6D-B3C1-143FBED6D855}";
private const string IntellitraceProfilerGuid = "{324F817A-7420-4E6D-B3C1-143FBED6D855}";
private const string IntellitraceProfilerGuid = "{9317ae81-bcd8-47b7-aaa1-a28062e41c71}";

private const string VanguardProfilerName = "vanguard";
private const string ClrIeProfilerName = "clrie";
Expand All @@ -35,18 +38,26 @@ internal DataCollectionTelemetryManager(IRequestData requestData)
this.requestData = requestData;
}

/// <inheritdoc/>
public void OnEnvironmentVariableAdded(DataCollectorInformation dataCollectorInformation, string name, string value)
{
AddProfilerMetricForNewVariable(CorProfilerVariable, CorProfilerTelemetryTemplate, dataCollectorInformation, name, value);
AddProfilerMetricForNewVariable(CoreClrProfilerVariable, CoreClrProfilerTelemetryTemplate, dataCollectorInformation, name, value);
}

/// <inheritdoc/>
public void OnEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string existingValue)
{
AddProfilerMetricForConflictedVariable(CorProfilerVariable, CorProfilerTelemetryTemplate, dataCollectorInformation, name, existingValue);
AddProfilerMetricForConflictedVariable(CoreClrProfilerVariable, CoreClrProfilerTelemetryTemplate, dataCollectorInformation, name, existingValue);
}

/// <inheritdoc/>
public IRequestData GetRequestData()
{
return requestData;
}

private void AddProfilerMetricForNewVariable(string profilerVariable, string telemetryTemplateName, DataCollectorInformation dataCollectorInformation, string name, string value)
{
if (!string.Equals(profilerVariable, name, StringComparison.Ordinal))
Expand All @@ -66,15 +77,16 @@ private void AddProfilerMetricForConflictedVariable(string profilerVariable, str

if (string.Equals(existingValue, ClrIeProfilerGuid, StringComparison.OrdinalIgnoreCase))
{
if (dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix32Variable)) &&
if (dataCollectorInformation.TestExecutionEnvironmentVariables != null &&
dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix32Variable)) &&
jakubch1 marked this conversation as resolved.
Show resolved Hide resolved
dataCollectorInformation.TestExecutionEnvironmentVariables.Any(pair => pair.Key.StartsWith(ClrIeInstrumentationMethodConfigurationPrefix64Variable)))
{
requestData.MetricsCollection.Add(string.Format(telemetryTemplateName, dataCollectorInformation.DataCollectorConfig.ExtensionUri.ToString()), ClrIeProfilerName);
requestData.MetricsCollection.Add(string.Format(telemetryTemplateName, dataCollectorInformation.DataCollectorConfig?.TypeUri?.ToString()), ClrIeProfilerName);
return;
}
}

requestData.MetricsCollection.Add(string.Format(telemetryTemplateName, dataCollectorInformation.DataCollectorConfig.ExtensionUri.ToString()), OverwrittenProfilerName);
requestData.MetricsCollection.Add(string.Format(telemetryTemplateName, dataCollectorInformation.DataCollectorConfig?.TypeUri?.ToString()), OverwrittenProfilerName);
AbhitejJohn marked this conversation as resolved.
Show resolved Hide resolved
}

private static string GetProfilerName(string profilerGuid)
Expand All @@ -94,10 +106,5 @@ private static string GetProfilerName(string profilerGuid)

return UnknownProfilerName;
}

public IRequestData GetRequestData()
{
return requestData;
}
}
}
36 changes: 35 additions & 1 deletion ...icrosoft.TestPlatform.Common/DataCollection/Interfaces/IDataCollectionTelemetryManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,49 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using Microsoft.VisualStudio.TestPlatform.Common.DataCollector;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;

namespace Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces
{
/// <summary>
/// The IDataCollectionTelemetryManager Interface.
/// </summary>
internal interface IDataCollectionTelemetryManager
{
/// <summary>
/// Stores telemetry regarding environment variable added.
/// </summary>
/// <param name="dataCollectorInformation">
/// Data collector information which requested environment variable.
/// </param>
/// <param name="name">
/// Environment variable name.
/// </param>
/// <param name="value">
/// Environment variable value.
/// </param>
void OnEnvironmentVariableAdded(DataCollectorInformation dataCollectorInformation, string name, string value);

/// <summary>
/// Stores telemetry regarding environment variable is conflicting.
/// </summary>
/// <param name="dataCollectorInformation">
/// Data collector information which requested environment variable.
/// </param>
/// <param name="name">
/// Environment variable name.
/// </param>
/// <param name="existingValue">
/// Environment variable value that was requested previously.
/// </param>
void OnEnvironmentVariableConflict(DataCollectorInformation dataCollectorInformation, string name, string existingValue);

/// <summary>
/// Provides telemetry object.
/// </summary>
/// <returns>
/// IRequestData object.
/// </returns>
IRequestData GetRequestData();
}
}
171 changes: 171 additions & 0 deletions test/datacollector.UnitTests/DataCollectionTelemetryManagerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
using Microsoft.VisualStudio.TestPlatform.Common.DataCollector;
using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.Interfaces;
using Microsoft.VisualStudio.TestPlatform.Common.DataCollector.UnitTests;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using System.Collections.Generic;

namespace Microsoft.VisualStudio.TestPlatform.DataCollector.UnitTests
{
[TestClass]
public class DataCollectionTelemetryManagerTests
{
private readonly Mock<IRequestData> mockRequestData;
private readonly Mock<IMetricsCollection> mockMetricsCollection;
private readonly DataCollectionTelemetryManager telemetryManager;
private readonly DataCollectorInformation dataCollectorInformation;

public DataCollectionTelemetryManagerTests()
{
this.mockRequestData = new Mock<IRequestData>();
this.mockMetricsCollection = new Mock<IMetricsCollection>();
this.mockRequestData.Setup(m => m.MetricsCollection).Returns(this.mockMetricsCollection.Object);

var dataCollectorMock = new Mock<CodeCoverageDataCollector>();
var evnVariablesMock = dataCollectorMock.As<ITestExecutionEnvironmentSpecifier>();
evnVariablesMock.Setup(a => a.GetTestExecutionEnvironmentVariables()).Returns(new KeyValuePair<string, string>[]
{
new KeyValuePair<string, string>("MicrosoftInstrumentationEngine_ConfigPath32_VanguardInstrumentationProfiler", "path1"),
new KeyValuePair<string, string>("MicrosoftInstrumentationEngine_ConfigPath64_VanguardInstrumentationProfiler", "path2")
});

this.dataCollectorInformation = new DataCollectorInformation(
dataCollectorMock.Object,
null,
new DataCollectorConfig(typeof(CustomDataCollector)),
null,
new Mock<IDataCollectionAttachmentManager>().Object,
new TestPlatformDataCollectionEvents(),
new Mock<IMessageSink>().Object,
string.Empty);

this.telemetryManager = new DataCollectionTelemetryManager(this.mockRequestData.Object);
}

[TestMethod]
public void GetRequestData_ShouldReturnCorrectObject()
{
// act
var result = this.telemetryManager.GetRequestData();

// assert
Assert.AreEqual(this.mockRequestData.Object, result);
}

[TestMethod]
public void OnEnvironmentVariableAdded_ShouldDoNothing_IfNotProfilerVariable()
{
// act
this.telemetryManager.OnEnvironmentVariableAdded(this.dataCollectorInformation, "key", "value");

// assert
this.mockMetricsCollection.Verify(c => c.Add(It.IsAny<string>(), It.IsAny<object>()), Times.Never);
}

[TestMethod]
public void OnEnvironmentVariableConflict_ShouldDoNothing_IfNotProfilerVariable()
{
// act
this.telemetryManager.OnEnvironmentVariableConflict(this.dataCollectorInformation, "key", "value");

// assert
this.mockMetricsCollection.Verify(c => c.Add(It.IsAny<string>(), It.IsAny<object>()), Times.Never);
}

[TestMethod]
[DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "vanguard")]
[DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}", "clrie")]
[DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "intellitrace")]
[DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "unknown")]
[DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "vanguard")]
[DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "clrie")]
[DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "intellitrace")]
public void OnEnvironmentVariableAdded_ShouldCollectTelemetry_IfCorProfilerVariable(string profilerGuid, string profilerName)
{
// act
this.telemetryManager.OnEnvironmentVariableAdded(this.dataCollectorInformation, "COR_PROFILER", profilerGuid);

// assert
this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.{dataCollectorInformation.DataCollectorConfig.TypeUri}.CorProfiler", profilerName), Times.Once);
}

[TestMethod]
[DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}", "vanguard")]
[DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}", "clrie")]
[DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}", "intellitrace")]
[DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}", "unknown")]
[DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}", "vanguard")]
[DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}", "clrie")]
[DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}", "intellitrace")]
public void OnEnvironmentVariableAdded_ShouldCollectTelemetry_IfCoreClrProfilerVariable(string profilerGuid, string profilerName)
{
// act
this.telemetryManager.OnEnvironmentVariableAdded(this.dataCollectorInformation, "CORECLR_PROFILER", profilerGuid);

// assert
this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.{dataCollectorInformation.DataCollectorConfig.TypeUri}.CoreClrProfiler", profilerName), Times.Once);
}

[TestMethod]
[DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}")]
[DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}")]
[DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}")]
[DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}")]
[DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}")]
[DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}")]
[DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}")]
public void OnEnvironmentVariableConflict_ShouldCollectOverwrittenTelemetry_IfCorProfilerVariable(string profilerGuid)
{
// act
this.telemetryManager.OnEnvironmentVariableConflict(this.dataCollectorInformation, "COR_PROFILER", profilerGuid);

// assert
this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.{dataCollectorInformation.DataCollectorConfig.TypeUri}.CorProfiler", "overwritten"), Times.Once);
}

[TestMethod]
[DataRow("{E5F256DC-7959-4DD6-8E4F-C11150AB28E0}")]
[DataRow("{324F817A-7420-4E6D-B3C1-143FBED6D855}")]
[DataRow("{9317ae81-bcd8-47b7-aaa1-a28062e41c71}")]
[DataRow("{aaaaaaaa-bcd8-47b7-aaa1-a28062e41c71}")]
[DataRow("{E5F256DC-7959-4DD6-8E4F-c11150AB28E0}")]
[DataRow("{324f817a-7420-4e6d-b3c1-143fbEd6d855}")]
[DataRow("{9317AE81-bcd8-47b7-AAA1-A28062E41C71}")]
public void OnEnvironmentVariableConflict_ShouldCollectOverwrittenTelemetry_IfCoreClrProfilerVariable(string profilerGuid)
{
// act
this.telemetryManager.OnEnvironmentVariableConflict(this.dataCollectorInformation, "CORECLR_PROFILER", profilerGuid);

// assert
this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.{dataCollectorInformation.DataCollectorConfig.TypeUri}.CoreClrProfiler", "overwritten"), Times.Once);
}

[TestMethod]
public void OnEnvironmentVariableConflict_ShouldCollectClrIeTelemetry_IfCorProfilerVariableAndCollectorSpecifiesClrIeProfile()
{
// arrange
this.dataCollectorInformation.SetTestExecutionEnvironmentVariables();

// act
this.telemetryManager.OnEnvironmentVariableConflict(this.dataCollectorInformation, "COR_PROFILER", "{324F817A-7420-4E6D-B3C1-143FBED6D855}");

// assert
this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.{dataCollectorInformation.DataCollectorConfig.TypeUri}.CorProfiler", "clrie"), Times.Once);
}

[TestMethod]
public void OnEnvironmentVariableConflict_ShouldCollectClrIeTelemetry_IfCoreClrProfilerVariableAndCollectorSpecifiesClrIeProfile()
{
// arrange
this.dataCollectorInformation.SetTestExecutionEnvironmentVariables();

// act
this.telemetryManager.OnEnvironmentVariableConflict(this.dataCollectorInformation, "CORECLR_PROFILER", "{324F817A-7420-4E6D-B3C1-143FBED6D855}");

// assert
this.mockMetricsCollection.Verify(c => c.Add($"VS.TestPlatform.DataCollector.{dataCollectorInformation.DataCollectorConfig.TypeUri}.CoreClrProfiler", "clrie"), Times.Once);
}
}
}