Add telemetry for testing usage (#16913)

* Add telemetry for testing usage

* Improve trigger telemetry.

Author: karthiknadig

src/client/common/constants.ts

@@ -23,11 +23,8 @@ export const AppinsightsKey = 'AIF-d9b70cd4-b9f9-4d70-929b-a071c400b217';
 export type Channel = 'stable' | 'insiders';
 
 export enum CommandSource {
-    auto = 'auto',
     ui = 'ui',
-    codelens = 'codelens',
     commandPalette = 'commandpalette',
-    testExplorer = 'testExplorer',
 }
 
 export namespace Commands {

src/client/telemetry/constants.ts

@@ -57,17 +57,18 @@ export enum EventName {
     DEBUGGER_ATTACH_TO_LOCAL_PROCESS = 'DEBUGGER.ATTACH_TO_LOCAL_PROCESS',
     DEBUGGER_CONFIGURATION_PROMPTS = 'DEBUGGER.CONFIGURATION.PROMPTS',
     DEBUGGER_CONFIGURATION_PROMPTS_IN_LAUNCH_JSON = 'DEBUGGER.CONFIGURATION.PROMPTS.IN.LAUNCH.JSON',
-    UNITTEST_STOP = 'UNITTEST.STOP',
-    UNITTEST_DISABLE = 'UNITTEST.DISABLE',
-    UNITTEST_RUN = 'UNITTEST.RUN',
-    UNITTEST_DISCOVER = 'UNITTEST.DISCOVER',
-    UNITTEST_DISCOVER_WITH_PYCODE = 'UNITTEST.DISCOVER.WITH.PYTHONCODE',
-    UNITTEST_CONFIGURE = 'UNITTEST.CONFIGURE',
+
+    // Python testing specific telemetry
     UNITTEST_CONFIGURING = 'UNITTEST.CONFIGURING',
-    UNITTEST_VIEW_OUTPUT = 'UNITTEST.VIEW_OUTPUT',
-    UNITTEST_NAVIGATE = 'UNITTEST.NAVIGATE',
-    UNITTEST_ENABLED = 'UNITTEST.ENABLED',
-    UNITTEST_EXPLORER_WORK_SPACE_COUNT = 'UNITTEST.TEST_EXPLORER.WORK_SPACE_COUNT',
+    UNITTEST_CONFIGURE = 'UNITTEST.CONFIGURE',
+    UNITTEST_DISCOVERY_TRIGGER = 'UNITTEST.DISCOVERY.TRIGGER',
+    UNITTEST_DISCOVERING = 'UNITTEST.DISCOVERING',
+    UNITTEST_DISCOVERING_STOP = 'UNITTEST.DISCOVERY.STOP',
+    UNITTEST_DISCOVERY_DONE = 'UNITTEST.DISCOVERY.DONE',
+    UNITTEST_RUN_STOP = 'UNITTEST.RUN.STOP',
+    UNITTEST_RUN = 'UNITTEST.RUN',
+    UNITTEST_RUN_ALL_FAILED = 'UNITTEST.RUN_ALL_FAILED',
+    UNITTEST_DISABLED = 'UNITTEST.DISABLED',
 
     PYTHON_EXPERIMENTS_OPT_IN_OPT_OUT_SETTINGS = 'PYTHON_EXPERIMENTS_OPT_IN_OPT_OUT_SETTINGS',
     EXTENSION_SURVEY_PROMPT = 'EXTENSION_SURVEY_PROMPT',

src/client/telemetry/index.ts

@@ -24,7 +24,6 @@ import {
     TensorBoardSessionStartResult,
     TensorBoardEntrypoint,
 } from '../tensorBoard/constants';
-import { TestProvider } from '../testing/types';
 import { EventName, PlatformErrors } from './constants';
 import type { LinterTrigger, TestTool } from './types';
 import { JupyterNotInstalledOrigin } from '../jupyter/types';
@@ -1507,65 +1506,53 @@ export interface IEventNamePropertyMapping {
         interpreterType?: EnvironmentType;
     };
     /**
-     * Telemetry event sent with details about discovering tests
+     * Telemetry event sent indicating the trigger source for discovery.
      */
-    [EventName.UNITTEST_DISCOVER]: {
-        /**
-         * The test framework used to discover tests
-         *
-         * @type {TestTool}
-         */
-        tool: TestTool;
+    [EventName.UNITTEST_DISCOVERY_TRIGGER]: {
         /**
          * Carries the source which triggered discovering of tests
          *
-         * @type {('ui' | 'commandpalette')}
+         * @type {('auto' | 'ui' | 'commandpalette' | 'watching' | 'interpreter')}
+         * auto           : Triggered by VS Code editor.
+         * ui             : Triggered by clicking a button.
+         * commandpalette : Triggered by running the command from the command palette.
+         * watching       : Triggered by filesystem or content changes.
+         * interpreter    : Triggered by interpreter change.
          */
-        trigger: 'ui' | 'commandpalette';
-        /**
-         * Carries `true` if discovering tests failed, `false` otherwise
-         *
-         * @type {boolean}
-         */
-        failed: boolean;
+        trigger: 'auto' | 'ui' | 'commandpalette' | 'watching' | 'interpreter';
     };
     /**
-     * Telemetry event is sent if we are doing test discovery using python code
-     */
-    [EventName.UNITTEST_DISCOVER_WITH_PYCODE]: never | undefined;
-    /**
-     * Telemetry event sent when user clicks a file, function, or suite in test explorer.
+     * Telemetry event sent with details about discovering tests
      */
-    [EventName.UNITTEST_NAVIGATE]: {
+    [EventName.UNITTEST_DISCOVERING]: {
         /**
-         * Carries `true` if user clicks a file, `false` otherwise
-         *
-         * @type {boolean}
-         */
-        byFile?: boolean;
-        /**
-         * Carries `true` if user clicks a function, `false` otherwise
+         * The test framework used to discover tests
          *
-         * @type {boolean}
+         * @type {TestTool}
          */
-        byFunction?: boolean;
+        tool: TestTool;
+    };
+    /**
+     * Telemetry event sent with details about discovering tests
+     */
+    [EventName.UNITTEST_DISCOVERY_DONE]: {
         /**
-         * Carries `true` if user clicks a suite, `false` otherwise
+         * The test framework used to discover tests
          *
-         * @type {boolean}
+         * @type {TestTool}
          */
-        bySuite?: boolean;
+        tool: TestTool;
         /**
-         * Carries `true` if we are changing focus to the suite/file/function, `false` otherwise
+         * Carries `true` if discovering tests failed, `false` otherwise
          *
          * @type {boolean}
          */
-        focusCode?: boolean;
+        failed: boolean;
     };
     /**
-     * Tracks number of workspace folders shown in test explorer
+     * Telemetry event sent when cancelling discovering tests
      */
-    [EventName.UNITTEST_EXPLORER_WORK_SPACE_COUNT]: { count: number };
+    [EventName.UNITTEST_DISCOVERING_STOP]: never | undefined;
     /**
      * Telemetry event sent with details about running the tests, what is being run, what framework is being used etc.
      */
@@ -1574,45 +1561,23 @@ export interface IEventNamePropertyMapping {
          * Framework being used to run tests
          */
         tool: TestTool;
-        /**
-         * Carries info what is being run
-         */
-        scope: 'currentFile' | 'all' | 'file' | 'class' | 'function' | 'failed';
         /**
          * Carries `true` if debugging, `false` otherwise
          */
         debugging: boolean;
-        /**
-         * Carries what triggered the execution of the tests
-         */
-        triggerSource: 'ui' | 'codelens' | 'commandpalette' | 'auto' | 'testExplorer';
-        /**
-         * Carries `true` if running tests failed, `false` otherwise
-         */
-        failed: boolean;
     };
     /**
-     * Telemetry event sent when cancelling running or discovering tests
+     * Telemetry event sent when cancelling running tests
      */
-    [EventName.UNITTEST_STOP]: never | undefined;
+    [EventName.UNITTEST_RUN_STOP]: never | undefined;
     /**
-     * Telemetry event sent when disabling all test frameworks
+     * Telemetry event sent when run all failed test command is triggered
      */
-    [EventName.UNITTEST_DISABLE]: never | undefined;
+    [EventName.UNITTEST_RUN_ALL_FAILED]: never | undefined;
     /**
-     * Telemetry event sent when viewing Python test log output
-     */
-    [EventName.UNITTEST_VIEW_OUTPUT]: never | undefined;
-    /**
-     * Tracks which testing framework has been enabled by the user.
-     * Telemetry is sent when settings have been modified by the user.
-     * Values sent include:
-     * unittest -   If this value is `true`, then unittest has been enabled by the user.
-     * pytest   -   If this value is `true`, then pytest has been enabled by the user.
-     * @type {(never | undefined)}
-     * @memberof IEventNamePropertyMapping
+     * Telemetry event sent when testing is disabled for a workspace.
      */
-    [EventName.UNITTEST_ENABLED]: Partial<Record<TestProvider, undefined | boolean>>;
+    [EventName.UNITTEST_DISABLED]: never | undefined;
     /**
      * Telemetry sent when building workspace symbols
      */

src/client/telemetry/types.ts

@@ -16,7 +16,7 @@ export type PythonInterpreterTelemetry = IEventNamePropertyMapping[EventName.PYT
 export type DebuggerTelemetry = IEventNamePropertyMapping[EventName.DEBUGGER];
 export type TestTool = 'pytest' | 'unittest';
 export type TestRunTelemetry = IEventNamePropertyMapping[EventName.UNITTEST_RUN];
-export type TestDiscoveryTelemetry = IEventNamePropertyMapping[EventName.UNITTEST_DISCOVER];
+export type TestDiscoveryTelemetry = IEventNamePropertyMapping[EventName.UNITTEST_DISCOVERY_DONE];
 export type TestConfiguringTelemetry = IEventNamePropertyMapping[EventName.UNITTEST_CONFIGURING];
 export const IImportTracker = Symbol('IImportTracker');
 export interface IImportTracker {}

src/client/testing/main.ts

@@ -9,7 +9,7 @@ import { IDisposableRegistry, IExperimentService, Product } from '../common/type
 import { IInterpreterService } from '../interpreter/contracts';
 import { IServiceContainer } from '../ioc/types';
 import { EventName } from '../telemetry/constants';
-import { captureTelemetry } from '../telemetry/index';
+import { captureTelemetry, sendTelemetryEvent } from '../telemetry/index';
 import { selectTestWorkspace } from './common/testUtils';
 import { TestSettingsPropertyNames } from './configuration/types';
 import { ITestConfigurationService, ITestsHelper } from './common/types';
@@ -155,10 +155,13 @@ export class UnitTestManagementService implements IExtensionActivationService {
                 constants.Commands.Test_Refresh,
                 async (
                     _,
-                    _cmdSource: constants.CommandSource = constants.CommandSource.commandPalette,
+                    cmdSource: constants.CommandSource = constants.CommandSource.commandPalette,
                     resource?: Uri,
                 ) => {
                     traceVerbose('Testing: Manually triggered test refresh');
+                    sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, {
+                        trigger: cmdSource,
+                    });
                     this.testController?.refreshTestData(resource, { forceRefresh: true });
                 },
             ),
@@ -168,6 +171,7 @@ export class UnitTestManagementService implements IExtensionActivationService {
             }),
             commandManager.registerCommand(constants.Commands.Test_Stop_Refreshing, () => {
                 traceVerbose('Testing: Stop refreshing clicked.');
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERING_STOP);
                 this.testController?.stopRefreshing();
             }),
         );
@@ -181,6 +185,7 @@ export class UnitTestManagementService implements IExtensionActivationService {
             }),
             interpreterService.onDidChangeInterpreter(async () => {
                 traceVerbose('Testing: Triggered refresh due to interpreter change.');
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'interpreter' });
                 await this.testController?.refreshTestData(undefined, { forceRefresh: true });
             }),
         );

src/client/testing/testController/controller.ts

@@ -20,6 +20,8 @@ import { IWorkspaceService } from '../../common/application/types';
 import { traceVerbose } from '../../common/logger';
 import { IConfigurationService, IDisposableRegistry, Resource } from '../../common/types';
 import { DelayedTrigger, IDelayedTrigger } from '../../common/utils/delayTrigger';
+import { sendTelemetryEvent } from '../../telemetry';
+import { EventName } from '../../telemetry/constants';
 import { PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../common/constants';
 import { getNodeByUri } from './common/testItemUtilities';
 import { ITestController, ITestFrameworkController, TestRefreshOptions } from './common/types';
@@ -119,6 +121,7 @@ export class PythonTestController implements ITestController {
             } else if (settings.testing.unittestEnabled) {
                 await this.unittest.refreshTestData(this.testController, uri, this.refreshCancellation.token);
             } else {
+                sendTelemetryEvent(EventName.UNITTEST_DISABLED);
                 // If we are here we may have to remove an existing node from the tree
                 // This handles the case where user removes test settings. Which should remove the
                 // tests for that particular case from the tree view
@@ -155,6 +158,7 @@ export class PythonTestController implements ITestController {
             }
         } else {
             traceVerbose('Testing: Refreshing all test data');
+            sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'auto' });
             const workspaces: readonly WorkspaceFolder[] = this.workspaceService.workspaceFolders || [];
             await Promise.all(workspaces.map((workspace) => this.refreshTestDataInternal(workspace.uri)));
         }
@@ -198,6 +202,10 @@ export class PythonTestController implements ITestController {
                     if (testItems.length > 0) {
                         const settings = this.configSettings.getSettings(workspace.uri);
                         if (settings.testing.pytestEnabled) {
+                            sendTelemetryEvent(EventName.UNITTEST_RUN, undefined, {
+                                tool: 'pytest',
+                                debugging: request.profile?.kind === TestRunProfileKind.Debug,
+                            });
                             return this.pytest.runTests(
                                 {
                                     includes: testItems,
@@ -210,6 +218,10 @@ export class PythonTestController implements ITestController {
                             );
                         }
                         if (settings.testing.unittestEnabled) {
+                            sendTelemetryEvent(EventName.UNITTEST_RUN, undefined, {
+                                tool: 'unittest',
+                                debugging: request.profile?.kind === TestRunProfileKind.Debug,
+                            });
                             return this.unittest.runTests(
                                 {
                                     includes: testItems,
@@ -264,18 +276,21 @@ export class PythonTestController implements ITestController {
         this.disposables.push(
             watcher.onDidChange((uri) => {
                 traceVerbose(`Testing: Trigger refresh after change in ${uri.fsPath}`);
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'watching' });
                 this.refreshData.trigger(uri, false);
             }),
         );
         this.disposables.push(
             watcher.onDidCreate((uri) => {
                 traceVerbose(`Testing: Trigger refresh after creating ${uri.fsPath}`);
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'watching' });
                 this.refreshData.trigger(uri, false);
             }),
         );
         this.disposables.push(
             watcher.onDidDelete((uri) => {
                 traceVerbose(`Testing: Trigger refresh after deleting in ${uri.fsPath}`);
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'watching' });
                 this.refreshData.trigger(uri, false);
             }),
         );
@@ -289,19 +304,22 @@ export class PythonTestController implements ITestController {
         this.disposables.push(
             watcher.onDidChange((uri) => {
                 traceVerbose(`Testing: Trigger refresh after change in ${uri.fsPath}`);
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'watching' });
                 // We want to invalidate tests for code change
                 this.refreshData.trigger(uri, true);
             }),
         );
         this.disposables.push(
             watcher.onDidCreate((uri) => {
                 traceVerbose(`Testing: Trigger refresh after creating ${uri.fsPath}`);
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'watching' });
                 this.refreshData.trigger(uri, false);
             }),
         );
         this.disposables.push(
             watcher.onDidDelete((uri) => {
                 traceVerbose(`Testing: Trigger refresh after deleting in ${uri.fsPath}`);
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_TRIGGER, undefined, { trigger: 'watching' });
                 this.refreshData.trigger(uri, false);
             }),
         );

src/client/testing/testController/pytest/pytestController.ts

@@ -11,6 +11,8 @@ import { traceError } from '../../../common/logger';
 import { runAdapter } from '../../../common/process/internal/scripts/testing_tools';
 import { IConfigurationService } from '../../../common/types';
 import { createDeferred, Deferred } from '../../../common/utils/async';
+import { sendTelemetryEvent } from '../../../telemetry';
+import { EventName } from '../../../telemetry/constants';
 import { PYTEST_PROVIDER } from '../../common/constants';
 import { TestDiscoveryOptions } from '../../common/types';
 import {
@@ -142,6 +144,7 @@ export class PytestController implements ITestFrameworkController {
     }
 
     public async refreshTestData(testController: TestController, uri: Uri, token?: CancellationToken): Promise<void> {
+        sendTelemetryEvent(EventName.UNITTEST_DISCOVERING, undefined, { tool: 'pytest' });
         const workspace = this.workspaceService.getWorkspaceFolder(uri);
         if (workspace) {
             // Discovery is expensive. So if it is already running then use the promise
@@ -199,6 +202,7 @@ export class PytestController implements ITestFrameworkController {
 
                 deferred.resolve();
             } catch (ex) {
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: 'pytest', failed: true });
                 const cancel = options.token?.isCancellationRequested ? 'Cancelled' : 'Error';
                 traceError(`${cancel} discovering pytest tests:\r\n`, ex);
                 const message = getTestDiscoveryExceptions((ex as Error).message);
@@ -252,6 +256,7 @@ export class PytestController implements ITestFrameworkController {
                 await this.resolveChildren(testController, newItem);
             }
         }
+        sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: 'pytest', failed: false });
         return Promise.resolve();
     }
 

src/client/testing/testController/unittest/unittestController.ts

@@ -29,6 +29,8 @@ import {
     updateTestItemFromRawData,
 } from '../common/testItemUtilities';
 import { traceError } from '../../../common/logger';
+import { sendTelemetryEvent } from '../../../telemetry';
+import { EventName } from '../../../telemetry/constants';
 
 @injectable()
 export class UnittestController implements ITestFrameworkController {
@@ -89,6 +91,7 @@ export class UnittestController implements ITestFrameworkController {
     }
 
     public async refreshTestData(testController: TestController, uri: Uri, token?: CancellationToken): Promise<void> {
+        sendTelemetryEvent(EventName.UNITTEST_DISCOVERING, undefined, { tool: 'unittest' });
         const workspace = this.workspaceService.getWorkspaceFolder(uri);
         if (workspace) {
             // Discovery is expensive. So if it is already running then use the promise
@@ -199,6 +202,7 @@ for error in loader_errors:
 
                 deferred.resolve();
             } catch (ex) {
+                sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: 'unittest', failed: true });
                 const cancel = options.token?.isCancellationRequested ? 'Cancelled' : 'Error';
                 traceError(`${cancel} discovering unittest tests:\r\n`, ex);
 
@@ -253,6 +257,7 @@ for error in loader_errors:
                 await this.resolveChildren(testController, newItem);
             }
         }
+        sendTelemetryEvent(EventName.UNITTEST_DISCOVERY_DONE, undefined, { tool: 'unittest', failed: false });
         return Promise.resolve();
     }