Skip to content

Commit 3a7c65d

Browse files
eleanorjboydCopilot
eleanorjboyd
and
Copilot
committed
Add Environments integration decision telemetry
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent f4c86e4 commit 3a7c65d

8 files changed

Lines changed: 217 additions & 12 deletions

File tree

‎src/client/envExt/api.internal.ts‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { executeCommand } from '../common/vscodeApis/commandApis';
1616
import { getConfiguration, getWorkspaceFolders } from '../common/vscodeApis/workspaceApis';
1717
import { traceError, traceLog } from '../logging';
1818
import { Interpreters } from '../common/utils/localize';
19+
import type { EnvsIntegrationDecisionReason, EnvsIntegrationDecisionTelemetry } from '../telemetry/types';
1920

2021
export const ENVS_EXTENSION_ID = 'ms-python.vscode-python-envs';
2122

@@ -54,18 +55,53 @@ export function shouldEnvExtHandleActivation(): boolean {
5455
return true;
5556
}
5657

58+
function getEnvExtensionDecisionInputs(): Pick<
59+
EnvsIntegrationDecisionTelemetry,
60+
'envsAvailableToHostNow' | 'envsResolvedSettingNow'
61+
> {
62+
const config = getConfiguration('python');
63+
return {
64+
envsAvailableToHostNow: !!getExtension(ENVS_EXTENSION_ID),
65+
envsResolvedSettingNow: config?.get<boolean>('useEnvironmentsExtension', false) ?? false,
66+
};
67+
}
68+
5769
let _useExt: boolean | undefined;
70+
let _envsDecisionReason: EnvsIntegrationDecisionReason | undefined;
5871
export function useEnvExtension(): boolean {
5972
if (_useExt !== undefined) {
6073
return _useExt;
6174
}
62-
const config = getConfiguration('python');
63-
const inExpSetting = config?.get<boolean>('useEnvironmentsExtension', false) ?? false;
75+
const { envsAvailableToHostNow, envsResolvedSettingNow } = getEnvExtensionDecisionInputs();
6476
// If extension is installed and in experiment, then use it.
65-
_useExt = !!getExtension(ENVS_EXTENSION_ID) && inExpSetting;
77+
_useExt = envsAvailableToHostNow && envsResolvedSettingNow;
78+
_envsDecisionReason = !envsAvailableToHostNow
79+
? 'extensionUnavailable'
80+
: envsResolvedSettingNow
81+
? 'enabled'
82+
: 'resolvedSettingFalse';
6683
return _useExt;
6784
}
6885

86+
/**
87+
* Reports the cached integration decision inputs and their current values without initializing or changing the cache.
88+
*/
89+
export function getEnvExtensionDecisionTelemetry(): EnvsIntegrationDecisionTelemetry {
90+
return {
91+
envsDecisionReason: _envsDecisionReason,
92+
...getEnvExtensionDecisionInputs(),
93+
envsCachedDecision: _useExt,
94+
};
95+
}
96+
97+
/**
98+
* Resets the cached integration decision for unit tests.
99+
*/
100+
export function _resetEnvExtensionDecisionCache(): void {
101+
_useExt = undefined;
102+
_envsDecisionReason = undefined;
103+
}
104+
69105
const onDidChangeEnvironmentEnvExtEmitter: EventEmitter<DidChangeEnvironmentEventArgs> = new EventEmitter<
70106
DidChangeEnvironmentEventArgs
71107
>();

‎src/client/envExt/telemetry.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
import { IDisposableRegistry } from '../common/types';
55
import { getConfiguration, getWorkspaceFolders, onDidChangeConfiguration } from '../common/vscodeApis/workspaceApis';
6+
import { getEnvExtensionDecisionTelemetry } from './api.internal';
67
import { sendTelemetryEvent } from '../telemetry';
78
import { EventName } from '../telemetry/constants';
89
import { EnvsExplicitFalseScope } from '../telemetry/types';
@@ -45,6 +46,7 @@ export function registerEnvironmentsExtensionTelemetry(disposables: IDisposableR
4546
if (event.affectsConfiguration('python.useEnvironmentsExtension')) {
4647
sendTelemetryEvent(EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED, undefined, {
4748
envsExplicitFalseScope: getEnvsExplicitFalseScope(),
49+
...getEnvExtensionDecisionTelemetry(),
4850
});
4951
}
5052
}),

‎src/client/startupTelemetry.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,18 @@ import { sendTelemetryEvent } from './telemetry';
1616
import { EventName } from './telemetry/constants';
1717
import { EditorLoadTelemetry } from './telemetry/types';
1818
import { IStartupDurations } from './types';
19-
import { useEnvExtension } from './envExt/api.internal';
19+
import { getEnvExtensionDecisionTelemetry, useEnvExtension } from './envExt/api.internal';
2020
import { getEnvsExplicitFalseScope } from './envExt/telemetry';
2121

22+
function getEnvExtensionStartupTelemetry() {
23+
const telemetry = getEnvExtensionDecisionTelemetry();
24+
return {
25+
envsDecisionReason: telemetry.envsDecisionReason,
26+
envsAvailableToHostNow: telemetry.envsAvailableToHostNow,
27+
envsResolvedSettingNow: telemetry.envsResolvedSettingNow,
28+
};
29+
}
30+
2231
export async function sendStartupTelemetry(
2332
activatedPromise: Promise<any>,
2433
durations: IStartupDurations,
@@ -99,6 +108,7 @@ async function getActivationTelemetryProps(
99108
terminal: terminalShellType,
100109
isFirstSession,
101110
envsExplicitFalseScope: getEnvsExplicitFalseScope(),
111+
...getEnvExtensionStartupTelemetry(),
102112
};
103113
}
104114
const interpreterService = serviceContainer.get<IInterpreterService>(IInterpreterService);
@@ -157,5 +167,6 @@ async function getActivationTelemetryProps(
157167
isFirstSession,
158168
usingEnvironmentsExtension,
159169
envsExplicitFalseScope: getEnvsExplicitFalseScope(),
170+
...getEnvExtensionStartupTelemetry(),
160171
};
161172
}

‎src/client/telemetry/index.ts‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { EnvironmentType, PythonEnvironment } from '../pythonEnvironments/info';
1313
import { TensorBoardPromptSelection } from '../tensorBoard/constants';
1414
import { EventName } from './constants';
1515
import type { UnitTestRunFailureCategory } from './constants';
16-
import type { EnvsExplicitFalseScope, TestTool } from './types';
16+
import type { EnvsExplicitFalseScope, EnvsIntegrationDecisionReason, TestTool } from './types';
1717

1818
/**
1919
* Checks whether telemetry is supported.
@@ -360,7 +360,10 @@ export interface IEventNamePropertyMapping {
360360
"usingglobalinterpreter" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" },
361361
"isfirstsession" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" },
362362
"usingenvironmentsextension" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eduardovil" },
363-
"envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
363+
"envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
364+
"envsdecisionreason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
365+
"envsavailabletohostnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
366+
"envsresolvedsettingnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
364367
}
365368
*/
366369
[EventName.EDITOR_LOAD]: {
@@ -416,18 +419,38 @@ export interface IEventNamePropertyMapping {
416419
* 'none' excludes defaults; 'multiple' means more than one of user, workspace, or folder.
417420
*/
418421
envsExplicitFalseScope?: EnvsExplicitFalseScope;
422+
/**
423+
* The inputs that produced the cached Python Environments integration decision.
424+
*/
425+
envsDecisionReason?: EnvsIntegrationDecisionReason;
426+
/**
427+
* Whether Python Environments is currently available to this extension host.
428+
*/
429+
envsAvailableToHostNow?: boolean;
430+
/**
431+
* The current resolved value of python.useEnvironmentsExtension.
432+
*/
433+
envsResolvedSettingNow?: boolean;
419434
};
420435
/**
421436
* Reports explicit-false scopes after a python.useEnvironmentsExtension configuration change.
422437
* This is a configuration notification, not proof of user intent or a changed integration decision.
423438
*/
424439
/* __GDPR__
425440
"environments_extension.setting_changed" : {
426-
"envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
441+
"envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
442+
"envsdecisionreason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
443+
"envsavailabletohostnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
444+
"envsresolvedsettingnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
445+
"envscacheddecision" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
427446
}
428447
*/
429448
[EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED]: {
430449
envsExplicitFalseScope: EnvsExplicitFalseScope;
450+
envsDecisionReason?: EnvsIntegrationDecisionReason;
451+
envsAvailableToHostNow: boolean;
452+
envsResolvedSettingNow: boolean;
453+
envsCachedDecision?: boolean;
431454
};
432455
/**
433456
* Telemetry event sent when substituting Environment variables to calculate value of variables

‎src/client/telemetry/types.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,15 @@ export type EditorLoadTelemetry = IEventNamePropertyMapping[EventName.EDITOR_LOA
1010

1111
export type EnvsExplicitFalseScope = 'none' | 'user' | 'workspace' | 'folder' | 'multiple';
1212

13+
export type EnvsIntegrationDecisionReason = 'enabled' | 'extensionUnavailable' | 'resolvedSettingFalse';
14+
15+
export type EnvsIntegrationDecisionTelemetry = {
16+
envsDecisionReason?: EnvsIntegrationDecisionReason;
17+
envsAvailableToHostNow: boolean;
18+
envsResolvedSettingNow: boolean;
19+
envsCachedDecision?: boolean;
20+
};
21+
1322
export type PythonInterpreterTelemetry = IEventNamePropertyMapping[EventName.PYTHON_INTERPRETER];
1423
export type TestTool = 'pytest' | 'unittest';
1524
export type TestRunTelemetry = IEventNamePropertyMapping[EventName.UNITTEST_RUN];

‎src/test/envExt/api.internal.unit.test.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import { assert } from 'chai';
5+
import * as sinon from 'sinon';
6+
import * as typemoq from 'typemoq';
7+
import { Extension, WorkspaceConfiguration } from 'vscode';
8+
import * as extensionsApi from '../../client/common/vscodeApis/extensionsApi';
9+
import * as workspaceApis from '../../client/common/vscodeApis/workspaceApis';
10+
import {
11+
_resetEnvExtensionDecisionCache,
12+
ENVS_EXTENSION_ID,
13+
getEnvExtensionDecisionTelemetry,
14+
useEnvExtension,
15+
} from '../../client/envExt/api.internal';
16+
17+
function configuration(resolvedSetting: boolean): WorkspaceConfiguration {
18+
const config = typemoq.Mock.ofType<WorkspaceConfiguration>();
19+
config.setup((c) => c.get<boolean>('useEnvironmentsExtension', false)).returns(() => resolvedSetting);
20+
return config.object;
21+
}
22+
23+
suite('Python Environments integration decision telemetry', () => {
24+
let getConfiguration: sinon.SinonStub;
25+
let getExtension: sinon.SinonStub;
26+
27+
setup(() => {
28+
_resetEnvExtensionDecisionCache();
29+
getConfiguration = sinon.stub(workspaceApis, 'getConfiguration').returns(configuration(false));
30+
getExtension = sinon.stub(extensionsApi, 'getExtension').returns(undefined);
31+
});
32+
33+
teardown(() => {
34+
_resetEnvExtensionDecisionCache();
35+
sinon.restore();
36+
});
37+
38+
const cases = [
39+
{
40+
name: 'extension unavailable',
41+
available: false,
42+
resolvedSetting: true,
43+
decision: false,
44+
reason: 'extensionUnavailable',
45+
},
46+
{
47+
name: 'resolved setting false',
48+
available: true,
49+
resolvedSetting: false,
50+
decision: false,
51+
reason: 'resolvedSettingFalse',
52+
},
53+
{
54+
name: 'integration enabled',
55+
available: true,
56+
resolvedSetting: true,
57+
decision: true,
58+
reason: 'enabled',
59+
},
60+
] as const;
61+
62+
cases.forEach(({ name, available, resolvedSetting, decision, reason }) => {
63+
test(`captures ${name}`, () => {
64+
getConfiguration.returns(configuration(resolvedSetting));
65+
getExtension.returns(available ? ({} as Extension<unknown>) : undefined);
66+
67+
assert.strictEqual(useEnvExtension(), decision);
68+
assert.deepEqual(getEnvExtensionDecisionTelemetry(), {
69+
envsDecisionReason: reason,
70+
envsAvailableToHostNow: available,
71+
envsResolvedSettingNow: resolvedSetting,
72+
envsCachedDecision: decision,
73+
});
74+
sinon.assert.alwaysCalledWithExactly(getExtension, ENVS_EXTENSION_ID);
75+
});
76+
});
77+
78+
test('reports live inputs without changing a stale cached decision', () => {
79+
assert.isFalse(useEnvExtension());
80+
getConfiguration.returns(configuration(true));
81+
getExtension.returns({} as Extension<unknown>);
82+
83+
assert.deepEqual(getEnvExtensionDecisionTelemetry(), {
84+
envsDecisionReason: 'extensionUnavailable',
85+
envsAvailableToHostNow: true,
86+
envsResolvedSettingNow: true,
87+
envsCachedDecision: false,
88+
});
89+
assert.isFalse(useEnvExtension());
90+
});
91+
92+
test('does not initialize the cached decision when telemetry is requested', () => {
93+
getConfiguration.returns(configuration(true));
94+
getExtension.returns({} as Extension<unknown>);
95+
96+
assert.deepEqual(getEnvExtensionDecisionTelemetry(), {
97+
envsDecisionReason: undefined,
98+
envsAvailableToHostNow: true,
99+
envsResolvedSettingNow: true,
100+
envsCachedDecision: undefined,
101+
});
102+
});
103+
});

‎src/test/envExt/telemetry.unit.test.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import * as typemoq from 'typemoq';
88
import { ConfigurationChangeEvent, EventEmitter, Uri, WorkspaceConfiguration, WorkspaceFolder } from 'vscode';
99
import { IDisposableRegistry } from '../../client/common/types';
1010
import * as workspaceApis from '../../client/common/vscodeApis/workspaceApis';
11+
import * as envExt from '../../client/envExt/api.internal';
1112
import { getEnvsExplicitFalseScope, registerEnvironmentsExtensionTelemetry } from '../../client/envExt/telemetry';
1213
import * as telemetry from '../../client/telemetry';
1314
import { EventName } from '../../client/telemetry/constants';
@@ -44,13 +45,20 @@ suite('Environments extension telemetry', () => {
4445
let sendTelemetryEvent: sinon.SinonStub;
4546
let changes: EventEmitter<ConfigurationChangeEvent>;
4647
let disposables: IDisposableRegistry;
48+
const decisionTelemetry = {
49+
envsDecisionReason: 'resolvedSettingFalse' as const,
50+
envsAvailableToHostNow: true,
51+
envsResolvedSettingNow: false,
52+
envsCachedDecision: false,
53+
};
4754

4855
setup(() => {
4956
getConfiguration = sinon.stub(workspaceApis, 'getConfiguration').returns(configuration({}));
5057
getWorkspaceFolders = sinon.stub(workspaceApis, 'getWorkspaceFolders').returns(undefined);
5158
sendTelemetryEvent = sinon.stub(telemetry, 'sendTelemetryEvent');
5259
changes = new EventEmitter<ConfigurationChangeEvent>();
5360
sinon.stub(workspaceApis, 'onDidChangeConfiguration').callsFake((listener) => changes.event(listener));
61+
sinon.stub(envExt, 'getEnvExtensionDecisionTelemetry').returns(decisionTelemetry);
5462
disposables = [];
5563
});
5664

@@ -170,7 +178,7 @@ suite('Environments extension telemetry', () => {
170178
['user', 'workspace', 'none'].map((envsExplicitFalseScope) => [
171179
EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED,
172180
undefined,
173-
{ envsExplicitFalseScope },
181+
{ envsExplicitFalseScope, ...decisionTelemetry },
174182
]),
175183
);
176184
});
@@ -185,7 +193,7 @@ suite('Environments extension telemetry', () => {
185193
sendTelemetryEvent,
186194
EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED,
187195
undefined,
188-
{ envsExplicitFalseScope: 'folder' },
196+
{ envsExplicitFalseScope: 'folder', ...decisionTelemetry },
189197
);
190198
});
191199

‎src/test/startupTelemetry.unit.test.ts‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => {
7373
.returns(() => interpreterPathService.object);
7474
sinon.stub(constants, 'isTestExecution').returns(false);
7575
sinon.stub(envExt, 'useEnvExtension').returns(false);
76+
sinon.stub(envExt, 'getEnvExtensionDecisionTelemetry').returns({
77+
envsDecisionReason: 'resolvedSettingFalse',
78+
envsAvailableToHostNow: true,
79+
envsResolvedSettingNow: false,
80+
envsCachedDecision: false,
81+
});
7682
sinon.stub(envExtTelemetry, 'getEnvsExplicitFalseScope').returns('workspace');
7783
sendTelemetryEvent = sinon.stub(telemetry, 'sendTelemetryEvent');
7884
durations = {
@@ -101,11 +107,18 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => {
101107
sendTelemetryEvent,
102108
EventName.EDITOR_LOAD,
103109
durations,
104-
sinon.match({ envsExplicitFalseScope: 'workspace', isFirstSession: false }),
110+
sinon.match({
111+
envsExplicitFalseScope: 'workspace',
112+
envsDecisionReason: 'resolvedSettingFalse',
113+
envsAvailableToHostNow: true,
114+
envsResolvedSettingNow: false,
115+
isFirstSession: false,
116+
}),
105117
);
106118
if (isTrusted) {
107119
expect(sendTelemetryEvent.firstCall.args[2].usingEnvironmentsExtension).to.equal(false);
108120
}
121+
expect(sendTelemetryEvent.firstCall.args[2]).not.to.have.property('envsCachedDecision');
109122
});
110123
});
111124
});
@@ -116,7 +129,7 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => {
116129
test(`Return false if using settings equals {globalValue: ${globalValue}, workspaceValue: ${workspaceValue}, workspaceFolderValue: ${workspaceFolderValue}}`, () => {
117130
interpreterPathService
118131
.setup((i) => i.inspect(resource))
119-
.returns(() => ({ globalValue, workspaceValue, workspaceFolderValue } as any));
132+
.returns(() => ({ globalValue, workspaceValue, workspaceFolderValue }) as any);
120133
const result = hasUserDefinedPythonPath(resource, serviceContainer.object);
121134
expect(result).to.equal(false, 'Should be false');
122135
});
@@ -127,7 +140,7 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => {
127140
test('Return true if using setting value equals something else', () => {
128141
interpreterPathService
129142
.setup((i) => i.inspect(resource))
130-
.returns(() => ({ globalValue: 'something else' } as any));
143+
.returns(() => ({ globalValue: 'something else' }) as any);
131144
const result = hasUserDefinedPythonPath(resource, serviceContainer.object);
132145
expect(result).to.equal(true, 'Should be true');
133146
});

0 commit comments

Comments
 (0)