diff --git a/build/test-requirements.txt b/build/test-requirements.txt index 6d64ff72ac7f..ff9afdfc8a2e 100644 --- a/build/test-requirements.txt +++ b/build/test-requirements.txt @@ -12,8 +12,8 @@ flask fastapi uvicorn django -testresources testscenarios +testtools # Integrated TensorBoard tests tensorboard diff --git a/package-lock.json b/package-lock.json index 6de6edae81c0..258726777690 100644 --- a/package-lock.json +++ b/package-lock.json @@ -103,7 +103,7 @@ "tsconfig-paths-webpack-plugin": "^3.2.0", "typemoq": "^2.1.0", "typescript": "~5.2", - "uuid": "^8.3.2", + "uuid": "^14.0.0", "webpack": "^5.105.0", "webpack-bundle-analyzer": "^4.5.0", "webpack-cli": "^4.9.2", @@ -285,6 +285,14 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, + "node_modules/@azure/core-rest-pipeline/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@azure/core-tracing": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.0.1.tgz", @@ -434,6 +442,15 @@ "node": ">=0.8.0" } }, + "node_modules/@azure/msal-node/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@azure/opentelemetry-instrumentation-azure-sdk": { "version": "1.0.0-beta.5", "resolved": "https://registry.npmjs.org/@azure/opentelemetry-instrumentation-azure-sdk/-/opentelemetry-instrumentation-azure-sdk-1.0.0-beta.5.tgz", @@ -9144,6 +9161,15 @@ "node": ">=8" } }, + "node_modules/istanbul-lib-processinfo/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/istanbul-lib-report": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", @@ -14470,11 +14496,16 @@ "dev": true }, "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.0.tgz", + "integrity": "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "bin": { - "uuid": "dist/bin/uuid" + "uuid": "dist-node/bin/uuid" } }, "node_modules/v8-compile-cache-lib": { @@ -15713,6 +15744,11 @@ "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" } } }, @@ -15844,6 +15880,12 @@ "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-14.12.0.tgz", "integrity": "sha512-IDDXmzfdwmDkv4SSmMEyAniJf6fDu3FJ7ncOjlxkDuT85uSnLEhZi3fGZpoR7T4XZpOMx9teM9GXBgrfJgyeBw==", "dev": true + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true } } }, @@ -22342,6 +22384,12 @@ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true } } }, @@ -26326,9 +26374,10 @@ "dev": true }, "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.0.tgz", + "integrity": "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==", + "dev": true }, "v8-compile-cache-lib": { "version": "3.0.0", diff --git a/package.json b/package.json index 2a27cddc0976..9f689b60ff34 100644 --- a/package.json +++ b/package.json @@ -1799,7 +1799,7 @@ "tsconfig-paths-webpack-plugin": "^3.2.0", "typemoq": "^2.1.0", "typescript": "~5.2", - "uuid": "^8.3.2", + "uuid": "^14.0.0", "webpack": "^5.105.0", "webpack-bundle-analyzer": "^4.5.0", "webpack-cli": "^4.9.2", diff --git a/python_files/tests/unittestadapter/.data/test_scenarios/tests/__init__.py b/python_files/tests/unittestadapter/.data/test_scenarios/tests/__init__.py index 6b8fbbc579ab..5b7f7a925cc0 100644 --- a/python_files/tests/unittestadapter/.data/test_scenarios/tests/__init__.py +++ b/python_files/tests/unittestadapter/.data/test_scenarios/tests/__init__.py @@ -1,15 +1,2 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - -import os - -import testresources -from testscenarios import generate_scenarios - -def load_tests(loader, tests, pattern): - this_dir = os.path.dirname(__file__) - mytests = loader.discover(start_dir=this_dir, pattern=pattern) - result = testresources.OptimisingTestSuite() - result.addTests(generate_scenarios(mytests)) - result.addTests(generate_scenarios(tests)) - return result diff --git a/python_files/tests/unittestadapter/.data/test_scenarios/tests/test_scene.py b/python_files/tests/unittestadapter/.data/test_scenarios/tests/test_scene.py index 1f66cbde4ef7..35c1c7002319 100644 --- a/python_files/tests/unittestadapter/.data/test_scenarios/tests/test_scene.py +++ b/python_files/tests/unittestadapter/.data/test_scenarios/tests/test_scene.py @@ -1,13 +1,27 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from testscenarios import TestWithScenarios +import unittest + +from testscenarios import TestWithScenarios, generate_scenarios + + +def load_tests(loader, standard_tests, pattern): # noqa: ARG001 + # Pre-expand ``TestWithScenarios`` scenarios at load time so individual + # scenario-multiplied test IDs (e.g. ``test_operations(add)``) can be + # resolved by ``unittest.TestLoader.loadTestsFromName``. Without this, + # ``TestWithScenarios`` only multiplies scenarios at ``run()`` time and + # loading a specific scenario by name raises ``AttributeError``. + result = unittest.TestSuite() + result.addTests(generate_scenarios(standard_tests)) + return result + class TestMathOperations(TestWithScenarios): scenarios = [ ('add', {'test_id': 'test_add', 'a': 5, 'b': 3, 'expected': 8}), ('subtract', {'test_id': 'test_subtract', 'a': 5, 'b': 3, 'expected': 2}), - ('multiply', {'test_id': 'test_multiply', 'a': 5, 'b': 3, 'expected': 15}) + ('multiply', {'test_id': 'test_multiply', 'a': 5, 'b': 3, 'expected': 15}), ] a: int = 0 b: int = 0 diff --git a/src/client/chat/baseTool.ts b/src/client/chat/baseTool.ts index 2eedbbe226e3..d8e2e1d60d42 100644 --- a/src/client/chat/baseTool.ts +++ b/src/client/chat/baseTool.ts @@ -16,8 +16,10 @@ import { IResourceReference, isCancellationError, resolveFilePath } from './util import { ErrorWithTelemetrySafeReason } from '../common/errors/errorUtils'; import { sendTelemetryEvent } from '../telemetry'; import { EventName } from '../telemetry/constants'; +import { StopWatch } from '../common/utils/stopWatch'; export abstract class BaseTool implements LanguageModelTool { + protected extraTelemetryProperties: Record = {}; constructor(private readonly toolName: string) {} async invoke( @@ -29,8 +31,10 @@ export abstract class BaseTool implements Language new LanguageModelTextPart('Cannot use this tool in an untrusted workspace.'), ]); } + this.extraTelemetryProperties = {}; let error: Error | undefined; const resource = resolveFilePath(options.input.resourcePath); + const stopWatch = new StopWatch(); try { return await this.invokeImpl(options, resource, token); } catch (ex) { @@ -46,10 +50,11 @@ export abstract class BaseTool implements Language ? error.telemetrySafeReason : 'error' : undefined; - sendTelemetryEvent(EventName.INVOKE_TOOL, undefined, { + sendTelemetryEvent(EventName.INVOKE_TOOL, stopWatch.elapsedTime, { toolName: this.toolName, failed, failureCategory, + ...this.extraTelemetryProperties, }); } } diff --git a/src/client/chat/configurePythonEnvTool.ts b/src/client/chat/configurePythonEnvTool.ts index 0634b9c9ac34..914a92f81c52 100644 --- a/src/client/chat/configurePythonEnvTool.ts +++ b/src/client/chat/configurePythonEnvTool.ts @@ -18,6 +18,7 @@ import { ICodeExecutionService } from '../terminals/types'; import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution'; import { getEnvDetailsForResponse, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, isCancellationError, @@ -58,6 +59,7 @@ export class ConfigurePythonEnvTool extends BaseTool ): Promise { const notebookResponse = getToolResponseIfNotebook(resource); if (notebookResponse) { + this.extraTelemetryProperties.resolveOutcome = 'notebook'; return notebookResponse; } @@ -67,6 +69,8 @@ export class ConfigurePythonEnvTool extends BaseTool ); if (workspaceSpecificEnv) { + this.extraTelemetryProperties.resolveOutcome = 'existingWorkspaceEnv'; + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(workspaceSpecificEnv); return getEnvDetailsForResponse( workspaceSpecificEnv, this.api, @@ -79,7 +83,9 @@ export class ConfigurePythonEnvTool extends BaseTool if (await this.createEnvTool.shouldCreateNewVirtualEnv(resource, token)) { try { - return await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token); + const result = await lm.invokeTool(CreateVirtualEnvTool.toolName, options, token); + this.extraTelemetryProperties.resolveOutcome = 'createdVirtualEnv'; + return result; } catch (ex) { if (isCancellationError(ex)) { const input: ISelectPythonEnvToolArguments = { @@ -87,6 +93,7 @@ export class ConfigurePythonEnvTool extends BaseTool reason: 'cancelled', }; // If the user cancelled the tool, then we should invoke the select env tool. + this.extraTelemetryProperties.resolveOutcome = 'selectedEnvAfterCancelledCreate'; return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token); } throw ex; @@ -95,6 +102,7 @@ export class ConfigurePythonEnvTool extends BaseTool const input: ISelectPythonEnvToolArguments = { ...options.input, }; + this.extraTelemetryProperties.resolveOutcome = 'selectedEnv'; return lm.invokeTool(SelectPythonEnvTool.toolName, { ...options, input }, token); } } diff --git a/src/client/chat/getExecutableTool.ts b/src/client/chat/getExecutableTool.ts index 746a540d14f8..38dabce644a7 100644 --- a/src/client/chat/getExecutableTool.ts +++ b/src/client/chat/getExecutableTool.ts @@ -19,6 +19,7 @@ import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/termin import { getEnvDisplayName, getEnvironmentDetails, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, raceCancellationError, @@ -53,6 +54,12 @@ export class GetExecutableTool extends BaseTool implements L return notebookResponse; } + const envPath = this.api.getActiveEnvironmentPath(resourcePath); + const environment = await raceCancellationError(this.api.resolveEnvironment(envPath), token); + if (environment) { + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); + } + const message = await getEnvironmentDetails( resourcePath, this.api, diff --git a/src/client/chat/getPythonEnvTool.ts b/src/client/chat/getPythonEnvTool.ts index ed1dd0374424..d25d72baeba8 100644 --- a/src/client/chat/getPythonEnvTool.ts +++ b/src/client/chat/getPythonEnvTool.ts @@ -17,7 +17,13 @@ import { IServiceContainer } from '../ioc/types'; import { ICodeExecutionService } from '../terminals/types'; import { TerminalCodeExecutionProvider } from '../terminals/codeExecution/terminalCodeExecution'; import { IProcessServiceFactory, IPythonExecutionFactory } from '../common/process/types'; -import { getEnvironmentDetails, getToolResponseIfNotebook, IResourceReference, raceCancellationError } from './utils'; +import { + getEnvironmentDetails, + getEnvTypeForTelemetry, + getToolResponseIfNotebook, + IResourceReference, + raceCancellationError, +} from './utils'; import { getPythonPackagesResponse } from './listPackagesTool'; import { ITerminalHelper } from '../common/terminal/types'; import { getEnvExtApi, useEnvExtension } from '../envExt/api.internal'; @@ -64,13 +70,16 @@ export class GetEnvironmentInfoTool extends BaseTool 'noEnvFound', ); } + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); let packages = ''; + let responsePackageCount = 0; if (useEnvExtension()) { const api = await getEnvExtApi(); const env = await api.getEnvironment(resourcePath); const pkgs = env ? await api.getPackages(env) : []; if (pkgs && pkgs.length > 0) { + responsePackageCount = pkgs.length; // Installed Python packages, each in the format or (). The version may be omitted if unknown. Returns an empty array if no packages are installed. const response = [ 'Below is a list of the Python packages, each in the format or (). The version may be omitted if unknown: ', @@ -90,7 +99,10 @@ export class GetEnvironmentInfoTool extends BaseTool resourcePath, token, ); + // Count lines starting with '- ' to get the number of packages + responsePackageCount = (packages.match(/^- /gm) || []).length; } + this.extraTelemetryProperties.responsePackageCount = String(responsePackageCount); const message = await getEnvironmentDetails( resourcePath, this.api, diff --git a/src/client/chat/installPackagesTool.ts b/src/client/chat/installPackagesTool.ts index f7795620cf13..5d3d456361f9 100644 --- a/src/client/chat/installPackagesTool.ts +++ b/src/client/chat/installPackagesTool.ts @@ -16,6 +16,7 @@ import { PythonExtension } from '../api/types'; import { IServiceContainer } from '../ioc/types'; import { getEnvDisplayName, + getEnvTypeForTelemetry, getToolResponseIfNotebook, IResourceReference, isCancellationError, @@ -51,6 +52,7 @@ export class InstallPackagesTool extends BaseTool ): Promise { const packageCount = options.input.packageList.length; const packagePlurality = packageCount === 1 ? 'package' : 'packages'; + this.extraTelemetryProperties.packageCount = String(packageCount); const notebookResponse = getToolResponseIfNotebook(resourcePath); if (notebookResponse) { return notebookResponse; @@ -84,9 +86,11 @@ export class InstallPackagesTool extends BaseTool 'noEnvFound', ); } + this.extraTelemetryProperties.envType = getEnvTypeForTelemetry(environment); const isConda = isCondaEnv(environment); const installers = this.serviceContainer.getAll(IModuleInstaller); const installerType = isConda ? ModuleInstallerType.Conda : ModuleInstallerType.Pip; + this.extraTelemetryProperties.installerType = isConda ? 'conda' : 'pip'; const installer = installers.find((i) => i.type === installerType); if (!installer) { throw new ErrorWithTelemetrySafeReason( diff --git a/src/client/chat/utils.ts b/src/client/chat/utils.ts index 84df2901341b..2309316bcbdd 100644 --- a/src/client/chat/utils.ts +++ b/src/client/chat/utils.ts @@ -76,6 +76,10 @@ export function isCondaEnv(env: ResolvedEnvironment) { return (env.environment?.type || '').toLowerCase() === 'conda'; } +export function getEnvTypeForTelemetry(env: ResolvedEnvironment): string { + return (env.environment?.type || 'unknown').toLowerCase(); +} + export async function getEnvironmentDetails( resourcePath: Uri | undefined, api: PythonExtension['environments'], diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 738c5f8a2776..763f7405aa0d 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -1988,11 +1988,16 @@ export interface IEventNamePropertyMapping { * Telemetry event sent when invoking a Tool */ /* __GDPR__ - "invokeTool" : { + "INVOKE_TOOL" : { "duration" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "owner": "donjayamanne" }, "toolName" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "donjayamanne" }, "failed": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Whether there was a failure. Common to most of the events.", "owner": "donjayamanne" }, - "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" } + "failureCategory": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"A reason that we generate (e.g. kerneldied, noipykernel, etc), more like a category of the error. Common to most of the events.", "owner": "donjayamanne" }, + "resolveOutcome": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which code path resolved the environment in configure_python_environment.", "owner": "donjayamanne" }, + "envType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"The type of Python environment (e.g. venv, conda, system).", "owner": "donjayamanne" }, + "packageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages requested for installation (install_python_packages only).", "owner": "donjayamanne" }, + "installerType": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Which installer was used: pip or conda (install_python_packages only).", "owner": "donjayamanne" }, + "responsePackageCount": {"classification":"SystemMetaData","purpose":"FeatureInsight","comment":"Number of packages in the environment response (get_python_environment_details only).", "owner": "donjayamanne" } } */ [EventName.INVOKE_TOOL]: { @@ -2009,6 +2014,26 @@ export interface IEventNamePropertyMapping { * A reason the error was thrown. */ failureCategory?: string; + /** + * Which code path resolved the environment (configure_python_environment only). + */ + resolveOutcome?: string; + /** + * The type of Python environment (e.g. venv, conda, system). + */ + envType?: string; + /** + * Number of packages requested for installation (install_python_packages only). + */ + packageCount?: string; + /** + * Which installer was used: pip or conda (install_python_packages only). + */ + installerType?: string; + /** + * Number of packages in the environment response (get_python_environment_details only). + */ + responsePackageCount?: string; }; /** * Telemetry event sent if and when user configure tests command. This command can be trigerred from multiple places in the extension. (Command palette, prompt etc.)