From c59eb26f9e34b99965122a663326a691a2e2664a Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 13 Oct 2023 19:35:24 -0700 Subject: [PATCH 1/9] implement django tests with custom runner --- pythonFiles/unittestadapter/django_runner.py | 107 ++++++++++++++++++ .../unittestadapter/django_test_runner.py | 17 +++ pythonFiles/unittestadapter/execution.py | 31 +++-- .../testing/testController/common/server.ts | 12 +- 4 files changed, 159 insertions(+), 8 deletions(-) create mode 100644 pythonFiles/unittestadapter/django_runner.py create mode 100644 pythonFiles/unittestadapter/django_test_runner.py diff --git a/pythonFiles/unittestadapter/django_runner.py b/pythonFiles/unittestadapter/django_runner.py new file mode 100644 index 000000000000..5cb1a546e998 --- /dev/null +++ b/pythonFiles/unittestadapter/django_runner.py @@ -0,0 +1,107 @@ +import subprocess +import os +import pathlib +import re +import sys + + +def find_settings_module(path_to_manage_py): + dj_settings_module = None + with open(path_to_manage_py, "r") as manage_py: + pattern = r"^os\.environ\.setdefault\((['\"])(DJANGO_SETTINGS_MODULE)\1, (['\"])(?P[\w.]+)\3\)$" + for line in manage_py.readlines(): + match_result = re.match(pattern, line.strip()) + if match_result is not None: + dj_settings_module = match_result.groupdict().get("settings_path", None) + break + return dj_settings_module + + +def configure_test_runner(path_to_manage_py): + # Getting the DJANGO_SETTINGS_MODULE from manage.py + dj_settings_module = find_settings_module(path_to_manage_py) + if dj_settings_module is None: + raise Exception("DJANGO_SETTINGS_MODULE not found in manage.py") + + # Construct the path to the settings.py file + settings_file = os.path.join( + os.path.dirname(dj_settings_module.replace(".", os.sep)), "settings.py" + ) + # Check if the settings.py file exists + if not os.path.exists(settings_file): + raise Exception(f"settings.py file not found at {settings_file}") + # Read the content of the existing settings.py file + with open(settings_file, "r") as f: + original_settings_content = f.read() + + # Check if TEST_RUNNER is already defined in the settings + if "TEST_RUNNER" in original_settings_content: + print("TEST_RUNNER is already defined in settings.py. but continuing") + print("settings_content: ", original_settings_content) + else: + # Add the custom test runner to the settings.py file + + # Get path to the custom_test_runner.py parent folder, add to sys.path + custom_test_runner_dir = pathlib.Path(__file__).parent + sys.path.insert(0, custom_test_runner_dir) + + # Import your custom test runner class + # from execution import UnittestTestResult + + # Set the TEST_RUNNER setting + setting_content = original_settings_content + ( + "\n\n" + + "# Use custom test runner\n" + + "import sys\n" + + f"sys.path.insert(0, '{custom_test_runner_dir}')\n" + + f"TEST_RUNNER = 'django_test_runner.CustomTestRunner'\n" + ) + + # Write the updated content back to the settings.py file + with open(settings_file, "w") as f: + f.write(setting_content) + + print("TEST_RUNNER setting added to settings.py.") + return settings_file, original_settings_content + + +# Define a cleanup method +def cleanup(settings_file, original_settings_content): + # Restore the original content of settings.py + with open(settings_file, "w") as f: + f.write(original_settings_content) + print("Settings.py has been restored to its original state.") + + return True + + +def runner(): + # Define the path to your manage.py file + # could get path to manage.py from environment variable + # get Django test boolean + django_test_enabled = os.environ.get("DJANGO_TEST_ENABLED") + manage_py_path = os.environ.get("MANAGE_PY_PATH") + + if ( + django_test_enabled is not None + and django_test_enabled.lower() == "true" + and manage_py_path is not None + ): + # attempt to configure and run tests as django tests + try: + settings_file, original_settings_content = configure_test_runner( + manage_py_path + ) + # Command to run 'python manage.py test' + python_executable = sys.executable + command = [python_executable, "manage.py", "test"] + print("running test command: ", command) + # Run the command + try: + subprocess.run(" ".join(command), shell=True, check=True) + # Cleanup + cleanup(settings_file, original_settings_content) + except subprocess.CalledProcessError as e: + print(f"Error running 'manage.py test': {e}") + except Exception as e: + print(f"Error configuring Django test runner: {e}") diff --git a/pythonFiles/unittestadapter/django_test_runner.py b/pythonFiles/unittestadapter/django_test_runner.py new file mode 100644 index 000000000000..317751fe9acb --- /dev/null +++ b/pythonFiles/unittestadapter/django_test_runner.py @@ -0,0 +1,17 @@ +from django.test.runner import DiscoverRunner +import sys +import os +import pathlib + +script_dir = pathlib.Path(__file__).parent +sys.path.append(os.fspath(script_dir)) + +from execution import UnittestTestResult + + +class CustomTestRunner(DiscoverRunner): + def get_test_runner_kwargs(self): + print("get_test_runner_kwargs") + kwargs = super().get_test_runner_kwargs() + kwargs["resultclass"] = UnittestTestResult + return kwargs diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 2a22bfff3486..9a86aa0d17c9 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -20,6 +20,7 @@ from testing_tools import process_json_util, socket_manager from typing_extensions import Literal, NotRequired, TypeAlias, TypedDict from unittestadapter.utils import parse_unittest_args +from django_runner import runner ErrorType = Union[ Tuple[Type[BaseException], BaseException, TracebackType], Tuple[None, None, None] @@ -103,7 +104,6 @@ def formatResult( subtest: Union[unittest.TestCase, None] = None, ): tb = None - message = "" # error is a tuple of the form returned by sys.exc_info(): (type, value, traceback). if error is not None: @@ -128,9 +128,16 @@ def formatResult( "subtest": subtest.id() if subtest else None, } self.formatted[test_id] = result - if testPort == 0 or testUuid == 0: - print("Error sending response, port or uuid unknown to python server.") - send_run_data(result, testPort, testUuid) + testPort2 = int(os.environ.get("TEST_PORT", DEFAULT_PORT)) + testUuid2 = os.environ.get("TEST_UUID") + if testPort2 == 0 or testUuid2 == 0: + print( + "Error sending response, port or uuid unknown to python server.", + testPort, + testUuid, + ) + + send_run_data(result, testPort2, testUuid2) class TestExecutionStatus(str, enum.Enum): @@ -318,9 +325,19 @@ def post_response( testUuid = "unknown" if test_ids_from_buffer: # Perform test execution. - payload = run_tests( - start_dir, test_ids_from_buffer, pattern, top_level_dir, testUuid - ) + + # get django test boolean + django_test_enabled = os.environ.get("DJANGO_TEST_ENABLED") + print("DJANGO_TEST_ENABLED = ", django_test_enabled) + if django_test_enabled: + # run django runner + print("running django runner") + runner() + else: + print("running unittest runner") + payload = run_tests( + start_dir, test_ids_from_buffer, pattern, top_level_dir, testUuid + ) else: cwd = os.path.abspath(start_dir) status = TestExecutionStatus.error diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index e496860526e4..db346c5de818 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -184,6 +184,16 @@ export class PythonTestServer implements ITestServer, Disposable { mutableEnv.TEST_PORT = this.getPort().toString(); mutableEnv.RUN_TEST_IDS_PORT = runTestIdPort; + const isRun = runTestIdPort !== undefined; + + // NEEDS TO BE UNCOMMENTED TO GET DJANGO WORKING + // if (isRun) { + // mutableEnv.DJANGO_TEST_ENABLED = 'true'; + // mutableEnv.MANAGE_PY_PATH = [options.cwd, 'manage.py'].join('/'); + // console.log('DJANGO_TEST_ENABLED', mutableEnv.DJANGO_TEST_ENABLED); + // console.log('MANAGE_PY_PATH', mutableEnv.MANAGE_PY_PATH); + // } + const spawnOptions: SpawnOptions = { token: options.token, cwd: options.cwd, @@ -191,7 +201,7 @@ export class PythonTestServer implements ITestServer, Disposable { outputChannel: options.outChannel, env: mutableEnv, }; - const isRun = runTestIdPort !== undefined; + // Create the Python environment in which to execute the command. const creationOptions: ExecutionFactoryCreateWithEnvironmentOptions = { allowEnvironmentFetchExceptions: false, From 31305e16e275ab5db33ab9862f2dc1e444cd7ecb Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 18 Oct 2023 13:09:03 -0700 Subject: [PATCH 2/9] switch to using --testrunner --- pythonFiles/unittestadapter/django_runner.py | 128 +++++------------- pythonFiles/unittestadapter/execution.py | 83 ++++++------ .../testing/testController/common/server.ts | 12 +- 3 files changed, 84 insertions(+), 139 deletions(-) diff --git a/pythonFiles/unittestadapter/django_runner.py b/pythonFiles/unittestadapter/django_runner.py index 5cb1a546e998..9aa306f5527e 100644 --- a/pythonFiles/unittestadapter/django_runner.py +++ b/pythonFiles/unittestadapter/django_runner.py @@ -1,107 +1,47 @@ import subprocess import os import pathlib -import re import sys +from typing import Union +from pythonFiles.unittestadapter.execution import VSCodeUnittestError -def find_settings_module(path_to_manage_py): - dj_settings_module = None - with open(path_to_manage_py, "r") as manage_py: - pattern = r"^os\.environ\.setdefault\((['\"])(DJANGO_SETTINGS_MODULE)\1, (['\"])(?P[\w.]+)\3\)$" - for line in manage_py.readlines(): - match_result = re.match(pattern, line.strip()) - if match_result is not None: - dj_settings_module = match_result.groupdict().get("settings_path", None) - break - return dj_settings_module +def django_execution_runner(start_dir: Union[str, None]): + # Get path to manage.py if set as an env var, otherwise use the default + manage_py_path = os.environ.get("MANAGE_PY_PATH") -def configure_test_runner(path_to_manage_py): - # Getting the DJANGO_SETTINGS_MODULE from manage.py - dj_settings_module = find_settings_module(path_to_manage_py) - if dj_settings_module is None: - raise Exception("DJANGO_SETTINGS_MODULE not found in manage.py") - - # Construct the path to the settings.py file - settings_file = os.path.join( - os.path.dirname(dj_settings_module.replace(".", os.sep)), "settings.py" - ) - # Check if the settings.py file exists - if not os.path.exists(settings_file): - raise Exception(f"settings.py file not found at {settings_file}") - # Read the content of the existing settings.py file - with open(settings_file, "r") as f: - original_settings_content = f.read() + if manage_py_path is None: + # Search for default manage.py path at the root of the workspace + if not start_dir: + print( + "Error running Django, no start_dir provided or value for MANAGE_PY_PATH" + ) - # Check if TEST_RUNNER is already defined in the settings - if "TEST_RUNNER" in original_settings_content: - print("TEST_RUNNER is already defined in settings.py. but continuing") - print("settings_content: ", original_settings_content) - else: - # Add the custom test runner to the settings.py file + cwd = os.path.abspath(start_dir) + manage_py_path = os.path.join(cwd, "manage.py") - # Get path to the custom_test_runner.py parent folder, add to sys.path + try: + # Get path to the custom_test_runner.py parent folder, add to sys.path. custom_test_runner_dir = pathlib.Path(__file__).parent sys.path.insert(0, custom_test_runner_dir) - - # Import your custom test runner class - # from execution import UnittestTestResult - - # Set the TEST_RUNNER setting - setting_content = original_settings_content + ( - "\n\n" - + "# Use custom test runner\n" - + "import sys\n" - + f"sys.path.insert(0, '{custom_test_runner_dir}')\n" - + f"TEST_RUNNER = 'django_test_runner.CustomTestRunner'\n" - ) - - # Write the updated content back to the settings.py file - with open(settings_file, "w") as f: - f.write(setting_content) - - print("TEST_RUNNER setting added to settings.py.") - return settings_file, original_settings_content - - -# Define a cleanup method -def cleanup(settings_file, original_settings_content): - # Restore the original content of settings.py - with open(settings_file, "w") as f: - f.write(original_settings_content) - print("Settings.py has been restored to its original state.") - - return True - - -def runner(): - # Define the path to your manage.py file - # could get path to manage.py from environment variable - # get Django test boolean - django_test_enabled = os.environ.get("DJANGO_TEST_ENABLED") - manage_py_path = os.environ.get("MANAGE_PY_PATH") - - if ( - django_test_enabled is not None - and django_test_enabled.lower() == "true" - and manage_py_path is not None - ): - # attempt to configure and run tests as django tests + custom_test_runner = "django_test_runner.CustomTestRunner" + + # Build command to run 'python manage.py test'. + python_executable = sys.executable + command = [ + python_executable, + "manage.py", + "test", + "--testrunner", + custom_test_runner, + ] + print("Running Django run tests with command: ", command) try: - settings_file, original_settings_content = configure_test_runner( - manage_py_path - ) - # Command to run 'python manage.py test' - python_executable = sys.executable - command = [python_executable, "manage.py", "test"] - print("running test command: ", command) - # Run the command - try: - subprocess.run(" ".join(command), shell=True, check=True) - # Cleanup - cleanup(settings_file, original_settings_content) - except subprocess.CalledProcessError as e: - print(f"Error running 'manage.py test': {e}") - except Exception as e: - print(f"Error configuring Django test runner: {e}") + subprocess.run(" ".join(command), shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"Error running 'manage.py test': {e}") + raise VSCodeUnittestError(f"Error running 'manage.py test': {e}") + except Exception as e: + print(f"Error configuring Django test runner: {e}") + raise VSCodeUnittestError(f"Error configuring Django test runner: {e}") diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 9a86aa0d17c9..4829d8905433 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -20,7 +20,7 @@ from testing_tools import process_json_util, socket_manager from typing_extensions import Literal, NotRequired, TypeAlias, TypedDict from unittestadapter.utils import parse_unittest_args -from django_runner import runner +from django_runner import django_execution_runner ErrorType = Union[ Tuple[Type[BaseException], BaseException, TracebackType], Tuple[None, None, None] @@ -31,6 +31,13 @@ DEFAULT_PORT = 45454 +class VSCodeUnittestError(Exception): + """A custom exception class for pytest errors.""" + + def __init__(self, message): + super().__init__(message) + + class TestOutcomeEnum(str, enum.Enum): error = "error" failure = "failure" @@ -310,46 +317,44 @@ def post_response( testPort = int(os.environ.get("TEST_PORT", DEFAULT_PORT)) testUuid = os.environ.get("TEST_UUID") - if testPort is DEFAULT_PORT: - print( - "Error[vscode-unittest]: TEST_PORT is not set.", - " TEST_UUID = ", - testUuid, - ) - if testUuid is None: - print( - "Error[vscode-unittest]: TEST_UUID is not set.", - " TEST_PORT = ", - testPort, - ) - testUuid = "unknown" - if test_ids_from_buffer: - # Perform test execution. - - # get django test boolean - django_test_enabled = os.environ.get("DJANGO_TEST_ENABLED") - print("DJANGO_TEST_ENABLED = ", django_test_enabled) - if django_test_enabled: - # run django runner - print("running django runner") - runner() - else: - print("running unittest runner") - payload = run_tests( - start_dir, test_ids_from_buffer, pattern, top_level_dir, testUuid + try: + if testPort is DEFAULT_PORT: + raise VSCodeUnittestError( + "Error[vscode-unittest]: TEST_PORT is not set.", + " TEST_UUID = ", + testUuid, ) - else: - cwd = os.path.abspath(start_dir) - status = TestExecutionStatus.error + if testUuid is None: + raise VSCodeUnittestError( + "Error[vscode-unittest]: TEST_UUID is not set.", + " TEST_PORT = ", + testPort, + ) + if test_ids_from_buffer: + # Perform test execution. + + # Check to see if we are running django tests. + django_test_enabled = os.environ.get("DJANGO_TEST_ENABLED") + print("DJANGO_TEST_ENABLED = ", django_test_enabled) + if django_test_enabled and django_test_enabled.lower() == "true": + # run django runner + print("running django runner") + django_execution_runner(start_dir) + else: + print("running unittest runner") + payload = run_tests( + start_dir, test_ids_from_buffer, pattern, top_level_dir, testUuid + ) + else: + raise VSCodeUnittestError("No test ids received from buffer") + except Exception as exception: payload: PayloadDict = { - "cwd": cwd, - "status": status, - "error": "No test ids received from buffer", + "cwd": os.path.abspath(start_dir) if start_dir else None, + "status": TestExecutionStatus.error, + "error": exception, "result": None, } + post_response(payload, testPort, "unknown") + eot_payload: EOTPayloadDict = {"command_type": "execution", "eot": True} - if testUuid is None: - print("Error sending response, uuid unknown to python server.") - post_response(eot_payload, testPort, "unknown") - else: - post_response(eot_payload, testPort, testUuid) + post_response(eot_payload, testPort, testUuid) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index db346c5de818..82a062881728 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -187,12 +187,12 @@ export class PythonTestServer implements ITestServer, Disposable { const isRun = runTestIdPort !== undefined; // NEEDS TO BE UNCOMMENTED TO GET DJANGO WORKING - // if (isRun) { - // mutableEnv.DJANGO_TEST_ENABLED = 'true'; - // mutableEnv.MANAGE_PY_PATH = [options.cwd, 'manage.py'].join('/'); - // console.log('DJANGO_TEST_ENABLED', mutableEnv.DJANGO_TEST_ENABLED); - // console.log('MANAGE_PY_PATH', mutableEnv.MANAGE_PY_PATH); - // } + if (isRun) { + mutableEnv.DJANGO_TEST_ENABLED = 'true'; + mutableEnv.MANAGE_PY_PATH = [options.cwd, 'manage.py'].join('/'); + console.log('DJANGO_TEST_ENABLED', mutableEnv.DJANGO_TEST_ENABLED); + console.log('MANAGE_PY_PATH', mutableEnv.MANAGE_PY_PATH); + } const spawnOptions: SpawnOptions = { token: options.token, From 395e36613abacf5de67b340e23eb546b6232ed9c Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 18 Oct 2023 13:12:24 -0700 Subject: [PATCH 3/9] add result class error msg --- pythonFiles/unittestadapter/django_test_runner.py | 4 ++++ src/client/testing/testController/common/server.ts | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pythonFiles/unittestadapter/django_test_runner.py b/pythonFiles/unittestadapter/django_test_runner.py index 317751fe9acb..9ff73b16e94b 100644 --- a/pythonFiles/unittestadapter/django_test_runner.py +++ b/pythonFiles/unittestadapter/django_test_runner.py @@ -13,5 +13,9 @@ class CustomTestRunner(DiscoverRunner): def get_test_runner_kwargs(self): print("get_test_runner_kwargs") kwargs = super().get_test_runner_kwargs() + if kwargs["resultclass"] is not None: + raise ValueError( + "Resultclass already set, cannot use custom test runner design for VS Code compatibility." + ) kwargs["resultclass"] = UnittestTestResult return kwargs diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 82a062881728..db346c5de818 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -187,12 +187,12 @@ export class PythonTestServer implements ITestServer, Disposable { const isRun = runTestIdPort !== undefined; // NEEDS TO BE UNCOMMENTED TO GET DJANGO WORKING - if (isRun) { - mutableEnv.DJANGO_TEST_ENABLED = 'true'; - mutableEnv.MANAGE_PY_PATH = [options.cwd, 'manage.py'].join('/'); - console.log('DJANGO_TEST_ENABLED', mutableEnv.DJANGO_TEST_ENABLED); - console.log('MANAGE_PY_PATH', mutableEnv.MANAGE_PY_PATH); - } + // if (isRun) { + // mutableEnv.DJANGO_TEST_ENABLED = 'true'; + // mutableEnv.MANAGE_PY_PATH = [options.cwd, 'manage.py'].join('/'); + // console.log('DJANGO_TEST_ENABLED', mutableEnv.DJANGO_TEST_ENABLED); + // console.log('MANAGE_PY_PATH', mutableEnv.MANAGE_PY_PATH); + // } const spawnOptions: SpawnOptions = { token: options.token, From 63337522aa7bf207b1028323262693d1961a33a1 Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Fri, 20 Oct 2023 13:32:24 -0700 Subject: [PATCH 4/9] add envFile's pythonpath before running tests (#22269) fixes https://github.com/microsoft/vscode-python/issues/22231 --------- Co-authored-by: Karthik Nadig --- src/client/testing/testController/common/server.ts | 3 ++- .../testController/pytest/pytestDiscoveryAdapter.ts | 3 ++- .../testController/pytest/pytestExecutionAdapter.ts | 7 +++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index cff23f67c97f..4e7a617a3ffd 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -182,7 +182,8 @@ export class PythonTestServer implements ITestServer, Disposable { const { uuid } = options; const isDiscovery = (testIds === undefined || testIds.length === 0) && runTestIdPort === undefined; const mutableEnv = { ...env }; - const pythonPathParts: string[] = process.env.PYTHONPATH?.split(path.delimiter) ?? []; + // get python path from mutable env, it contains process.env as well + const pythonPathParts: string[] = mutableEnv.PYTHONPATH?.split(path.delimiter) ?? []; const pythonPathCommand = [options.cwd, ...pythonPathParts].join(path.delimiter); mutableEnv.PYTHONPATH = pythonPathCommand; mutableEnv.TEST_UUID = uuid.toString(); diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index daaaec04ee1c..7599513497d7 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -73,7 +73,8 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { const mutableEnv = { ...(await this.envVarsService?.getEnvironmentVariables(uri)), }; - const pythonPathParts: string[] = process.env.PYTHONPATH?.split(path.delimiter) ?? []; + // get python path from mutable env, it contains process.env as well + const pythonPathParts: string[] = mutableEnv.PYTHONPATH?.split(path.delimiter) ?? []; const pythonPathCommand = [fullPluginPath, ...pythonPathParts].join(path.delimiter); mutableEnv.PYTHONPATH = pythonPathCommand; mutableEnv.TEST_UUID = uuid.toString(); diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index fd61251d33fc..5812df48600c 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -109,8 +109,11 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { const { pytestArgs } = settings.testing; const cwd = settings.testing.cwd && settings.testing.cwd.length > 0 ? settings.testing.cwd : uri.fsPath; // get and edit env vars - const mutableEnv = { ...(await this.envVarsService?.getEnvironmentVariables(uri)) }; - const pythonPathParts: string[] = process.env.PYTHONPATH?.split(path.delimiter) ?? []; + const mutableEnv = { + ...(await this.envVarsService?.getEnvironmentVariables(uri)), + }; + // get python path from mutable env, it contains process.env as well + const pythonPathParts: string[] = mutableEnv.PYTHONPATH?.split(path.delimiter) ?? []; const pythonPathCommand = [fullPluginPath, ...pythonPathParts].join(path.delimiter); mutableEnv.PYTHONPATH = pythonPathCommand; mutableEnv.TEST_UUID = uuid.toString(); From 802cf0c136fa30d5a4e69b6cee60e20fb650636d Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 20 Oct 2023 14:00:28 -0700 Subject: [PATCH 5/9] Reveal selection after editing deactivate script (#22271) --- .../envCollectionActivation/deactivatePrompt.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/client/terminals/envCollectionActivation/deactivatePrompt.ts b/src/client/terminals/envCollectionActivation/deactivatePrompt.ts index 43d1e77957bc..c1a7e3d08f49 100644 --- a/src/client/terminals/envCollectionActivation/deactivatePrompt.ts +++ b/src/client/terminals/envCollectionActivation/deactivatePrompt.ts @@ -2,7 +2,16 @@ // Licensed under the MIT License. import { inject, injectable } from 'inversify'; -import { Position, Uri, WorkspaceEdit, Range, TextEditorRevealType, ProgressLocation, Terminal } from 'vscode'; +import { + Position, + Uri, + WorkspaceEdit, + Range, + TextEditorRevealType, + ProgressLocation, + Terminal, + Selection, +} from 'vscode'; import { IApplicationEnvironment, IApplicationShell, @@ -141,12 +150,17 @@ ${content} // If script already has the hook, don't add it again. const editor = await this.documentManager.showTextDocument(document); if (document.getText().includes(hookMarker)) { + editor.revealRange( + new Range(new Position(document.lineCount - 3, 0), new Position(document.lineCount, 0)), + TextEditorRevealType.AtTop, + ); return; } const editorEdit = new WorkspaceEdit(); editorEdit.insert(document.uri, new Position(document.lineCount, 0), content); await this.documentManager.applyEdit(editorEdit); // Reveal the edits. + editor.selection = new Selection(new Position(document.lineCount - 3, 0), new Position(document.lineCount, 0)); editor.revealRange( new Range(new Position(document.lineCount - 3, 0), new Position(document.lineCount, 0)), TextEditorRevealType.AtTop, From f57be20d7af6c623774cd148f07ed77c31607ee7 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 20 Oct 2023 15:59:11 -0700 Subject: [PATCH 6/9] Support powershell activation even if Powershell execution is disabled on the system (#22273) Closes https://github.com/microsoft/vscode-python/issues/22252 Error out if stderr is related to Powershell execution being disabled on system, even though environment variables are returned in this case, we've observed they're not valid in this case. --- src/client/interpreter/activation/service.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/client/interpreter/activation/service.ts b/src/client/interpreter/activation/service.ts index 02d621c0ccda..5fda5a9380dc 100644 --- a/src/client/interpreter/activation/service.ts +++ b/src/client/interpreter/activation/service.ts @@ -329,7 +329,12 @@ export class EnvironmentActivationService implements IEnvironmentActivationServi } if (result.stderr) { if (returnedEnv) { - traceWarn('Got env variables but with errors', result.stderr); + traceWarn('Got env variables but with errors', result.stderr, returnedEnv); + if (result.stderr.includes('running scripts is disabled')) { + throw new Error( + `Skipping returned result when powershell execution is disabled, stderr ${result.stderr} for ${command}`, + ); + } } else { throw new Error(`StdErr from ShellExec, ${result.stderr} for ${command}`); } From d75874110cbd53bae8a2e2b006814ad1fd21331a Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 20 Oct 2023 20:06:13 -0700 Subject: [PATCH 7/9] Add telemetry for deactivate prompt (#22274) --- src/client/telemetry/constants.ts | 1 + src/client/telemetry/index.ts | 19 ++++++++++++++++++- .../deactivatePrompt.ts | 10 ++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/client/telemetry/constants.ts b/src/client/telemetry/constants.ts index de0980ada257..9e29ef808d0d 100644 --- a/src/client/telemetry/constants.ts +++ b/src/client/telemetry/constants.ts @@ -27,6 +27,7 @@ export enum EventName { PYTHON_INTERPRETER_ACTIVATE_ENVIRONMENT_PROMPT = 'PYTHON_INTERPRETER_ACTIVATE_ENVIRONMENT_PROMPT', PYTHON_NOT_INSTALLED_PROMPT = 'PYTHON_NOT_INSTALLED_PROMPT', CONDA_INHERIT_ENV_PROMPT = 'CONDA_INHERIT_ENV_PROMPT', + TERMINAL_DEACTIVATE_PROMPT = 'TERMINAL_DEACTIVATE_PROMPT', REQUIRE_JUPYTER_PROMPT = 'REQUIRE_JUPYTER_PROMPT', ACTIVATED_CONDA_ENV_LAUNCH = 'ACTIVATED_CONDA_ENV_LAUNCH', ENVFILE_VARIABLE_SUBSTITUTION = 'ENVFILE_VARIABLE_SUBSTITUTION', diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index cc600d2d59a4..f9ed98eb3764 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -1226,13 +1226,30 @@ export interface IEventNamePropertyMapping { selection: 'Allow' | 'Close' | undefined; }; /** - * Telemetry event sent with details when user attempts to run in interactive window when Jupyter is not installed. + * Telemetry event sent with details when user clicks the prompt with the following message: + * + * 'We noticed you're using a conda environment. If you are experiencing issues with this environment in the integrated terminal, we suggest the "terminal.integrated.inheritEnv" setting to be changed to false. Would you like to update this setting?' */ /* __GDPR__ "conda_inherit_env_prompt" : { "selection" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "karrtikr" } } */ + [EventName.TERMINAL_DEACTIVATE_PROMPT]: { + /** + * `Yes` When 'Allow' option is selected + * `Close` When 'Close' option is selected + */ + selection: 'Edit script' | "Don't show again" | undefined; + }; + /** + * Telemetry event sent with details when user attempts to run in interactive window when Jupyter is not installed. + */ + /* __GDPR__ + "require_jupyter_prompt" : { + "selection" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "karrtikr" } + } + */ [EventName.REQUIRE_JUPYTER_PROMPT]: { /** * `Yes` When 'Yes' option is selected diff --git a/src/client/terminals/envCollectionActivation/deactivatePrompt.ts b/src/client/terminals/envCollectionActivation/deactivatePrompt.ts index c1a7e3d08f49..5871cbac92c8 100644 --- a/src/client/terminals/envCollectionActivation/deactivatePrompt.ts +++ b/src/client/terminals/envCollectionActivation/deactivatePrompt.ts @@ -34,6 +34,8 @@ import { isTestExecution } from '../../common/constants'; import { ProgressService } from '../../common/application/progressService'; import { copyFile, createFile, pathExists } from '../../common/platform/fs-paths'; import { getOSType, OSType } from '../../common/utils/platform'; +import { sendTelemetryEvent } from '../../telemetry'; +import { EventName } from '../../telemetry/constants'; export const terminalDeactivationPromptKey = 'TERMINAL_DEACTIVATION_PROMPT_KEY'; @injectable() @@ -115,12 +117,20 @@ export class TerminalDeactivateLimitationPrompt implements IExtensionSingleActiv // Shell integration is not supported for these shells, in which case this workaround won't work. return; } + const telemetrySelections: ['Edit script', "Don't show again"] = ['Edit script', "Don't show again"]; const { initScript, source, destination } = scriptInfo; const prompts = [Common.editSomething.format(initScript.displayName), Common.doNotShowAgain]; const selection = await this.appShell.showWarningMessage( Interpreters.terminalDeactivatePrompt.format(initScript.displayName), ...prompts, ); + let index = selection ? prompts.indexOf(selection) : 0; + if (selection === prompts[0]) { + index = 0; + } + sendTelemetryEvent(EventName.TERMINAL_DEACTIVATE_PROMPT, undefined, { + selection: selection ? telemetrySelections[index] : undefined, + }); if (!selection) { return; } From 6edfcd13f622fa93530206c418c10df861924aee Mon Sep 17 00:00:00 2001 From: mh-firouzjah Date: Sat, 21 Oct 2023 12:47:37 +0330 Subject: [PATCH 8/9] dj-tests-fix: Fixed discovery part --- .../testing_tools/unittest_discovery.py | 18 +++++ pythonFiles/unittestadapter/discovery.py | 14 +++- pythonFiles/unittestadapter/django_runner.py | 71 +++++++++++++++++-- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/pythonFiles/testing_tools/unittest_discovery.py b/pythonFiles/testing_tools/unittest_discovery.py index 2988092c387c..4a36e17a7466 100644 --- a/pythonFiles/testing_tools/unittest_discovery.py +++ b/pythonFiles/testing_tools/unittest_discovery.py @@ -9,6 +9,24 @@ top_level_dir = sys.argv[3] if len(sys.argv) >= 4 else None sys.path.insert(0, os.getcwd()) +import os.path + +sys.path.insert( + 1, + os.path.dirname( # pythonFiles + os.path.dirname( # pythonFiles/testing_tools + os.path.abspath(__file__) # this file + ) + ), +) + +from django_runner import setup_django_env + +django_test_enabled = os.environ.get("DJANGO_TEST_ENABLED", "False") +if django_test_enabled.lower() == "true": + print(f"DJANGO TEST DECLEARED = {django_test_enabled}") + django_env_enabled = setup_django_env(start_dir) + print(f"DJANGO ENV ENABLED = {django_env_enabled}") def get_sourceline(obj): try: diff --git a/pythonFiles/unittestadapter/discovery.py b/pythonFiles/unittestadapter/discovery.py index 7525f33cda61..f14058112666 100644 --- a/pythonFiles/unittestadapter/discovery.py +++ b/pythonFiles/unittestadapter/discovery.py @@ -3,7 +3,7 @@ import json import os -import pathlib +import pathlib # TODO: pathlib added in python v3.4 - this file used to used os.path.dirname | commit/0b6fc5b44c70fed32294b460e3ea45854ae220e4 import sys import traceback import unittest @@ -19,6 +19,8 @@ # If I use from utils then there will be an import error in test_discovery.py. from unittestadapter.utils import TestNode, build_test_tree, parse_unittest_args +from unittestadapter.django_runner import setup_django_env + DEFAULT_PORT = 45454 @@ -46,7 +48,7 @@ def discover_tests( - cwd: Absolute path to the test start directory; - uuid: UUID sent by the caller of the Python script, that needs to be sent back as an integrity check; - status: Test discovery status, can be "success" or "error"; - - tests: Discoverered tests if any, not present otherwise. Note that the status can be "error" but the payload can still contain tests; + - tests: Discovered tests if any, not present otherwise. Note that the status can be "error" but the payload can still contain tests; - error: Discovery error if any, not present otherwise. Payload format for a successful discovery: @@ -78,7 +80,7 @@ def discover_tests( loader = unittest.TestLoader() suite = loader.discover(start_dir, pattern, top_level_dir) - tests, error = build_test_tree(suite, cwd) # test tree built succesfully here. + tests, error = build_test_tree(suite, cwd) # test tree built successfully here. except Exception: error.append(traceback.format_exc()) @@ -121,6 +123,12 @@ def post_response( start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) + django_test_enabled = os.environ.get("DJANGO_TEST_ENABLED", "False") + if django_test_enabled.lower() == "true": + print(f"DJANGO TEST DECLEARED = {django_test_enabled}") + django_env_enabled = setup_django_env(start_dir) + print(f"DJANGO ENV ENABLED = {django_env_enabled}") + testPort = int(os.environ.get("TEST_PORT", DEFAULT_PORT)) testUuid = os.environ.get("TEST_UUID") if testPort is DEFAULT_PORT: diff --git a/pythonFiles/unittestadapter/django_runner.py b/pythonFiles/unittestadapter/django_runner.py index 9aa306f5527e..ecb1606754e0 100644 --- a/pythonFiles/unittestadapter/django_runner.py +++ b/pythonFiles/unittestadapter/django_runner.py @@ -1,13 +1,30 @@ import subprocess import os -import pathlib import sys from typing import Union from pythonFiles.unittestadapter.execution import VSCodeUnittestError +def setup_django_env(start_dir: Union[str, None]): + """Configures the Django environment to run Django tests. + + If Django is not installed or if manage.py can not be found, the function fails quietly. + + Args: + start_dir (str): The root directory of the Django project. + + Returns: + boolean: either succeeded or failed. + """ + + # To avoid false positive ModuleNotFoundError from django.setup() due to missing current workspace in sys.path + sys.path.insert(0, os.getcwd()) + + try: + import django + except ImportError: + return False -def django_execution_runner(start_dir: Union[str, None]): # Get path to manage.py if set as an env var, otherwise use the default manage_py_path = os.environ.get("MANAGE_PY_PATH") @@ -21,10 +38,56 @@ def django_execution_runner(start_dir: Union[str, None]): cwd = os.path.abspath(start_dir) manage_py_path = os.path.join(cwd, "manage.py") + # NOTE: Unnecessary if django_settings_module is determined otherway.👇🏻>>> + import re + try: + with open(manage_py_path, "r") as f: + manage_py_module = f.readlines() + except FileNotFoundError: + print("Error running Django, manage.py not found") + return False + + django_settings_module = None + pattern = r"^os\.environ\.setdefault\((\'|\")DJANGO_SETTINGS_MODULE(\'|\"), (\'|\")(?P[\w.]+)(\'|\")\)$" + for line in manage_py_module: + matched = re.match(pattern, line.strip()) + if matched is not None: + django_settings_module = matched.groupdict().get("settings_path", None) + break + # NOTE: Unnecessary if django_settings_module is determined otherway.👆🏻<<< + + if django_settings_module is None: + print("Error running Django, django settings module not found") + return False + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", django_settings_module) + + try: + django.setup() + except ModuleNotFoundError: + print("Error running Django, Drat!") + return False + + return True + +def django_execution_runner(start_dir: Union[str, None]): + + _ = setup_django_env(start_dir) + try: # Get path to the custom_test_runner.py parent folder, add to sys.path. - custom_test_runner_dir = pathlib.Path(__file__).parent - sys.path.insert(0, custom_test_runner_dir) + + # TODO: Check backward compatibility https://docs.python.org/3/library/pathlib.html -> New in version 3.4. + # import pathlib + # custom_test_runner_dir = pathlib.Path(__file__).parent + # sys.path.insert(0, custom_test_runner_dir) + + sys.path.insert( + 0, + os.path.dirname( # pythonFiles/unittestadapter + os.path.abspath(__file__) # this file + ) + ) custom_test_runner = "django_test_runner.CustomTestRunner" # Build command to run 'python manage.py test'. From 2fa094522ae55c14c81bb3eb587149a7f2af9eea Mon Sep 17 00:00:00 2001 From: Mahdi Firouzjah <44068054+mh-firouzjah@users.noreply.github.com> Date: Sun, 22 Oct 2023 19:52:30 +0330 Subject: [PATCH 9/9] Updated: try to read DJANGO_SETTINGS_MODULE from env first --- pythonFiles/unittestadapter/django_runner.py | 34 +++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pythonFiles/unittestadapter/django_runner.py b/pythonFiles/unittestadapter/django_runner.py index ecb1606754e0..7378d50f50c8 100644 --- a/pythonFiles/unittestadapter/django_runner.py +++ b/pythonFiles/unittestadapter/django_runner.py @@ -38,23 +38,25 @@ def setup_django_env(start_dir: Union[str, None]): cwd = os.path.abspath(start_dir) manage_py_path = os.path.join(cwd, "manage.py") - # NOTE: Unnecessary if django_settings_module is determined otherway.👇🏻>>> - import re - try: - with open(manage_py_path, "r") as f: - manage_py_module = f.readlines() - except FileNotFoundError: - print("Error running Django, manage.py not found") - return False + django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE", None) - django_settings_module = None - pattern = r"^os\.environ\.setdefault\((\'|\")DJANGO_SETTINGS_MODULE(\'|\"), (\'|\")(?P[\w.]+)(\'|\")\)$" - for line in manage_py_module: - matched = re.match(pattern, line.strip()) - if matched is not None: - django_settings_module = matched.groupdict().get("settings_path", None) - break - # NOTE: Unnecessary if django_settings_module is determined otherway.👆🏻<<< + if django_settings_module is None: + print("Warning running Django, missing django settings module in environment, reading from manage.py") + + import re + try: + with open(manage_py_path, "r") as f: + manage_py_module = f.readlines() + except FileNotFoundError: + print("Error running Django, manage.py not found") + return False + + pattern = r"^os\.environ\.setdefault\((\'|\")DJANGO_SETTINGS_MODULE(\'|\"), (\'|\")(?P[\w.]+)(\'|\")\)$" + for line in manage_py_module: + matched = re.match(pattern, line.strip()) + if matched is not None: + django_settings_module = matched.groupdict().get("settings_path", None) + break if django_settings_module is None: print("Error running Django, django settings module not found")