diff --git a/lib/ansible/_internal/_ansiballz/_builder.py b/lib/ansible/_internal/_ansiballz/_builder.py index eff6392904c..76c756fe195 100644 --- a/lib/ansible/_internal/_ansiballz/_builder.py +++ b/lib/ansible/_internal/_ansiballz/_builder.py @@ -6,7 +6,7 @@ import json import typing as t from ansible.module_utils._internal._ansiballz import _extensions -from ansible.module_utils._internal._ansiballz._extensions import _pydevd, _coverage +from ansible.module_utils._internal._ansiballz._extensions import _debugpy, _pydevd, _coverage from ansible.constants import config _T = t.TypeVar('_T') @@ -17,15 +17,18 @@ class ExtensionManager: def __init__( self, - debugger: _pydevd.Options | None = None, + pydevd: _pydevd.Options | None = None, + debugpy: _debugpy.Options | None = None, coverage: _coverage.Options | None = None, ) -> None: options = dict( - _pydevd=debugger, + _pydevd=pydevd, + _debugpy=debugpy, _coverage=coverage, ) - self._debugger = debugger + self._pydevd = pydevd + self._debugpy = debugpy self._coverage = coverage self._extension_names = tuple(name for name, option in options.items() if option) self._module_names = tuple(f'{_extensions.__name__}.{name}' for name in self._extension_names) @@ -35,7 +38,7 @@ class ExtensionManager: @property def debugger_enabled(self) -> bool: """Returns True if the debugger extension is enabled, otherwise False.""" - return bool(self._debugger) + return bool(self._pydevd or self._debugpy) @property def extension_names(self) -> tuple[str, ...]: @@ -51,10 +54,16 @@ class ExtensionManager: """Return the configured extensions and their options.""" extension_options: dict[str, t.Any] = {} - if self._debugger: + if self._debugpy: + extension_options['_debugpy'] = dataclasses.replace( + self._debugpy, + source_mapping=self._get_source_mapping(self._debugpy.source_mapping), + ) + + if self._pydevd: extension_options['_pydevd'] = dataclasses.replace( - self._debugger, - source_mapping=self._get_source_mapping(), + self._pydevd, + source_mapping=self._get_source_mapping(self._pydevd.source_mapping), ) if self._coverage: @@ -64,18 +73,19 @@ class ExtensionManager: return extensions - def _get_source_mapping(self) -> dict[str, str]: + def _get_source_mapping(self, debugger_mapping: dict[str, str]) -> dict[str, str]: """Get the source mapping, adjusting the source root as needed.""" - if self._debugger.source_mapping: - source_mapping = {self._translate_path(key): value for key, value in self.source_mapping.items()} + if debugger_mapping: + source_mapping = {self._translate_path(key, debugger_mapping): value for key, value in self.source_mapping.items()} else: source_mapping = self.source_mapping return source_mapping - def _translate_path(self, path: str) -> str: + @staticmethod + def _translate_path(path: str, debugger_mapping: dict[str, str]) -> str: """Translate a local path to a foreign path.""" - for replace, match in self._debugger.source_mapping.items(): + for replace, match in debugger_mapping.items(): if path.startswith(match): return replace + path[len(match) :] @@ -85,7 +95,8 @@ class ExtensionManager: def create(cls, task_vars: dict[str, object]) -> t.Self: """Create an instance using the provided task vars.""" return cls( - debugger=cls._get_options('_ANSIBALLZ_DEBUGGER_CONFIG', _pydevd.Options, task_vars), + pydevd=cls._get_options('_ANSIBALLZ_PYDEVD_CONFIG', _pydevd.Options, task_vars), + debugpy=cls._get_options('_ANSIBALLZ_DEBUGPY_CONFIG', _debugpy.Options, task_vars), coverage=cls._get_options('_ANSIBALLZ_COVERAGE_CONFIG', _coverage.Options, task_vars), ) diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 81c1b7c56e7..e9632326a67 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -11,16 +11,26 @@ _ANSIBALLZ_COVERAGE_CONFIG: vars: - {name: _ansible_ansiballz_coverage_config} version_added: '2.19' -_ANSIBALLZ_DEBUGGER_CONFIG: - name: Configure the AnsiballZ remote debugging extension +_ANSIBALLZ_DEBUGPY_CONFIG: + name: Configure the AnsiballZ remote debugging extension for debugpy description: - - Enables and configures the AnsiballZ remote debugging extension. + - Enables and configures the AnsiballZ remote debugging extension for debugpy. - This is for internal use only. env: - - {name: _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG} + - {name: _ANSIBLE_ANSIBALLZ_DEBUGPY_CONFIG} vars: - - {name: _ansible_ansiballz_debugger_config} - version_added: '2.19' + - {name: _ansible_ansiballz_debugpy_config} + version_added: '2.20' +_ANSIBALLZ_PYDEVD_CONFIG: + name: Configure the AnsiballZ remote debugging extension for pydevd + description: + - Enables and configures the AnsiballZ remote debugging extension for pydevd. + - This is for internal use only. + env: + - {name: _ANSIBLE_ANSIBALLZ_PYDEVD_CONFIG} + vars: + - {name: _ansible_ansiballz_pydevd_config} + version_added: '2.20' _ANSIBLE_CONNECTION_PATH: env: - name: _ANSIBLE_CONNECTION_PATH diff --git a/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_debugpy.py b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_debugpy.py new file mode 100644 index 00000000000..df411962e2c --- /dev/null +++ b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_debugpy.py @@ -0,0 +1,97 @@ +""" +Remote debugging support for AnsiballZ modules with debugpy. + +To use with VS Code: + +1) Choose an available port for VS Code to listen on (e.g. 5678). +2) Ensure `debugpy` is installed for the interpreter(s) which will run the code being debugged. +3) Create the following launch.json configuration + + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debug Server", + "type": "debugpy", + "request": "attach", + "listen": { + "host": "localhost", + "port": 5678, + }, + }, + { + "name": "ansible-playbook main.yml", + "type": "debugpy", + "request": "launch", + "module": "ansible", + "args": [ + "playbook", + "main.yml" + ], + "env": { + "_ANSIBLE_ANSIBALLZ_DEBUGPY_CONFIG": "{\"host\": \"localhost\", \"port\": 5678}" + }, + "console": "integratedTerminal", + } + ], + "compounds": [ + { + "name": "Test Module Debugging", + "configurations": [ + "Python Debug Server", + "ansible-playbook main.yml" + ], + "stopAll": true + } + ] + } + +4) Set any desired breakpoints. +5) Configure the Run and Debug view to use the "Test Module Debugging" compound configuration. +6) Press F5 to start debugging. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import pathlib + +import typing as t + + +@dataclasses.dataclass(frozen=True) +class Options: + """Debugger options for debugpy.""" + + host: str = 'localhost' + """The host to connect to for remote debugging.""" + port: int = 5678 + """The port to connect to for remote debugging.""" + connect: dict[str, object] = dataclasses.field(default_factory=dict) + """The options to pass to the `debugpy.connect` method.""" + source_mapping: dict[str, str] = dataclasses.field(default_factory=dict) + """ + A mapping of source paths to provide to debugpy. + This setting is used internally by AnsiballZ and is not required unless Ansible CLI commands are run from a different system than your IDE. + In that scenario, use this setting instead of configuring source mapping in your IDE. + The key is a path known to the IDE. + The value is the same path as known to the Ansible CLI. + Both file paths and directories are supported. + """ + + +def run(args: dict[str, t.Any]) -> None: # pragma: nocover + """Enable remote debugging.""" + import debugpy + + options = Options(**args) + temp_dir = pathlib.Path(__file__).parent.parent.parent.parent.parent.parent + path_mapping = [[key, str(temp_dir / value)] for key, value in options.source_mapping.items()] + + os.environ['PATHS_FROM_ECLIPSE_TO_PYTHON'] = json.dumps(path_mapping) + + debugpy.connect((options.host, options.port), **options.connect) + + pass # A convenient place to put a breakpoint diff --git a/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py index eec234d9d10..c82232d91a4 100644 --- a/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py +++ b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py @@ -7,14 +7,12 @@ To use with PyCharm: 2) Create a Python Debug Server using that port. 3) Start the Python Debug Server. 4) Ensure the correct version of `pydevd-pycharm` is installed for the interpreter(s) which will run the code being debugged. -5) Configure Ansible with the `_ANSIBALLZ_DEBUGGER_CONFIG` option. +5) Configure Ansible with the `_ANSIBALLZ_PYDEVD_CONFIG` option. See `Options` below for the structure of the debugger configuration. Example configuration using an environment variable: - export _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG='{"module": "pydevd_pycharm", "settrace": {"host": "localhost", "port": 5678, "suspend": false}}' + export _ANSIBLE_ANSIBALLZ_PYDEVD_CONFIG='{"module": "pydevd_pycharm", "settrace": {"host": "localhost", "port": 5678, "suspend": false}}' 6) Set any desired breakpoints. 7) Run Ansible commands. - -A similar process should work for other pydevd based debuggers, such as Visual Studio Code, but they have not been tested. """ from __future__ import annotations diff --git a/test/integration/targets/ansiballz_debugging/tasks/main.yml b/test/integration/targets/ansiballz_debugging/tasks/main.yml index 5d052ed967c..9d8dd7e248b 100644 --- a/test/integration/targets/ansiballz_debugging/tasks/main.yml +++ b/test/integration/targets/ansiballz_debugging/tasks/main.yml @@ -1,7 +1,7 @@ - name: Run a module with remote debugging configured to use a bogus debugger module ping: vars: - _ansible_ansiballz_debugger_config: + _ansible_ansiballz_pydevd_config: module: not_a_valid_debugger_module register: result ignore_errors: yes diff --git a/test/integration/targets/ansible-test-debugging-env/aliases b/test/integration/targets/ansible-test-debugging-env/aliases new file mode 100644 index 00000000000..bf60838a88a --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-env/aliases @@ -0,0 +1,2 @@ +shippable/generic/group1 # runs in the default test container +context/controller diff --git a/test/integration/targets/ansible-test-debugging-env/runme.sh b/test/integration/targets/ansible-test-debugging-env/runme.sh new file mode 100755 index 00000000000..1ec9edd7ad8 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-env/runme.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Used to support the ansible-test-debugging integration test. + +env diff --git a/test/integration/targets/ansible-test-debugging-inventory/aliases b/test/integration/targets/ansible-test-debugging-inventory/aliases new file mode 100644 index 00000000000..bf60838a88a --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-inventory/aliases @@ -0,0 +1,2 @@ +shippable/generic/group1 # runs in the default test container +context/controller diff --git a/test/integration/targets/ansible-test-debugging-inventory/runme.sh b/test/integration/targets/ansible-test-debugging-inventory/runme.sh new file mode 100755 index 00000000000..48d4c626190 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-inventory/runme.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Used to support the ansible-test-debugging integration test. + +cat "${INVENTORY_PATH}" diff --git a/test/integration/targets/ansible-test-debugging/aliases b/test/integration/targets/ansible-test-debugging/aliases new file mode 100644 index 00000000000..69cffa10864 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging/aliases @@ -0,0 +1,5 @@ +shippable/generic/group1 # runs in the default test container +needs/target/ansible-test-debugging-env # indirectly used by ansible-test, included here for change detection +needs/target/ansible-test-debugging-inventory # indirectly used by ansible-test, included here for change detection +context/controller +gather_facts/no diff --git a/test/integration/targets/ansible-test-debugging/tasks/main.yml b/test/integration/targets/ansible-test-debugging/tasks/main.yml new file mode 100644 index 00000000000..f91159441b5 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging/tasks/main.yml @@ -0,0 +1,98 @@ +- name: Define supported debuggers + set_fact: + debuggers: + - pydevd + - debugpy + +- name: Run without debugging features enabled + command: ansible-test shell -v -- env + register: result + +- assert: + that: + - result.stderr_lines is not contains 'Debugging' + +- name: Run a command which does not support debugging + command: ansible-test env -v + register: result + +- assert: + that: + - result.stderr_lines is not contains 'Debugging' + +- name: Verify on-demand debugging gracefully handles not running under a debugger + command: ansible-test shell -v --dev-debug-on-demand -- env + register: result + +- assert: + that: + - result.stderr_lines is contains 'Debugging disabled because no debugger was detected.' + +- name: Verify manual debugging gracefully handles lack of configuration + command: ansible-test shell -v --dev-debug-cli -- env + register: result + +- assert: + that: + - result.stderr_lines is contains 'Debugging disabled because no debugger configuration was provided.' + +- name: Verify invalid debugger configuration is handled + command: ansible-test shell --dev-debug-cli -- env + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": "{\"invalid_key\": true}"}') | from_json }} + register: result + loop: "{{ debuggers }}" + ignore_errors: yes + +- assert: + that: + - item.stderr is search("Invalid " + item.item + " settings.*invalid_key") + loop: "{{ result.results }}" + +- name: Verify CLI debugger can be manually enabled (shell) + command: ansible-test shell --dev-debug-cli -- env + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains "ANSIBLE_TEST_DEBUGGER_CONFIG" + loop: "{{ result.results }}" + +- name: Verify CLI debugger can be manually enabled (integration) + command: ansible-test integration ansible-test-debugging-env --dev-debug-cli + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains "ANSIBLE_TEST_DEBUGGER_CONFIG" + loop: "{{ result.results }}" + +- name: Verify AnsiballZ debugger can be manually enabled (shell) + command: ansible-test shell --dev-debug-ansiballz -- env + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains("_ANSIBLE_ANSIBALLZ_" + item.item.upper() + "_CONFIG") + loop: "{{ result.results }}" + +- name: Verify AnsiballZ debugger can be manually enabled (integration) + command: ansible-test integration ansible-test-debugging-inventory --dev-debug-ansiballz + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains("_ansible_ansiballz_" + item.item + "_config") + loop: "{{ result.results }}" diff --git a/test/lib/ansible_test/_internal/commands/shell/__init__.py b/test/lib/ansible_test/_internal/commands/shell/__init__.py index dc14b1659b7..89fbd8d72c8 100644 --- a/test/lib/ansible_test/_internal/commands/shell/__init__.py +++ b/test/lib/ansible_test/_internal/commands/shell/__init__.py @@ -2,11 +2,16 @@ from __future__ import annotations +import contextlib import dataclasses import os import sys import typing as t +from ...data import ( + data_context, +) + from ...util import ( ApplicationError, OutputStream, @@ -101,19 +106,33 @@ def command_shell(args: ShellConfig) -> None: if args.export: return - if args.cmd: - # Running a command is assumed to be non-interactive. Only a shell (no command) is interactive. - # If we want to support interactive commands in the future, we'll need an `--interactive` command line option. - # Command stderr output is allowed to mix with our own output, which is all sent to stderr. - con.run(args.cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL) - return - if isinstance(con, LocalConnection) and isinstance(target_profile, DebuggableProfile) and target_profile.debugging_enabled: - # HACK: ensure the pydevd port visible in the shell is the forwarded port, not the original - args.metadata.debugger_settings = dataclasses.replace(args.metadata.debugger_settings, port=target_profile.pydevd_port) + # HACK: ensure the debugger port visible in the shell is the forwarded port, not the original + args.metadata.debugger_settings = dataclasses.replace(args.metadata.debugger_settings, port=target_profile.debugger_port) - with metadata_context(args): - interactive_shell(args, target_profile, con) + with contextlib.nullcontext() if data_context().content.unsupported else metadata_context(args): + if args.cmd: + non_interactive_shell(args, target_profile, con) + else: + interactive_shell(args, target_profile, con) + + +def non_interactive_shell( + args: ShellConfig, + target_profile: SshTargetHostProfile, + con: Connection, +) -> None: + """Run a non-interactive shell command.""" + if isinstance(target_profile, PosixProfile): + env = get_environment_variables(args, target_profile, con) + cmd = get_env_command(env) + args.cmd + else: + cmd = args.cmd + + # Running a command is assumed to be non-interactive. Only a shell (no command) is interactive. + # If we want to support interactive commands in the future, we'll need an `--interactive` command line option. + # Command stderr output is allowed to mix with our own output, which is all sent to stderr. + con.run(cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL) def interactive_shell( @@ -135,23 +154,8 @@ def interactive_shell( python = target_profile.python # make sure the python interpreter has been initialized before opening a shell display.info(f'Target Python {python.version} is at: {python.path}') - optional_vars = ( - 'TERM', # keep backspace working - ) - - env = {name: os.environ[name] for name in optional_vars if name in os.environ} - - if isinstance(con, LocalConnection): # configure the controller environment - env.update(ansible_environment(args)) - env.update(get_injector_env(target_profile.python, env)) - env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path)) - - if isinstance(target_profile, DebuggableProfile): - env.update(target_profile.get_ansiballz_environment_variables()) - env.update(target_profile.get_ansible_cli_environment_variables()) - - if env: - cmd = ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()] + env = get_environment_variables(args, target_profile, con) + cmd = get_env_command(env) cmd += [shell, '-i'] else: @@ -175,3 +179,38 @@ def interactive_shell( raise HostConnectionError(f'SSH shell connection failed for host {target_profile.config}: {ex}', callback) from ex raise + + +def get_env_command(env: dict[str, str]) -> list[str]: + """Get an `env` command to set the given environment variables, if any.""" + if not env: + return [] + + return ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()] + + +def get_environment_variables( + args: ShellConfig, + target_profile: PosixProfile, + con: Connection, +) -> dict[str, str]: + """Get the environment variables to expose to the shell.""" + if data_context().content.unsupported: + return {} + + optional_vars = ( + 'TERM', # keep backspace working + ) + + env = {name: os.environ[name] for name in optional_vars if name in os.environ} + + if isinstance(con, LocalConnection): # configure the controller environment + env.update(ansible_environment(args)) + env.update(get_injector_env(target_profile.python, env)) + env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path)) + + if isinstance(target_profile, DebuggableProfile): + env.update(target_profile.get_ansiballz_environment_variables()) + env.update(target_profile.get_ansible_cli_environment_variables()) + + return env diff --git a/test/lib/ansible_test/_internal/debugging.py b/test/lib/ansible_test/_internal/debugging.py index bd5fc452ad9..b3c4a605ec8 100644 --- a/test/lib/ansible_test/_internal/debugging.py +++ b/test/lib/ansible_test/_internal/debugging.py @@ -2,16 +2,25 @@ from __future__ import annotations +import abc import dataclasses +import importlib import json import os import re +import sys +import typing as t from .util import ( cache, display, raw_command, ApplicationError, + get_subclasses, +) + +from .util_common import ( + CommonConfig, ) from .processes import ( @@ -24,76 +33,308 @@ from .config import ( ) from .metadata import ( - DebuggerSettings, DebuggerFlags, ) -from . import ( +from .data import ( data_context, - CommonConfig, ) -def initialize_debugger(args: CommonConfig) -> None: - """Initialize the debugger settings before delegation.""" - if not isinstance(args, EnvironmentConfig): - return +class DebuggerProfile(t.Protocol): + """Protocol for debugger profiles.""" - if args.metadata.loaded: - return # after delegation + @property + def debugger_host(self) -> str: + """The hostname to expose to the debugger.""" - if collection := data_context().content.collection: - args.metadata.collection_root = collection.root - - load_debugger_settings(args) + @property + def debugger_port(self) -> int: + """The port to expose to the debugger.""" + def get_source_mapping(self) -> dict[str, str]: + """The source mapping to expose to the debugger.""" + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebuggerSettings(metaclass=abc.ABCMeta): + """Common debugger settings.""" + + port: int = 5678 + """ + The port on the origin host which is listening for incoming connections from the debugger. + SSH port forwarding will be automatically configured for non-local hosts to connect to this port as needed. + """ + + def as_dict(self) -> dict[str, object]: + """Convert this instance to a dict.""" + data = dataclasses.asdict(self) + data.update(__type__=self.__class__.__name__) + + return data + + @classmethod + def from_dict(cls, value: dict[str, t.Any]) -> t.Self: + """Load an instance from a dict.""" + debug_cls = globals()[value.pop('__type__')] + + return debug_cls(**value) + + @classmethod + def get_debug_type(cls) -> str: + """Return the name for this debugger.""" + return cls.__name__.removesuffix('Settings').lower() + + @classmethod + def get_config_env_var_name(cls) -> str: + """Return the name of the environment variable used to customize settings for this debugger.""" + return f'ANSIBLE_TEST_REMOTE_DEBUGGER_{cls.get_debug_type().upper()}' + + @classmethod + def parse(cls, value: str) -> t.Self: + """Parse debugger settings from the given JSON and apply defaults.""" + try: + settings = cls(**json.loads(value)) + except Exception as ex: + raise ApplicationError(f"Invalid {cls.get_debug_type()} settings: {ex}") from ex + + return cls.apply_defaults(settings) + + @classmethod + @abc.abstractmethod + def is_active(cls) -> bool: + """Detect if the debugger is active.""" + + @classmethod + @abc.abstractmethod + def apply_defaults(cls, settings: t.Self) -> t.Self: + """Apply defaults to the given settings.""" + + @abc.abstractmethod + def get_python_package(self) -> str: + """The Python package to install for debugging.""" + + @abc.abstractmethod + def activate_debugger(self, profile: DebuggerProfile) -> None: + """Activate the debugger in ansible-test after delegation.""" + + @abc.abstractmethod + def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]: + """Gets the extra configuration data for the AnsiballZ extension module.""" + + @abc.abstractmethod + def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]: + """Get command line arguments for the debugger when running Ansible CLI programs.""" + + @abc.abstractmethod + def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]: + """Get environment variables needed to configure the debugger for debugging.""" + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class PydevdSettings(DebuggerSettings): + """Settings for the pydevd debugger.""" + + package: str | None = None + """ + The Python package to install for debugging. + If `None` then the package will be auto-detected. + If an empty string, then no package will be installed. + """ + + module: str | None = None + """ + The Python module to import for debugging. + This should be pydevd or a derivative. + If not provided it will be auto-detected. + """ + + settrace: dict[str, object] = dataclasses.field(default_factory=dict) + """ + Options to pass to the `{module}.settrace` method. + Used for running AnsiballZ modules only. + The `host` and `port` options will be provided by ansible-test. + The `suspend` option defaults to `False`. + """ + + args: list[str] = dataclasses.field(default_factory=list) + """ + Arguments to pass to `pydevd` on the command line. + Used for running Ansible CLI programs only. + The `--client` and `--port` options will be provided by ansible-test. + """ + + @classmethod + def is_active(cls) -> bool: + return detect_pydevd_port() is not None + + @classmethod + def apply_defaults(cls, settings: t.Self) -> t.Self: + if not settings.module: + if not settings.package or 'pydevd-pycharm' in settings.package: + module = 'pydevd_pycharm' + else: + module = 'pydevd' -def parse_debugger_settings(value: str) -> DebuggerSettings: - """Parse remote debugger settings and apply defaults.""" - try: - settings = DebuggerSettings(**json.loads(value)) - except Exception as ex: - raise ApplicationError(f"Invalid debugger settings: {ex}") from ex + settings = dataclasses.replace(settings, module=module) - if not settings.module: - if not settings.package or 'pydevd-pycharm' in settings.package: - module = 'pydevd_pycharm' + if settings.package is None: + if settings.module == 'pydevd_pycharm': + if pycharm_version := detect_pycharm_version(): + package = f'pydevd-pycharm~={pycharm_version}' + else: + package = None + else: + package = 'pydevd' + + settings = dataclasses.replace(settings, package=package) + + settings.settrace.setdefault('suspend', False) + + if port := detect_pydevd_port(): + settings = dataclasses.replace(settings, port=port) + + if detect_pycharm_process(): + # This only works with the default PyCharm debugger. + # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers. + # Further investigation is required to understand the cause. + settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess']) + + return settings + + def get_python_package(self) -> str: + if self.package is None and self.module == 'pydevd_pycharm': + display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version was not detected.') + + return self.package + + def activate_debugger(self, profile: DebuggerProfile) -> None: + debugging_module = importlib.import_module(self.module) + debugging_module.settrace(**self._get_settrace_arguments(profile)) + + def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]: + return dict( + module=self.module, + settrace=self._get_settrace_arguments(profile), + source_mapping=profile.get_source_mapping(), + ) + + def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]: + # Although `pydevd_pycharm` can be used to invoke `settrace`, it cannot be used to run the debugger on the command line. + return ['-m', 'pydevd', '--client', profile.debugger_host, '--port', str(profile.debugger_port)] + self.args + ['--file'] + + def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]: + return dict( + PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(profile.get_source_mapping().items())), + PYDEVD_DISABLE_FILE_VALIDATION="1", + ) + + def _get_settrace_arguments(self, profile: DebuggerProfile) -> dict[str, object]: + """Get settrace arguments for pydevd.""" + return self.settrace | dict( + host=profile.debugger_host, + port=profile.debugger_port, + ) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebugpySettings(DebuggerSettings): + """Settings for the debugpy debugger.""" + + connect: dict[str, object] = dataclasses.field(default_factory=dict) + """ + Options to pass to the `debugpy.connect` method. + Used for running AnsiballZ modules and ansible-test after delegation. + The endpoint addr, `access_token`, and `parent_session_pid` options will be provided by ansible-test. + """ + + args: list[str] = dataclasses.field(default_factory=list) + """ + Arguments to pass to `debugpy` on the command line. + Used for running Ansible CLI programs only. + The `--connect`, `--adapter-access-token`, and `--parent-session-pid` options will be provided by ansible-test. + """ + + @classmethod + def is_active(cls) -> bool: + return detect_debugpy_options() is not None + + @classmethod + def apply_defaults(cls, settings: t.Self) -> t.Self: + if options := detect_debugpy_options(): + settings = dataclasses.replace(settings, port=options.port) + settings.connect.update( + access_token=options.adapter_access_token, + parent_session_pid=os.getpid(), + ) else: - module = 'pydevd' + display.warning('Debugging will be limited to the first connection. Run ansible-test under debugpy to support multiple connections.') - settings = dataclasses.replace(settings, module=module) + return settings - if settings.package is None: - if settings.module == 'pydevd_pycharm': - if pycharm_version := detect_pycharm_version(): - package = f'pydevd-pycharm~={pycharm_version}' - else: - package = None - else: - package = 'pydevd' + def get_python_package(self) -> str: + return 'debugpy' + + def activate_debugger(self, profile: DebuggerProfile) -> None: + import debugpy # pylint: disable=import-error + + debugpy.connect((profile.debugger_host, profile.debugger_port), **self.connect) + + def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]: + return dict( + host=profile.debugger_host, + port=profile.debugger_port, + connect=self.connect, + source_mapping=profile.get_source_mapping(), + ) + + def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]: + cli_args = ['-m', 'debugpy', '--connect', f"{profile.debugger_host}:{profile.debugger_port}"] + + if access_token := self.connect.get('access_token'): + cli_args += ['--adapter-access-token', str(access_token)] + + if session_pid := self.connect.get('parent_session_pid'): + cli_args += ['--parent-session-pid', str(session_pid)] + + if self.args: + cli_args += self.args - settings = dataclasses.replace(settings, package=package) + return cli_args - settings.settrace.setdefault('suspend', False) + def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]: + return dict( + PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(profile.get_source_mapping().items())), + PYDEVD_DISABLE_FILE_VALIDATION="1", + ) - if port := detect_pydevd_port(): - settings = dataclasses.replace(settings, port=port) - if detect_pycharm_process(): - # This only works with the default PyCharm debugger. - # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers. - # Further investigation is required to understand the cause. - settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess']) +def initialize_debugger(args: CommonConfig) -> None: + """Initialize the debugger settings before delegation.""" + if not isinstance(args, EnvironmentConfig): + return - return settings + if args.metadata.loaded: + return # after delegation + + if collection := data_context().content.collection: + args.metadata.collection_root = collection.root + + load_debugger_settings(args) def load_debugger_settings(args: EnvironmentConfig) -> None: """Load the remote debugger settings.""" + use_debugger: type[DebuggerSettings] | None = None + if args.metadata.debugger_flags.on_demand: # On-demand debugging only enables debugging if we're running under a debugger, otherwise it's a no-op. - if not detect_pydevd_port(): + for candidate_debugger in get_subclasses(DebuggerSettings): + if candidate_debugger.is_active(): + use_debugger = candidate_debugger + break + else: display.info('Debugging disabled because no debugger was detected.', verbosity=1) args.metadata.debugger_flags = DebuggerFlags.all(False) return @@ -107,13 +348,22 @@ def load_debugger_settings(args: EnvironmentConfig) -> None: if not args.metadata.debugger_flags.enable: return - value = os.environ.get('ANSIBLE_TEST_REMOTE_DEBUGGER') or '{}' - settings = parse_debugger_settings(value) - - display.info(f'>>> Debugger Settings\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3) + if not use_debugger: # detect debug type based on env var + for candidate_debugger in get_subclasses(DebuggerSettings): + if candidate_debugger.get_config_env_var_name() in os.environ: + use_debugger = candidate_debugger + break + else: + display.info('Debugging disabled because no debugger configuration was provided.', verbosity=1) + args.metadata.debugger_flags = DebuggerFlags.all(False) + return + config = os.environ.get(use_debugger.get_config_env_var_name()) or '{}' + settings = use_debugger.parse(config) args.metadata.debugger_settings = settings + display.info(f'>>> Debugger Settings ({use_debugger.get_debug_type()})\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3) + @cache def detect_pydevd_port() -> int | None: @@ -140,8 +390,6 @@ def detect_pycharm_version() -> str | None: display.info(f'Detected PyCharm version {version}.', verbosity=1) return version - display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version could not be detected.') - return None @@ -164,3 +412,43 @@ def detect_pycharm_process() -> Process | None: def get_current_process_cached() -> Process: """Return the current process. The result is cached.""" return get_current_process() + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebugpyOptions: + """Options detected from the debugpy instance hosting this process.""" + + port: int + adapter_access_token: str | None + + +@cache +def detect_debugpy_options() -> DebugpyOptions | None: + """Return the options for the debugpy instance hosting this process, or `None` if not detected.""" + if "debugpy" not in sys.modules: + return None + + import debugpy # pylint: disable=import-error + + # get_cli_options is the new public API introduced after debugpy 1.8.15. + # We should remove the debugpy.server cli fallback once the new version is + # released. + if hasattr(debugpy, 'get_cli_options'): + opts = debugpy.get_cli_options() + else: + from debugpy.server import cli # pylint: disable=import-error + opts = cli.options + + # address can be None if the debugger is not configured through the CLI as + # we expected. + if not opts.address: + return None + + port = opts.address[1] + + display.info(f'Detected debugpy debugger port {port}.', verbosity=1) + + return DebugpyOptions( + port=port, + adapter_access_token=opts.adapter_access_token, + ) diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index dc28b66c17a..f8d5fbf1e19 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -4,7 +4,6 @@ from __future__ import annotations import abc import dataclasses -import importlib import json import os import pathlib @@ -140,6 +139,11 @@ from .dev.container_probe import ( check_container_cgroup_status, ) +from .debugging import ( + DebuggerProfile, + DebuggerSettings, +) + TControllerHostConfig = t.TypeVar('TControllerHostConfig', bound=ControllerHostConfig) THostConfig = t.TypeVar('THostConfig', bound=HostConfig) TPosixConfig = t.TypeVar('TPosixConfig', bound=PosixConfig) @@ -292,12 +296,17 @@ class HostProfile(t.Generic[THostConfig], metaclass=abc.ABCMeta): return f'{self.__class__.__name__}: {self.name}' -class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): +class DebuggableProfile(HostProfile[THostConfig], DebuggerProfile, metaclass=abc.ABCMeta): """Base class for profiles remote debugging.""" - __PYDEVD_PORT_KEY = 'pydevd_port' + __DEBUGGING_PORT_KEY = 'debugging_port' __DEBUGGING_FORWARDER_KEY = 'debugging_forwarder' + @property + def debugger(self) -> DebuggerSettings | None: + """The debugger settings for this host if present and enabled, otherwise None.""" + return self.args.metadata.debugger_settings + @property def debugging_enabled(self) -> bool: """Returns `True` if debugging is enabled for this profile, otherwise `False`.""" @@ -307,9 +316,14 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): return self.args.metadata.debugger_flags.ansiballz @property - def pydevd_port(self) -> int: - """The pydevd port to use.""" - return self.state.get(self.__PYDEVD_PORT_KEY) or self.origin_pydev_port + def debugger_host(self) -> str: + """The debugger host to use.""" + return 'localhost' + + @property + def debugger_port(self) -> int: + """The debugger port to use.""" + return self.state.get(self.__DEBUGGING_PORT_KEY) or self.origin_debugger_port @property def debugging_forwarder(self) -> SshProcess | None: @@ -322,23 +336,23 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): self.cache[self.__DEBUGGING_FORWARDER_KEY] = value @property - def origin_pydev_port(self) -> int: - """The pydevd port on the origin.""" - return self.args.metadata.debugger_settings.port + def origin_debugger_port(self) -> int: + """The debugger port on the origin.""" + return self.debugger.port def enable_debugger_forwarding(self, ssh: SshConnectionDetail) -> None: - """Enable pydevd port forwarding from the origin.""" + """Enable debugger port forwarding from the origin.""" if not self.debugging_enabled: return - endpoint = ('localhost', self.origin_pydev_port) + endpoint = ('localhost', self.origin_debugger_port) forwards = [endpoint] self.debugging_forwarder = create_ssh_port_forwards(self.args, ssh, forwards) port_forwards = self.debugging_forwarder.collect_port_forwards() - self.state[self.__PYDEVD_PORT_KEY] = port = port_forwards[endpoint] + self.state[self.__DEBUGGING_PORT_KEY] = port = port_forwards[endpoint] display.info(f'Remote debugging of {self.name!r} is available on port {port}.', verbosity=1) @@ -355,19 +369,6 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): self.debugging_forwarder.wait() - def get_pydevd_settrace_arguments(self) -> dict[str, object]: - """Get settrace arguments for pydevd.""" - return self.args.metadata.debugger_settings.settrace | dict( - host="localhost", - port=self.pydevd_port, - ) - - def get_pydevd_environment_variables(self) -> dict[str, str]: - """Get environment variables needed to configure pydevd for debugging.""" - return dict( - PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(self.get_source_mapping().items())), - ) - def get_source_mapping(self) -> dict[str, str]: """Get the source mapping from the given metadata.""" from . import data_context @@ -396,10 +397,9 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): display.info('Activating remote debugging of ansible-test.', verbosity=1) - os.environ.update(self.get_pydevd_environment_variables()) + os.environ.update(self.debugger.get_environment_variables(self)) - debugging_module = importlib.import_module(self.args.metadata.debugger_settings.module) - debugging_module.settrace(**self.get_pydevd_settrace_arguments()) + self.debugger.activate_debugger(self) pass # pylint: disable=unnecessary-pass # when suspend is True, execution pauses here -- it's also a convenient place to put a breakpoint @@ -411,9 +411,11 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): if not self.args.metadata.debugger_flags.ansiballz: return {} - return dict( - _ansible_ansiballz_debugger_config=json.dumps(self.get_ansiballz_debugger_config()), - ) + debug_type = self.debugger.get_debug_type() + + return { + f"_ansible_ansiballz_{debug_type}_config": json.dumps(self.get_ansiballz_debugger_config()), + } def get_ansiballz_environment_variables(self) -> dict[str, t.Any]: """ @@ -423,20 +425,18 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): if not self.args.metadata.debugger_flags.ansiballz: return {} - return dict( - _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG=json.dumps(self.get_ansiballz_debugger_config()), - ) + debug_type = self.debugger.get_debug_type().upper() + + return { + f"_ANSIBLE_ANSIBALLZ_{debug_type}_CONFIG": json.dumps(self.get_ansiballz_debugger_config()), + } def get_ansiballz_debugger_config(self) -> dict[str, t.Any]: """ Return config for remote debugging of AnsiballZ modules. When delegating, this function must be called after delegation. """ - debugger_config = dict( - module=self.args.metadata.debugger_settings.module, - settrace=self.get_pydevd_settrace_arguments(), - source_mapping=self.get_source_mapping(), - ) + debugger_config = self.debugger.get_ansiballz_config(self) display.info(f'>>> Debugger Config ({self.name} AnsiballZ)\n{json.dumps(debugger_config, indent=4)}', verbosity=3) @@ -451,8 +451,8 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): return {} debugger_config = dict( - args=['-m', 'pydevd', '--client', 'localhost', '--port', str(self.pydevd_port)] + self.args.metadata.debugger_settings.args + ['--file'], - env=self.get_pydevd_environment_variables(), + args=self.debugger.get_cli_arguments(self), + env=self.debugger.get_environment_variables(self), ) display.info(f'>>> Debugger Config ({self.name} Ansible CLI)\n{json.dumps(debugger_config, indent=4)}', verbosity=3) @@ -597,9 +597,9 @@ class ControllerProfile(SshTargetHostProfile[ControllerConfig], PosixProfile[Con return self.controller_profile.name @property - def pydevd_port(self) -> int: + def debugger_port(self) -> int: """The pydevd port to use.""" - return self.controller_profile.pydevd_port + return self.controller_profile.debugger_port def get_controller_target_connections(self) -> list[SshConnection]: """Return SSH connection(s) for accessing the host as a target from the controller.""" diff --git a/test/lib/ansible_test/_internal/metadata.py b/test/lib/ansible_test/_internal/metadata.py index 5aaceecd31c..7f9f6e0bc30 100644 --- a/test/lib/ansible_test/_internal/metadata.py +++ b/test/lib/ansible_test/_internal/metadata.py @@ -22,6 +22,9 @@ from .diff import ( FileDiff, ) +if t.TYPE_CHECKING: + from .debugging import DebuggerSettings + class Metadata: """Metadata object for passing data to delegated tests.""" @@ -71,7 +74,7 @@ class Metadata: ansible_test_root=self.ansible_test_root, collection_root=self.collection_root, debugger_flags=dataclasses.asdict(self.debugger_flags), - debugger_settings=dataclasses.asdict(self.debugger_settings) if self.debugger_settings else None, + debugger_settings=self.debugger_settings.as_dict() if self.debugger_settings else None, ) def to_file(self, path: str) -> None: @@ -91,6 +94,8 @@ class Metadata: @staticmethod def from_dict(data: dict[str, t.Any]) -> Metadata: """Return metadata loaded from the specified dictionary.""" + from .debugging import DebuggerSettings + metadata = Metadata( debugger_flags=DebuggerFlags(**data['debugger_flags']), ) @@ -103,7 +108,7 @@ class Metadata: metadata.ansible_lib_root = data['ansible_lib_root'] metadata.ansible_test_root = data['ansible_test_root'] metadata.collection_root = data['collection_root'] - metadata.debugger_settings = DebuggerSettings(**data['debugger_settings']) if data['debugger_settings'] else None + metadata.debugger_settings = DebuggerSettings.from_dict(data['debugger_settings']) if data['debugger_settings'] else None metadata.loaded = True return metadata @@ -155,46 +160,6 @@ class ChangeDescription: return changes -@dataclasses.dataclass(frozen=True, kw_only=True) -class DebuggerSettings: - """Settings for remote debugging.""" - - module: str | None = None - """ - The Python module to import. - This should be pydevd or a derivative. - If not provided it will be auto-detected. - """ - - package: str | None = None - """ - The Python package to install for debugging. - If `None` then the package will be auto-detected. - If an empty string, then no package will be installed. - """ - - settrace: dict[str, object] = dataclasses.field(default_factory=dict) - """ - Options to pass to the `{module}.settrace` method. - Used for running AnsiballZ modules only. - The `host` and `port` options will be provided by ansible-test. - The `suspend` option defaults to `False`. - """ - - args: list[str] = dataclasses.field(default_factory=list) - """ - Arguments to pass to `pydevd` on the command line. - Used for running Ansible CLI programs only. - The `--client` and `--port` options will be provided by ansible-test. - """ - - port: int = 5678 - """ - The port on the origin host which is listening for incoming connections from pydevd. - SSH port forwarding will be automatically configured for non-local hosts to connect to this port as needed. - """ - - @dataclasses.dataclass(frozen=True, kw_only=True) class DebuggerFlags: """Flags for enabling specific debugging features.""" diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index 68077542eaf..32041555900 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -170,11 +170,11 @@ def install_requirements( from .host_profiles import DebuggableProfile - if isinstance(host_profile, DebuggableProfile) and host_profile.debugging_enabled and args.metadata.debugger_settings.package: + if isinstance(host_profile, DebuggableProfile) and host_profile.debugger and host_profile.debugger.get_python_package(): commands.append(PipInstall( requirements=[], constraints=[], - packages=[args.metadata.debugger_settings.package], + packages=[host_profile.debugger.get_python_package()], )) if not commands: diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg index d9219b5d4ae..42aba85c7db 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg @@ -21,6 +21,7 @@ disable= broad-exception-raised, # many exceptions with no need for a custom type too-few-public-methods, too-many-public-methods, + too-many-ancestors, too-many-arguments, too-many-branches, too-many-instance-attributes, diff --git a/test/sanity/code-smell/mypy/ansible-core.ini b/test/sanity/code-smell/mypy/ansible-core.ini index 94c3596ae50..ae51059471b 100644 --- a/test/sanity/code-smell/mypy/ansible-core.ini +++ b/test/sanity/code-smell/mypy/ansible-core.ini @@ -133,3 +133,6 @@ ignore_missing_imports = True [mypy-jinja2.nativetypes] ignore_missing_imports = True + +[mypy-debugpy] +ignore_missing_imports = True diff --git a/test/sanity/code-smell/mypy/ansible-test.ini b/test/sanity/code-smell/mypy/ansible-test.ini index 81a5d64e4fc..2fd72dd9a95 100644 --- a/test/sanity/code-smell/mypy/ansible-test.ini +++ b/test/sanity/code-smell/mypy/ansible-test.ini @@ -77,3 +77,9 @@ ignore_missing_imports = True [mypy-py._path.local] ignore_missing_imports = True + +[mypy-debugpy] +ignore_missing_imports = True + +[mypy-debugpy.server] +ignore_missing_imports = True