From 865d69dab109e0210ba46dc6878bb4e1a809ed66 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 12 Jun 2025 11:57:36 -0700 Subject: [PATCH] ansible-test - Add remote debugging support (#85317) (cherry picked from commit 7ef13cb29e23fd431b955acb40312791cf3ba7c1) --- .../fragments/ansible-test-debugging.yml | 4 + test/lib/ansible_test/_internal/__init__.py | 5 + .../ansible_test/_internal/ansible_util.py | 2 +- .../_internal/cli/environments.py | 41 ++++ .../_internal/commands/coverage/__init__.py | 2 +- .../commands/integration/__init__.py | 22 +- .../_internal/commands/sanity/__init__.py | 2 + .../_internal/commands/shell/__init__.py | 42 +++- .../_internal/commands/units/__init__.py | 5 +- test/lib/ansible_test/_internal/config.py | 10 +- test/lib/ansible_test/_internal/debugging.py | 166 +++++++++++++++ test/lib/ansible_test/_internal/delegation.py | 25 ++- .../ansible_test/_internal/host_profiles.py | 201 +++++++++++++++++- test/lib/ansible_test/_internal/inventory.py | 4 + test/lib/ansible_test/_internal/metadata.py | 94 +++++++- test/lib/ansible_test/_internal/processes.py | 80 +++++++ .../_internal/python_requirements.py | 27 +++ .../lib/ansible_test/_internal/util_common.py | 16 +- .../_util/target/injector/python.py | 8 + 19 files changed, 726 insertions(+), 30 deletions(-) create mode 100644 changelogs/fragments/ansible-test-debugging.yml create mode 100644 test/lib/ansible_test/_internal/debugging.py create mode 100644 test/lib/ansible_test/_internal/processes.py diff --git a/changelogs/fragments/ansible-test-debugging.yml b/changelogs/fragments/ansible-test-debugging.yml new file mode 100644 index 00000000000..18c65c9b156 --- /dev/null +++ b/changelogs/fragments/ansible-test-debugging.yml @@ -0,0 +1,4 @@ +minor_changes: + - ansible-test - Added experimental support for remote debugging. + - ansible-test - The ``shell`` command has been augmented to propagate remote debug configurations and other test-related settings when running on the controller. + Use the ``--raw`` argument to bypass the additional environment configuration. diff --git a/test/lib/ansible_test/_internal/__init__.py b/test/lib/ansible_test/_internal/__init__.py index 48b33c5524d..0e248936aa0 100644 --- a/test/lib/ansible_test/_internal/__init__.py +++ b/test/lib/ansible_test/_internal/__init__.py @@ -59,6 +59,10 @@ from .config import ( TestConfig, ) +from .debugging import ( + initialize_debugger, +) + def main(cli_args: t.Optional[list[str]] = None) -> None: """Wrapper around the main program function to invoke cleanup functions at exit.""" @@ -77,6 +81,7 @@ def main_internal(cli_args: t.Optional[list[str]] = None) -> None: display.redact = config.redact display.color = config.color display.fd = sys.stderr if config.display_stderr else sys.stdout + initialize_debugger(config) configure_timeout(config) report_locale(isinstance(config, TestConfig) and not config.delegate) diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py index 1b4b76f1384..1c42ccae951 100644 --- a/test/lib/ansible_test/_internal/ansible_util.py +++ b/test/lib/ansible_test/_internal/ansible_util.py @@ -330,6 +330,6 @@ def run_playbook( if args.verbosity: cmd.append('-%s' % ('v' * args.verbosity)) - install_requirements(args, args.controller_python, ansible=True) # run_playbook() + install_requirements(args, None, args.controller_python, ansible=True) # run_playbook() env = ansible_environment(args) intercept_python(args, args.controller_python, cmd, env, capture=capture) diff --git a/test/lib/ansible_test/_internal/cli/environments.py b/test/lib/ansible_test/_internal/cli/environments.py index d8c5d4d29f7..d265479a001 100644 --- a/test/lib/ansible_test/_internal/cli/environments.py +++ b/test/lib/ansible_test/_internal/cli/environments.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import enum import functools +import os import typing as t from ..constants import ( @@ -154,11 +155,13 @@ def add_global_options( global_parser.add_argument( '--metadata', + default=os.environ.get('ANSIBLE_TEST_METADATA_PATH'), help=argparse.SUPPRESS, # for internal use only by ansible-test ) add_global_remote(global_parser, controller_mode) add_global_docker(global_parser, controller_mode) + add_global_debug(global_parser) def add_composite_environment_options( @@ -445,6 +448,44 @@ def add_global_docker( ) +def add_global_debug( + parser: argparse.ArgumentParser, +) -> None: + """Add global debug options.""" + # These `--dev-*` options are experimental features that may change or be removed without regard for backward compatibility. + # Additionally, they're features that are not likely to be used by most users. + # To avoid confusion, they're hidden from `--help` and tab completion by default, except for ansible-core-ci users. + suppress = None if get_ci_provider().supports_core_ci_auth() else argparse.SUPPRESS + + parser.add_argument( + '--dev-debug-on-demand', + action='store_true', + default=False, + help=suppress or 'enable remote debugging only under a debugger', + ) + + parser.add_argument( + '--dev-debug-cli', + action='store_true', + default=False, + help=suppress or 'enable remote debugging for the Ansible CLI', + ) + + parser.add_argument( + '--dev-debug-ansiballz', + action='store_true', + default=False, + help=suppress or 'enable remote debugging for AnsiballZ modules', + ) + + parser.add_argument( + '--dev-debug-self', + action='store_true', + default=False, + help=suppress or 'enable remote debugging for ansible-test', + ) + + def add_environment_docker( exclusive_parser: argparse.ArgumentParser, environments_parser: argparse.ArgumentParser, diff --git a/test/lib/ansible_test/_internal/commands/coverage/__init__.py b/test/lib/ansible_test/_internal/commands/coverage/__init__.py index 3ae1cfab374..0583679fb75 100644 --- a/test/lib/ansible_test/_internal/commands/coverage/__init__.py +++ b/test/lib/ansible_test/_internal/commands/coverage/__init__.py @@ -77,7 +77,7 @@ class CoverageConfig(EnvironmentConfig): def initialize_coverage(args: CoverageConfig, host_state: HostState) -> coverage_module: """Delegate execution if requested, install requirements, then import and return the coverage module. Raises an exception if coverage is not available.""" configure_pypi_proxy(args, host_state.controller_profile) # coverage - install_requirements(args, host_state.controller_profile.python, coverage=True) # coverage + install_requirements(args, host_state.controller_profile, host_state.controller_profile.python, coverage=True) # coverage try: import coverage diff --git a/test/lib/ansible_test/_internal/commands/integration/__init__.py b/test/lib/ansible_test/_internal/commands/integration/__init__.py index 43e6548ff76..499870d14f6 100644 --- a/test/lib/ansible_test/_internal/commands/integration/__init__.py +++ b/test/lib/ansible_test/_internal/commands/integration/__init__.py @@ -105,6 +105,7 @@ from ...host_profiles import ( HostProfile, PosixProfile, SshTargetHostProfile, + DebuggableProfile, ) from ...provisioning import ( @@ -459,10 +460,10 @@ def command_integration_filtered( if isinstance(target_profile, ControllerProfile): if host_state.controller_profile.python.path != target_profile.python.path: - install_requirements(args, target_python, command=True, controller=False) # integration + install_requirements(args, target_profile, target_python, command=True, controller=False) # integration elif isinstance(target_profile, SshTargetHostProfile): connection = target_profile.get_controller_target_connections()[0] - install_requirements(args, target_python, command=True, controller=False, connection=connection) # integration + install_requirements(args, target_profile, target_python, command=True, controller=False, connection=connection) # integration coverage_manager = CoverageManager(args, host_state, inventory_path) coverage_manager.setup() @@ -616,7 +617,7 @@ def command_integration_script( if args.verbosity: cmd.append('-' + ('v' * args.verbosity)) - env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env) + env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env, host_state) cwd = os.path.join(test_env.targets_dir, target.relative_path) env.update( @@ -737,7 +738,7 @@ def command_integration_role( if args.verbosity: cmd.append('-' + ('v' * args.verbosity)) - env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env) + env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config, test_env, host_state) cwd = test_env.integration_dir env.update( @@ -793,6 +794,7 @@ def integration_environment( ansible_config: t.Optional[str], env_config: t.Optional[CloudEnvironmentConfig], test_env: IntegrationEnvironment, + host_state: HostState, ) -> dict[str, str]: """Return a dictionary of environment variables to use when running the given integration test target.""" env = ansible_environment(args, ansible_config=ansible_config) @@ -813,6 +815,9 @@ def integration_environment( if args.debug_strategy: env.update(ANSIBLE_STRATEGY='debug') + if isinstance(host_state.controller_profile, DebuggableProfile): + env.update(host_state.controller_profile.get_ansible_cli_environment_variables()) + if 'non_local/' in target.aliases: if args.coverage: display.warning('Skipping coverage reporting on Ansible modules for non-local test: %s' % target.name) @@ -974,6 +979,13 @@ def requirements(host_profile: HostProfile) -> None: """Install requirements after bootstrapping and delegation.""" if isinstance(host_profile, ControllerHostProfile) and host_profile.controller: configure_pypi_proxy(host_profile.args, host_profile) # integration, windows-integration, network-integration - install_requirements(host_profile.args, host_profile.python, ansible=True, command=True) # integration, windows-integration, network-integration + + install_requirements( # integration, windows-integration, network-integration + args=host_profile.args, + host_profile=host_profile, + python=host_profile.python, + ansible=True, + command=True, + ) elif isinstance(host_profile, PosixProfile) and not isinstance(host_profile, ControllerProfile): configure_pypi_proxy(host_profile.args, host_profile) # integration diff --git a/test/lib/ansible_test/_internal/commands/sanity/__init__.py b/test/lib/ansible_test/_internal/commands/sanity/__init__.py index de886f47751..579ad2ca358 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/__init__.py +++ b/test/lib/ansible_test/_internal/commands/sanity/__init__.py @@ -76,6 +76,7 @@ from ...python_requirements import ( PipInstall, collect_requirements, run_pip, + install_requirements, ) from ...config import ( @@ -178,6 +179,7 @@ def command_sanity(args: SanityConfig) -> None: if args.delegate: raise Delegate(host_state=host_state, require=changes, exclude=args.exclude) + install_requirements(args, host_state.controller_profile, host_state.controller_profile.python) # sanity configure_pypi_proxy(args, host_state.controller_profile) # sanity if disabled: diff --git a/test/lib/ansible_test/_internal/commands/shell/__init__.py b/test/lib/ansible_test/_internal/commands/shell/__init__.py index 0c3cf15aef3..550d4d13efc 100644 --- a/test/lib/ansible_test/_internal/commands/shell/__init__.py +++ b/test/lib/ansible_test/_internal/commands/shell/__init__.py @@ -14,6 +14,10 @@ from ...util import ( HostConnectionError, ) +from ...ansible_util import ( + ansible_environment, +) + from ...config import ( ShellConfig, ) @@ -32,6 +36,7 @@ from ...host_profiles import ( ControllerProfile, PosixProfile, SshTargetHostProfile, + DebuggableProfile, ) from ...provisioning import ( @@ -39,7 +44,6 @@ from ...provisioning import ( ) from ...host_configs import ( - ControllerConfig, OriginConfig, ) @@ -48,12 +52,21 @@ from ...inventory import ( create_posix_inventory, ) +from ...python_requirements import ( + install_requirements, +) + +from ...util_common import ( + get_injector_env, +) + +from ...delegation import ( + metadata_context, +) + def command_shell(args: ShellConfig) -> None: """Entry point for the `shell` command.""" - if args.raw and isinstance(args.targets[0], ControllerConfig): - raise ApplicationError('The --raw option has no effect on the controller.') - if not args.export and not args.cmd and not sys.stdin.isatty(): raise ApplicationError('Standard input must be a TTY to launch a shell.') @@ -62,6 +75,8 @@ def command_shell(args: ShellConfig) -> None: if args.delegate: raise Delegate(host_state=host_state) + install_requirements(args, host_state.controller_profile, host_state.controller_profile.python) # shell + if args.raw and not isinstance(args.controller, OriginConfig): display.warning('The --raw option will only be applied to the target.') @@ -92,6 +107,16 @@ def command_shell(args: ShellConfig) -> None: con.run(args.cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL) return + with metadata_context(args): + interactive_shell(args, target_profile, con) + + +def interactive_shell( + args: ShellConfig, + target_profile: SshTargetHostProfile, + con: Connection, +) -> None: + """Run an interactive shell.""" if isinstance(con, SshConnection) and args.raw: cmd: list[str] = [] elif isinstance(target_profile, PosixProfile): @@ -111,6 +136,15 @@ def command_shell(args: ShellConfig) -> None: env = {name: os.environ[name] for name in optional_vars if name in os.environ} + if isinstance(con, LocalConnection): # configure the controller environment + env.update(ansible_environment(args)) + env.update(get_injector_env(target_profile.python, env)) + env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path)) + + if isinstance(target_profile, DebuggableProfile): + env.update(target_profile.get_ansiballz_environment_variables()) + env.update(target_profile.get_ansible_cli_environment_variables()) + if env: cmd = ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()] diff --git a/test/lib/ansible_test/_internal/commands/units/__init__.py b/test/lib/ansible_test/_internal/commands/units/__init__.py index 126e670e869..1e2f84b38be 100644 --- a/test/lib/ansible_test/_internal/commands/units/__init__.py +++ b/test/lib/ansible_test/_internal/commands/units/__init__.py @@ -64,6 +64,7 @@ from ...executor import ( from ...python_requirements import ( install_requirements, + post_install, ) from ...content_config import ( @@ -230,7 +231,9 @@ def command_units(args: UnitsConfig) -> None: controller = any(test_context == TestContext.controller for test_context, python, paths, env in final_candidates) if args.requirements_mode != 'skip': - install_requirements(args, target_profile.python, ansible=controller, command=True, controller=False) # units + install_requirements(args, target_profile, target_profile.python, ansible=controller, command=True, controller=False) # units + else: + post_install(target_profile) test_sets.extend(final_candidates) diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index 1c3098d8a40..d559871a243 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -20,6 +20,7 @@ from .util_common import ( from .metadata import ( Metadata, + DebuggerFlags, ) from .data import ( @@ -118,7 +119,14 @@ class EnvironmentConfig(CommonConfig): self.dev_systemd_debug: bool = args.dev_systemd_debug self.dev_probe_cgroups: t.Optional[str] = args.dev_probe_cgroups - self.metadata = Metadata.from_file(args.metadata) if args.metadata else Metadata() + debugger_flags = DebuggerFlags( + on_demand=args.dev_debug_on_demand, + cli=args.dev_debug_cli, + ansiballz=args.dev_debug_ansiballz, + self=args.dev_debug_self, + ) + + self.metadata = Metadata.from_file(args.metadata) if args.metadata else Metadata(debugger_flags=debugger_flags) self.metadata_path: t.Optional[str] = None def metadata_callback(payload_config: PayloadConfig) -> None: diff --git a/test/lib/ansible_test/_internal/debugging.py b/test/lib/ansible_test/_internal/debugging.py new file mode 100644 index 00000000000..bd5fc452ad9 --- /dev/null +++ b/test/lib/ansible_test/_internal/debugging.py @@ -0,0 +1,166 @@ +"""Setup and configure remote debugging.""" + +from __future__ import annotations + +import dataclasses +import json +import os +import re + +from .util import ( + cache, + display, + raw_command, + ApplicationError, +) + +from .processes import ( + Process, + get_current_process, +) + +from .config import ( + EnvironmentConfig, +) + +from .metadata import ( + DebuggerSettings, + DebuggerFlags, +) + +from . import ( + data_context, + CommonConfig, +) + + +def initialize_debugger(args: CommonConfig) -> None: + """Initialize the debugger settings before delegation.""" + if not isinstance(args, EnvironmentConfig): + return + + if args.metadata.loaded: + return # after delegation + + if collection := data_context().content.collection: + args.metadata.collection_root = collection.root + + load_debugger_settings(args) + + +def parse_debugger_settings(value: str) -> DebuggerSettings: + """Parse remote debugger settings and apply defaults.""" + try: + settings = DebuggerSettings(**json.loads(value)) + except Exception as ex: + raise ApplicationError(f"Invalid debugger settings: {ex}") from ex + + if not settings.module: + if not settings.package or 'pydevd-pycharm' in settings.package: + module = 'pydevd_pycharm' + else: + module = 'pydevd' + + settings = dataclasses.replace(settings, module=module) + + if settings.package is None: + if settings.module == 'pydevd_pycharm': + if pycharm_version := detect_pycharm_version(): + package = f'pydevd-pycharm~={pycharm_version}' + else: + package = None + else: + package = 'pydevd' + + settings = dataclasses.replace(settings, package=package) + + settings.settrace.setdefault('suspend', False) + + if port := detect_pydevd_port(): + settings = dataclasses.replace(settings, port=port) + + if detect_pycharm_process(): + # This only works with the default PyCharm debugger. + # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers. + # Further investigation is required to understand the cause. + settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess']) + + return settings + + +def load_debugger_settings(args: EnvironmentConfig) -> None: + """Load the remote debugger settings.""" + if args.metadata.debugger_flags.on_demand: + # On-demand debugging only enables debugging if we're running under a debugger, otherwise it's a no-op. + + if not detect_pydevd_port(): + display.info('Debugging disabled because no debugger was detected.', verbosity=1) + args.metadata.debugger_flags = DebuggerFlags.all(False) + return + + display.info('Enabling on-demand debugging.', verbosity=1) + + if not args.metadata.debugger_flags.enable: + # Assume the user wants all debugging features enabled, since on-demand debugging with no features is pointless. + args.metadata.debugger_flags = DebuggerFlags.all(True) + + if not args.metadata.debugger_flags.enable: + return + + value = os.environ.get('ANSIBLE_TEST_REMOTE_DEBUGGER') or '{}' + settings = parse_debugger_settings(value) + + display.info(f'>>> Debugger Settings\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3) + + args.metadata.debugger_settings = settings + + +@cache +def detect_pydevd_port() -> int | None: + """Return the port for the pydevd instance hosting this process, or `None` if not detected.""" + current_process = get_current_process_cached() + args = current_process.args + + if any('/pydevd.py' in arg for arg in args) and (port_idx := args.index('--port')): + port = int(args[port_idx + 1]) + display.info(f'Detected pydevd debugger port {port}.', verbosity=1) + return port + + return None + + +@cache +def detect_pycharm_version() -> str | None: + """Return the version of PyCharm running ansible-test, or `None` if PyCharm was not detected. The result is cached.""" + if pycharm := detect_pycharm_process(): + output = raw_command([pycharm.args[0], '--version'], capture=True)[0] + + if match := re.search('^Build #PY-(?P[0-9.]+)$', output, flags=re.MULTILINE): + version = match.group('version') + display.info(f'Detected PyCharm version {version}.', verbosity=1) + return version + + display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version could not be detected.') + + return None + + +@cache +def detect_pycharm_process() -> Process | None: + """Return the PyCharm process running ansible-test, or `None` if PyCharm was not detected. The result is cached.""" + current_process = get_current_process_cached() + parent = current_process.parent + + while parent: + if parent.path.name == 'pycharm': + return parent + + parent = parent.parent + + return None + + +@cache +def get_current_process_cached() -> Process: + """Return the current process. The result is cached.""" + return get_current_process() diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py index abe6ad789b6..74c4adbfc6a 100644 --- a/test/lib/ansible_test/_internal/delegation.py +++ b/test/lib/ansible_test/_internal/delegation.py @@ -117,14 +117,21 @@ def delegate(args: CommonConfig, host_state: HostState, exclude: list[str], requ make_dirs(ResultType.TMP.path) - with tempfile.NamedTemporaryFile(prefix='metadata-', suffix='.json', dir=ResultType.TMP.path) as metadata_fd: - args.metadata_path = os.path.join(ResultType.TMP.relative_path, os.path.basename(metadata_fd.name)) - args.metadata.to_file(args.metadata_path) + with metadata_context(args): + delegate_command(args, host_state, exclude, require) - try: - delegate_command(args, host_state, exclude, require) - finally: - args.metadata_path = None + +@contextlib.contextmanager +def metadata_context(args: EnvironmentConfig) -> t.Generator[None]: + """A context manager which exports delegation metadata.""" + with tempfile.NamedTemporaryFile(prefix='metadata-', suffix='.json', dir=ResultType.TMP.path) as metadata_fd: + args.metadata_path = os.path.join(ResultType.TMP.relative_path, os.path.basename(metadata_fd.name)) + args.metadata.to_file(args.metadata_path) + + try: + yield + finally: + args.metadata_path = None def delegate_command(args: EnvironmentConfig, host_state: HostState, exclude: list[str], require: list[str]) -> None: @@ -186,6 +193,10 @@ def delegate_command(args: EnvironmentConfig, host_state: HostState, exclude: li networks = container.get_network_names() if networks is not None: + if args.metadata.debugger_flags.enable: + networks = [] + display.warning('Skipping network isolation to enable remote debugging.') + for network in networks: try: con.disconnect_network(network) diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index 5a7fe755d8b..4e5856688ce 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -4,7 +4,10 @@ from __future__ import annotations import abc import dataclasses +import importlib +import json import os +import pathlib import shlex import tempfile import time @@ -58,6 +61,9 @@ from .util import ( HostConnectionError, ANSIBLE_TEST_TARGET_ROOT, WINDOWS_CONNECTION_VARIABLES, + ANSIBLE_SOURCE_ROOT, + ANSIBLE_LIB_ROOT, + ANSIBLE_TEST_ROOT, ) from .util_common import ( @@ -92,6 +98,8 @@ from .venv import ( from .ssh import ( SshConnectionDetail, + create_ssh_port_forwards, + SshProcess, ) from .ansible_util import ( @@ -284,6 +292,176 @@ class HostProfile(t.Generic[THostConfig], metaclass=abc.ABCMeta): return f'{self.__class__.__name__}: {self.name}' +class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): + """Base class for profiles remote debugging.""" + + __PYDEVD_PORT_KEY = 'pydevd_port' + __DEBUGGING_FORWARDER_KEY = 'debugging_forwarder' + + @property + def debugging_enabled(self) -> bool: + """Returns `True` if debugging is enabled for this profile, otherwise `False`.""" + if self.controller: + return self.args.metadata.debugger_flags.enable + + return self.args.metadata.debugger_flags.ansiballz + + @property + def pydevd_port(self) -> int: + """The pydevd port to use.""" + return self.state.get(self.__PYDEVD_PORT_KEY) or self.origin_pydev_port + + @property + def debugging_forwarder(self) -> SshProcess | None: + """The SSH forwarding process, if enabled.""" + return self.cache.get(self.__DEBUGGING_FORWARDER_KEY) + + @debugging_forwarder.setter + def debugging_forwarder(self, value: SshProcess) -> None: + """The SSH forwarding process, if enabled.""" + self.cache[self.__DEBUGGING_FORWARDER_KEY] = value + + @property + def origin_pydev_port(self) -> int: + """The pydevd port on the origin.""" + return self.args.metadata.debugger_settings.port + + def enable_debugger_forwarding(self, ssh: SshConnectionDetail) -> None: + """Enable pydevd port forwarding from the origin.""" + if not self.debugging_enabled: + return + + endpoint = ('localhost', self.origin_pydev_port) + forwards = [endpoint] + + self.debugging_forwarder = create_ssh_port_forwards(self.args, ssh, forwards) + + port_forwards = self.debugging_forwarder.collect_port_forwards() + + self.state[self.__PYDEVD_PORT_KEY] = port = port_forwards[endpoint] + + display.info(f'Remote debugging of {self.name!r} is available on port {port}.', verbosity=1) + + def deprovision(self) -> None: + """Deprovision the host after delegation has completed.""" + super().deprovision() + + if not self.debugging_forwarder: + return # forwarding not in use + + self.debugging_forwarder.terminate() + + display.info(f'Waiting for the {self.name!r} remote debugging SSH port forwarding process to terminate.', verbosity=1) + + self.debugging_forwarder.wait() + + def get_pydevd_settrace_arguments(self) -> dict[str, object]: + """Get settrace arguments for pydevd.""" + return self.args.metadata.debugger_settings.settrace | dict( + host="localhost", + port=self.pydevd_port, + ) + + def get_pydevd_environment_variables(self) -> dict[str, str]: + """Get environment variables needed to configure pydevd for debugging.""" + return dict( + PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(self.get_source_mapping().items())), + ) + + def get_source_mapping(self) -> dict[str, str]: + """Get the source mapping from the given metadata.""" + from . import data_context + + if collection := data_context().content.collection: + source_mapping = { + f"{self.args.metadata.ansible_test_root}/": f'{ANSIBLE_TEST_ROOT}/', + f"{self.args.metadata.ansible_lib_root}/": f'{ANSIBLE_LIB_ROOT}/', + f'{self.args.metadata.collection_root}/ansible_collections/': f'{collection.root}/ansible_collections/', + } + else: + ansible_source_root = pathlib.Path(self.args.metadata.ansible_lib_root).parent.parent + + source_mapping = { + f"{ansible_source_root}/": f'{ANSIBLE_SOURCE_ROOT}/', + } + + source_mapping = {key: value for key, value in source_mapping.items() if key != value} + + return source_mapping + + def activate_debugger(self) -> None: + """Activate the debugger after delegation.""" + if not self.args.metadata.loaded or not self.args.metadata.debugger_flags.self: + return + + display.info('Activating remote debugging of ansible-test.', verbosity=1) + + os.environ.update(self.get_pydevd_environment_variables()) + + debugging_module = importlib.import_module(self.args.metadata.debugger_settings.module) + debugging_module.settrace(**self.get_pydevd_settrace_arguments()) + + pass # pylint: disable=unnecessary-pass # when suspend is True, execution pauses here -- it's also a convenient place to put a breakpoint + + def get_ansiballz_inventory_variables(self) -> dict[str, t.Any]: + """ + Return inventory variables for remote debugging of AnsiballZ modules. + When delegating, this function must be called after delegation. + """ + if not self.args.metadata.debugger_flags.ansiballz: + return {} + + return dict( + _ansible_ansiballz_debugger_config=json.dumps(self.get_ansiballz_debugger_config()), + ) + + def get_ansiballz_environment_variables(self) -> dict[str, t.Any]: + """ + Return environment variables for remote debugging of AnsiballZ modules. + When delegating, this function must be called after delegation. + """ + if not self.args.metadata.debugger_flags.ansiballz: + return {} + + return dict( + _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG=json.dumps(self.get_ansiballz_debugger_config()), + ) + + def get_ansiballz_debugger_config(self) -> dict[str, t.Any]: + """ + Return config for remote debugging of AnsiballZ modules. + When delegating, this function must be called after delegation. + """ + debugger_config = dict( + module=self.args.metadata.debugger_settings.module, + settrace=self.get_pydevd_settrace_arguments(), + source_mapping=self.get_source_mapping(), + ) + + display.info(f'>>> Debugger Config ({self.name} AnsiballZ)\n{json.dumps(debugger_config, indent=4)}', verbosity=3) + + return debugger_config + + def get_ansible_cli_environment_variables(self) -> dict[str, t.Any]: + """ + Return environment variables for remote debugging of the Ansible CLI. + When delegating, this function must be called after delegation. + """ + if not self.args.metadata.debugger_flags.cli: + return {} + + debugger_config = dict( + args=['-m', 'pydevd', '--client', 'localhost', '--port', str(self.pydevd_port)] + self.args.metadata.debugger_settings.args + ['--file'], + env=self.get_pydevd_environment_variables(), + ) + + display.info(f'>>> Debugger Config ({self.name} Ansible CLI)\n{json.dumps(debugger_config, indent=4)}', verbosity=3) + + return dict( + ANSIBLE_TEST_DEBUGGER_CONFIG=json.dumps(debugger_config), + ) + + class PosixProfile(HostProfile[TPosixConfig], metaclass=abc.ABCMeta): """Base class for POSIX host profiles.""" @@ -306,7 +484,7 @@ class PosixProfile(HostProfile[TPosixConfig], metaclass=abc.ABCMeta): return python -class ControllerHostProfile(PosixProfile[TControllerHostConfig], metaclass=abc.ABCMeta): +class ControllerHostProfile(PosixProfile[TControllerHostConfig], DebuggableProfile[TControllerHostConfig], metaclass=abc.ABCMeta): """Base class for profiles usable as a controller.""" @abc.abstractmethod @@ -410,7 +588,7 @@ class RemoteProfile(SshTargetHostProfile[TRemoteConfig], metaclass=abc.ABCMeta): ) -class ControllerProfile(SshTargetHostProfile[ControllerConfig], PosixProfile[ControllerConfig]): +class ControllerProfile(SshTargetHostProfile[ControllerConfig], PosixProfile[ControllerConfig], DebuggableProfile[ControllerConfig]): """Host profile for the controller as a target.""" @property @@ -418,6 +596,11 @@ class ControllerProfile(SshTargetHostProfile[ControllerConfig], PosixProfile[Con """The name of the host profile.""" return self.controller_profile.name + @property + def pydevd_port(self) -> int: + """The pydevd port to use.""" + return self.controller_profile.pydevd_port + def get_controller_target_connections(self) -> list[SshConnection]: """Return SSH connection(s) for accessing the host as a target from the controller.""" settings = SshConnectionDetail( @@ -432,7 +615,7 @@ class ControllerProfile(SshTargetHostProfile[ControllerConfig], PosixProfile[Con return [SshConnection(self.args, settings)] -class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[DockerConfig]): +class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[DockerConfig], DebuggableProfile[DockerConfig]): """Host profile for a docker instance.""" MARKER = 'ansible-test-marker' @@ -487,7 +670,7 @@ class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[Do image=self.config.image, name=f'ansible-test-{self.label}', ports=[22], - publish_ports=not self.controller, # connections to the controller over SSH are not required + publish_ports=self.debugging_enabled or not self.controller, # SSH to the controller is not required unless remote debugging is enabled options=init_config.options, cleanup=False, cmd=self.build_init_command(init_config, init_probe), @@ -1000,6 +1183,9 @@ class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[Do docker_logs(self.args, self.container_name) raise + if self.debugging_enabled: + self.enable_debugger_forwarding(self.get_ssh_connection_detail(HostType.origin)) + def deprovision(self) -> None: """Deprovision the host after delegation has completed.""" super().deprovision() @@ -1248,13 +1434,18 @@ class OriginProfile(ControllerHostProfile[OriginConfig]): return os.getcwd() -class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile[PosixRemoteConfig]): +class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile[PosixRemoteConfig], DebuggableProfile[PosixRemoteConfig]): """Host profile for a POSIX remote instance.""" def wait(self) -> None: """Wait for the instance to be ready. Executed before delegation for the controller and after delegation for targets.""" self.wait_until_ready() + def setup(self) -> None: + """Perform out-of-band setup before delegation.""" + if self.debugging_enabled: + self.enable_debugger_forwarding(self.get_origin_controller_connection().settings) + def configure(self) -> None: """Perform in-band configuration. Executed before delegation for the controller and after delegation for targets.""" # a target uses a single python version, but a controller may include additional versions for targets running on the controller diff --git a/test/lib/ansible_test/_internal/inventory.py b/test/lib/ansible_test/_internal/inventory.py index 0650644db4c..26f8f8bd650 100644 --- a/test/lib/ansible_test/_internal/inventory.py +++ b/test/lib/ansible_test/_internal/inventory.py @@ -30,6 +30,7 @@ from .host_profiles import ( SshTargetHostProfile, WindowsInventoryProfile, WindowsRemoteProfile, + DebuggableProfile, ) from .ssh import ( @@ -59,6 +60,9 @@ def get_common_variables(target_profile: HostProfile, controller: bool = False) # To compensate for this we'll perform a `cd /` before running any commands after `sudo` succeeds. common_variables.update(ansible_sudo_chdir='/') + if isinstance(target_profile, DebuggableProfile): + common_variables.update(target_profile.get_ansiballz_inventory_variables()) + return common_variables diff --git a/test/lib/ansible_test/_internal/metadata.py b/test/lib/ansible_test/_internal/metadata.py index 882a5620e98..5aaceecd31c 100644 --- a/test/lib/ansible_test/_internal/metadata.py +++ b/test/lib/ansible_test/_internal/metadata.py @@ -1,11 +1,15 @@ """Test metadata for passing data to delegated tests.""" from __future__ import annotations + +import dataclasses import typing as t from .util import ( display, generate_name, + ANSIBLE_TEST_ROOT, + ANSIBLE_LIB_ROOT, ) from .io import ( @@ -22,13 +26,19 @@ from .diff import ( class Metadata: """Metadata object for passing data to delegated tests.""" - def __init__(self) -> None: + def __init__(self, debugger_flags: DebuggerFlags) -> None: """Initialize metadata.""" self.changes: dict[str, tuple[tuple[int, int], ...]] = {} self.cloud_config: t.Optional[dict[str, dict[str, t.Union[int, str, bool]]]] = None self.change_description: t.Optional[ChangeDescription] = None self.ci_provider: t.Optional[str] = None self.session_id = generate_name() + self.ansible_lib_root = ANSIBLE_LIB_ROOT + self.ansible_test_root = ANSIBLE_TEST_ROOT + self.collection_root: str | None = None + self.debugger_flags = debugger_flags + self.debugger_settings: DebuggerSettings | None = None + self.loaded = False def populate_changes(self, diff: t.Optional[list[str]]) -> None: """Populate the changeset using the given diff.""" @@ -57,6 +67,11 @@ class Metadata: ci_provider=self.ci_provider, change_description=self.change_description.to_dict() if self.change_description else None, session_id=self.session_id, + ansible_lib_root=self.ansible_lib_root, + ansible_test_root=self.ansible_test_root, + collection_root=self.collection_root, + debugger_flags=dataclasses.asdict(self.debugger_flags), + debugger_settings=dataclasses.asdict(self.debugger_settings) if self.debugger_settings else None, ) def to_file(self, path: str) -> None: @@ -76,12 +91,20 @@ class Metadata: @staticmethod def from_dict(data: dict[str, t.Any]) -> Metadata: """Return metadata loaded from the specified dictionary.""" - metadata = Metadata() + metadata = Metadata( + debugger_flags=DebuggerFlags(**data['debugger_flags']), + ) + metadata.changes = data['changes'] metadata.cloud_config = data['cloud_config'] metadata.ci_provider = data['ci_provider'] metadata.change_description = ChangeDescription.from_dict(data['change_description']) if data['change_description'] else None metadata.session_id = data['session_id'] + metadata.ansible_lib_root = data['ansible_lib_root'] + metadata.ansible_test_root = data['ansible_test_root'] + metadata.collection_root = data['collection_root'] + metadata.debugger_settings = DebuggerSettings(**data['debugger_settings']) if data['debugger_settings'] else None + metadata.loaded = True return metadata @@ -130,3 +153,70 @@ class ChangeDescription: changes.no_integration_paths = data['no_integration_paths'] return changes + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebuggerSettings: + """Settings for remote debugging.""" + + module: str | None = None + """ + The Python module to import. + This should be pydevd or a derivative. + If not provided it will be auto-detected. + """ + + package: str | None = None + """ + The Python package to install for debugging. + If `None` then the package will be auto-detected. + If an empty string, then no package will be installed. + """ + + settrace: dict[str, object] = dataclasses.field(default_factory=dict) + """ + Options to pass to the `{module}.settrace` method. + Used for running AnsiballZ modules only. + The `host` and `port` options will be provided by ansible-test. + The `suspend` option defaults to `False`. + """ + + args: list[str] = dataclasses.field(default_factory=list) + """ + Arguments to pass to `pydevd` on the command line. + Used for running Ansible CLI programs only. + The `--client` and `--port` options will be provided by ansible-test. + """ + + port: int = 5678 + """ + The port on the origin host which is listening for incoming connections from pydevd. + SSH port forwarding will be automatically configured for non-local hosts to connect to this port as needed. + """ + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebuggerFlags: + """Flags for enabling specific debugging features.""" + + self: bool = False + """Debug ansible-test itself.""" + + ansiballz: bool = False + """Debug AnsiballZ modules.""" + + cli: bool = False + """Debug Ansible CLI programs other than ansible-test.""" + + on_demand: bool = False + """Enable debugging features only when ansible-test is running under a debugger.""" + + @property + def enable(self) -> bool: + """Return `True` if any debugger feature other than on-demand is enabled.""" + return any(getattr(self, field.name) for field in dataclasses.fields(self) if field.name != 'on_demand') + + @classmethod + def all(cls, enabled: bool) -> t.Self: + """Return a `DebuggerFlags` instance with all flags enabled or disabled.""" + return cls(**{field.name: enabled for field in dataclasses.fields(cls)}) diff --git a/test/lib/ansible_test/_internal/processes.py b/test/lib/ansible_test/_internal/processes.py new file mode 100644 index 00000000000..21897a12b9a --- /dev/null +++ b/test/lib/ansible_test/_internal/processes.py @@ -0,0 +1,80 @@ +"""Wrappers around `ps` for querying running processes.""" + +from __future__ import annotations + +import collections +import dataclasses +import os +import pathlib +import shlex + +from ansible_test._internal.util import raw_command + + +@dataclasses.dataclass(frozen=True) +class ProcessData: + """Data about a running process.""" + + pid: int + ppid: int + command: str + + +@dataclasses.dataclass(frozen=True) +class Process: + """A process in the process tree.""" + + pid: int + command: str + parent: Process | None = None + children: tuple[Process, ...] = dataclasses.field(default_factory=tuple) + + @property + def args(self) -> list[str]: + """The list of arguments that make up `command`.""" + return shlex.split(self.command) + + @property + def path(self) -> pathlib.Path: + """The path to the process.""" + return pathlib.Path(self.args[0]) + + +def get_process_data(pids: list[int] | None = None) -> list[ProcessData]: + """Return a list of running processes.""" + if pids: + args = ['-p', ','.join(map(str, pids))] + else: + args = ['-A'] + + lines = raw_command(['ps'] + args + ['-o', 'pid,ppid,command'], capture=True)[0].splitlines()[1:] + processes = [ProcessData(pid=int(pid), ppid=int(ppid), command=command) for pid, ppid, command in (line.split(maxsplit=2) for line in lines)] + + return processes + + +def get_process_tree() -> dict[int, Process]: + """Return the process tree.""" + processes = get_process_data() + pid_to_process: dict[int, Process] = {} + pid_to_children: dict[int, list[Process]] = collections.defaultdict(list) + + for data in processes: + pid_to_process[data.pid] = process = Process(pid=data.pid, command=data.command) + + if data.ppid: + pid_to_children[data.ppid].append(process) + + for data in processes: + pid_to_process[data.pid] = dataclasses.replace( + pid_to_process[data.pid], + parent=pid_to_process.get(data.ppid), + children=tuple(pid_to_children[data.pid]), + ) + + return pid_to_process + + +def get_current_process() -> Process: + """Return the current process along with its ancestors and descendants.""" + return get_process_tree()[os.getpid()] diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index 8cf4b5df881..68077542eaf 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -55,6 +55,11 @@ from .coverage_util import ( get_coverage_version, ) +if t.TYPE_CHECKING: + from .host_profiles import ( + HostProfile, + ) + QUIET_PIP_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'quiet_pip.py') REQUIREMENTS_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'requirements.py') @@ -122,6 +127,7 @@ class PipBootstrap(PipCommand): def install_requirements( args: EnvironmentConfig, + host_profile: HostProfile | None, python: PythonConfig, ansible: bool = False, command: bool = False, @@ -133,6 +139,7 @@ def install_requirements( create_result_directories(args) if not requirements_allowed(args, controller): + post_install(host_profile) return if command and isinstance(args, (UnitsConfig, IntegrationConfig)) and args.coverage: @@ -161,7 +168,17 @@ def install_requirements( sanity=None, ) + from .host_profiles import DebuggableProfile + + if isinstance(host_profile, DebuggableProfile) and host_profile.debugging_enabled and args.metadata.debugger_settings.package: + commands.append(PipInstall( + requirements=[], + constraints=[], + packages=[args.metadata.debugger_settings.package], + )) + if not commands: + post_install(host_profile) return run_pip(args, python, commands, connection) @@ -170,6 +187,16 @@ def install_requirements( if any(isinstance(command, PipInstall) and command.has_package('pyyaml') for command in commands): check_pyyaml(python) + post_install(host_profile) + + +def post_install(host_profile: HostProfile) -> None: + """Operations to perform after requirements are installed.""" + from .host_profiles import DebuggableProfile + + if isinstance(host_profile, DebuggableProfile): + host_profile.activate_debugger() + def collect_bootstrap(python: PythonConfig) -> list[PipCommand]: """Return the details necessary to bootstrap pip into an empty virtual environment.""" diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py index 0653976ec9b..d2464b56aa3 100644 --- a/test/lib/ansible_test/_internal/util_common.py +++ b/test/lib/ansible_test/_internal/util_common.py @@ -451,10 +451,20 @@ def intercept_python( """ Run a command while intercepting invocations of Python to control the version used. If the specified Python is an ansible-test managed virtual environment, it will be added to PATH to activate it. - Otherwise a temporary directory will be created to ensure the correct Python can be found in PATH. + Otherwise, a temporary directory will be created to ensure the correct Python can be found in PATH. """ - env = env.copy() cmd = list(cmd) + env = get_injector_env(python, env) + + return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd, always=always) + + +def get_injector_env( + python: PythonConfig, + env: dict[str, str], +) -> dict[str, str]: + """Get the environment variables needed to inject the given Python interpreter into the environment.""" + env = env.copy() inject_path = get_injector_path() # make sure scripts (including injector.py) find the correct Python interpreter @@ -467,7 +477,7 @@ def intercept_python( env['ANSIBLE_TEST_PYTHON_VERSION'] = python.version env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = python.path - return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd, always=always) + return env def run_command( diff --git a/test/lib/ansible_test/_util/target/injector/python.py b/test/lib/ansible_test/_util/target/injector/python.py index 8b9fa86455e..3a328b5e4fb 100644 --- a/test/lib/ansible_test/_util/target/injector/python.py +++ b/test/lib/ansible_test/_util/target/injector/python.py @@ -15,6 +15,7 @@ def main(): args = [sys.executable] ansible_lib_root = os.environ.get('ANSIBLE_TEST_ANSIBLE_LIB_ROOT') + debugger_config = os.environ.get('ANSIBLE_TEST_DEBUGGER_CONFIG') coverage_config = os.environ.get('COVERAGE_CONF') coverage_output = os.environ.get('COVERAGE_FILE') @@ -28,6 +29,13 @@ def main(): sys.exit('ERROR: Could not find `coverage` module. ' 'Did you use a virtualenv created without --system-site-packages or with the wrong interpreter?') + if debugger_config: + import json + + debugger_options = json.loads(debugger_config) + os.environ.update(debugger_options['env']) + args += debugger_options['args'] + if name == 'python.py': if sys.argv[1] == '-c': # prevent simple misuse of python.py with -c which does not work with coverage