From d9e4781d11efb154379c127069dd7187c30f8fa6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 6 Apr 2018 17:22:45 +0100 Subject: [PATCH] ansible: rename helpers.py to target.py, to reflect its purpose --- ansible_mitogen/connection.py | 20 +- ansible_mitogen/helpers.py | 306 ------------------ ansible_mitogen/mixins.py | 12 +- ansible_mitogen/planner.py | 8 +- .../plugins/actions/mitogen_async_status.py | 4 +- ansible_mitogen/runner.py | 12 +- ansible_mitogen/strategy.py | 4 +- 7 files changed, 30 insertions(+), 336 deletions(-) delete mode 100644 ansible_mitogen/helpers.py diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index db182930..65664b31 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -39,7 +39,7 @@ import ansible.plugins.connection import mitogen.unix import mitogen.utils -import ansible_mitogen.helpers +import ansible_mitogen.target import ansible_mitogen.process from ansible_mitogen.services import ContextService @@ -306,7 +306,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None): """ Implement exec_command() by calling the corresponding - ansible_mitogen.helpers function in the target. + ansible_mitogen.target function in the target. :param str cmd: Shell command to execute. @@ -317,7 +317,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): """ emulate_tty = (not in_data and sudoable) rc, stdout, stderr = self.call( - ansible_mitogen.helpers.exec_command, + ansible_mitogen.target.exec_command, cmd=mitogen.utils.cast(cmd), in_data=mitogen.utils.cast(in_data), chdir=mitogen_chdir, @@ -333,39 +333,39 @@ class Connection(ansible.plugins.connection.ConnectionBase): def fetch_file(self, in_path, out_path): """ Implement fetch_file() by calling the corresponding - ansible_mitogen.helpers function in the target. + ansible_mitogen.target function in the target. :param str in_path: Remote filesystem path to read. :param str out_path: Local filesystem path to write. """ - output = self.call(ansible_mitogen.helpers.read_path, + output = self.call(ansible_mitogen.target.read_path, mitogen.utils.cast(in_path)) - ansible_mitogen.helpers.write_path(out_path, output) + ansible_mitogen.target.write_path(out_path, output) def put_data(self, out_path, data): """ Implement put_file() by caling the corresponding - ansible_mitogen.helpers function in the target. + ansible_mitogen.target function in the target. :param str in_path: Local filesystem path to read. :param str out_path: Remote filesystem path to write. """ - self.call(ansible_mitogen.helpers.write_path, + self.call(ansible_mitogen.target.write_path, mitogen.utils.cast(out_path), mitogen.utils.cast(data)) def put_file(self, in_path, out_path): """ Implement put_file() by caling the corresponding - ansible_mitogen.helpers function in the target. + ansible_mitogen.target function in the target. :param str in_path: Local filesystem path to read. :param str out_path: Remote filesystem path to write. """ - self.put_data(out_path, ansible_mitogen.helpers.read_path(in_path)) + self.put_data(out_path, ansible_mitogen.target.read_path(in_path)) diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py deleted file mode 100644 index 167afbb5..00000000 --- a/ansible_mitogen/helpers.py +++ /dev/null @@ -1,306 +0,0 @@ -# Copyright 2017, David Wilson -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -from __future__ import absolute_import -import json -import logging -import operator -import os -import pwd -import random -import re -import stat -import subprocess -import tempfile -import threading -import zlib - -import mitogen.core -import mitogen.service -import ansible_mitogen.runner -import ansible_mitogen.services - - -LOG = logging.getLogger(__name__) - -#: Caching of fetched file data. -_file_cache = {} - -#: Mapping of job_id<->result dict -_result_by_job_id = {} - -#: Mapping of job_id<->threading.Thread -_thread_by_job_id = {} - - -def get_file(context, path): - """ - Basic in-memory caching module fetcher. This generates an one roundtrip for - every previously unseen module, so it is only temporary. - - :param context: - Context we should direct FileService requests to. For now (and probably - forever) this is just the top-level Mitogen connection manager process. - :param path: - Path to fetch from FileService, must previously have been registered by - a privileged context using the `register` command. - :returns: - Bytestring file data. - """ - if path not in _file_cache: - _file_cache[path] = zlib.decompress( - mitogen.service.call( - context, - ansible_mitogen.services.FileService.handle, - ('fetch', path) - ) - ) - - return _file_cache[path] - - -def run_module(kwargs): - """ - Set up the process environment in preparation for running an Ansible - module. This monkey-patches the Ansible libraries in various places to - prevent it from trying to kill the process on completion, and to prevent it - from reading sys.stdin. - """ - runner_name = kwargs.pop('runner_name') - klass = getattr(ansible_mitogen.runner, runner_name) - impl = klass(**kwargs) - return impl.run() - - -def _async_main(job_id, runner_name, kwargs): - """ - Implementation for the thread that implements asynchronous module - execution. - """ - try: - rc = run_module(runner_name, kwargs) - except Exception, e: - rc = mitogen.core.CallError(e) - - _result_by_job_id[job_id] = rc - - -def make_temp_directory(base_dir): - """ - Handle creation of `base_dir` if it is absent, in addition to a unique - temporary directory within `base_dir`. - - :returns: - Newly created temporary directory. - """ - if not os.path.exists(base_dir): - os.makedirs(base_dir, mode=int('0700', 8)) - return tempfile.mkdtemp( - dir=base_dir, - prefix='ansible-mitogen-tmp-', - ) - -def run_module_async(runner_name, kwargs): - """ - Arrange for an Ansible module to be executed in a thread of the current - process, with results available via :py:func:`get_async_result`. - """ - job_id = '%08x' % random.randint(0, 2**32-1) - _result_by_job_id[job_id] = None - _thread_by_job_id[job_id] = threading.Thread( - target=_async_main, - kwargs={ - 'job_id': job_id, - 'runner_name': runner_name, - 'kwargs': kwargs, - } - ) - _thread_by_job_id[job_id].start() - return json.dumps({ - 'ansible_job_id': job_id, - 'changed': True - }) - - -def get_async_result(job_id): - """ - Poll for the result of an asynchronous task. - - :param str job_id: - Job ID to poll for. - :returns: - ``None`` if job is still running, JSON-encoded result dictionary if - execution completed normally, or :py:class:`mitogen.core.CallError` if - an exception was thrown. - """ - if not _thread_by_job_id[job_id].isAlive(): - return _result_by_job_id[job_id] - - -def get_user_shell(): - """ - For commands executed directly via an SSH command-line, SSH looks up the - user's shell via getpwuid() and only defaults to /bin/sh if that field is - missing or empty. - """ - try: - pw_shell = pwd.getpwuid(os.geteuid()).pw_shell - except KeyError: - pw_shell = None - - return pw_shell or '/bin/sh' - - -def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False): - """ - Run a command in a subprocess, emulating the argument handling behaviour of - SSH. - - :param list[str]: - Argument vector. - :param bytes in_data: - Optional standard input for the command. - :param bool emulate_tty: - If :data:`True`, arrange for stdout and stderr to be merged into the - stdout pipe and for LF to be translated into CRLF, emulating the - behaviour of a TTY. - :return: - (return code, stdout bytes, stderr bytes) - """ - LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir) - assert isinstance(args, list) - - if emulate_tty: - stderr = subprocess.STDOUT - else: - stderr = subprocess.PIPE - - proc = subprocess.Popen( - args=args, - stdout=subprocess.PIPE, - stderr=stderr, - stdin=subprocess.PIPE, - cwd=chdir, - ) - stdout, stderr = proc.communicate(in_data) - - if emulate_tty: - stdout = stdout.replace('\n', '\r\n') - return proc.returncode, stdout, stderr or '' - - -def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False): - """ - Run a command in a subprocess, emulating the argument handling behaviour of - SSH. - - :param bytes cmd: - String command line, passed to user's shell. - :param bytes in_data: - Optional standard input for the command. - :return: - (return code, stdout bytes, stderr bytes) - """ - assert isinstance(cmd, basestring) - return exec_args( - args=[get_user_shell(), '-c', cmd], - in_data=in_data, - chdir=chdir, - shell=shell, - emulate_tty=emulate_tty, - ) - - -def read_path(path): - """ - Fetch the contents of a filesystem `path` as bytes. - """ - return open(path, 'rb').read() - - -def write_path(path, s): - """ - Writes bytes `s` to a filesystem `path`. - """ - open(path, 'wb').write(s) - - -CHMOD_CLAUSE_PAT = re.compile(r'([uoga]*)([+\-=])([ugo]|[rwx]*)') -CHMOD_MASKS = { - 'u': stat.S_IRWXU, - 'g': stat.S_IRWXG, - 'o': stat.S_IRWXO, - 'a': (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO), -} -CHMOD_BITS = { - 'u': {'r': stat.S_IRUSR, 'w': stat.S_IWUSR, 'x': stat.S_IXUSR}, - 'g': {'r': stat.S_IRGRP, 'w': stat.S_IWGRP, 'x': stat.S_IXGRP}, - 'o': {'r': stat.S_IROTH, 'w': stat.S_IWOTH, 'x': stat.S_IXOTH}, - 'a': { - 'r': (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH), - 'w': (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH), - 'x': (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - } -} - - -def apply_mode_spec(spec, mode): - """ - Given a symbolic file mode change specification in the style of chmod(1) - `spec`, apply changes in the specification to the numeric file mode `mode`. - """ - for clause in spec.split(','): - match = CHMOD_CLAUSE_PAT.match(clause) - who, op, perms = match.groups() - for ch in who or 'a': - mask = CHMOD_MASKS[ch] - bits = CHMOD_BITS[ch] - cur_perm_bits = mode & mask - new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0) - mode &= ~mask - if op == '=': - mode |= new_perm_bits - elif op == '+': - mode |= new_perm_bits | cur_perm_bits - else: - mode |= cur_perm_bits & ~new_perm_bits - return mode - - -def set_file_mode(path, spec): - """ - Update the permissions of a file using the same syntax as chmod(1). - """ - mode = os.stat(path).st_mode - - if spec.isdigit(): - new_mode = int(spec, 8) - else: - new_mode = apply_mode_spec(spec, mode) - - os.chmod(path, new_mode) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 0efa95b4..2d025445 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -54,7 +54,7 @@ from mitogen.utils import cast import ansible_mitogen.connection import ansible_mitogen.planner -import ansible_mitogen.helpers +import ansible_mitogen.target from ansible.module_utils._text import to_text @@ -117,7 +117,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ Arrange for a Python function to be called in the target context, which should be some function from the standard library or - ansible_mitogen.helpers module. This junction point exists mainly as a + ansible_mitogen.target module. This junction point exists mainly as a nice place to insert print statements during debugging. """ return self._connection.call(func, *args, **kwargs) @@ -200,7 +200,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): # The copy action plugin violates layering and grabs this attribute # directly. self._connection._shell.tmpdir = self.call( - ansible_mitogen.helpers.make_temp_directory, + ansible_mitogen.target.make_temp_directory, base_dir=self._get_remote_tmp(), ) LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) @@ -255,7 +255,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): paths, mode, sudoable) return self.fake_shell(lambda: mitogen.master.Select.all( self._connection.call_async( - ansible_mitogen.helpers.set_file_mode, path, mode + ansible_mitogen.target.set_file_mode, path, mode ) for path in paths )) @@ -299,7 +299,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): delete_remote_tmp=True, wrap_async=False): """ Collect up a module's execution environment then use it to invoke - helpers.run_module() or helpers.run_module_async() in the target + target.run_module() or helpers.run_module_async() in the target context. """ if module_name is None: @@ -358,7 +358,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): chdir=None): """ Override the base implementation by simply calling - helpers.exec_command() in the target context. + target.exec_command() in the target context. """ LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)', cmd, type(in_data), executable, chdir) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 93461584..cd477fb0 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -48,7 +48,7 @@ except ImportError: # Ansible <2.4 import mitogen import mitogen.service -import ansible_mitogen.helpers +import ansible_mitogen.target import ansible_mitogen.services @@ -89,7 +89,7 @@ def parse_script_interpreter(source): class Invocation(object): """ Collect up a module's execution environment then use it to invoke - helpers.run_module() or helpers.run_module_async() in the target context. + target.run_module() or helpers.run_module_async() in the target context. """ def __init__(self, action, connection, module_name, module_args, remote_tmp, task_vars, templar, env, wrap_async): @@ -335,9 +335,9 @@ def invoke(invocation): kwargs = planner.plan(invocation) if invocation.wrap_async: - helper = ansible_mitogen.helpers.run_module_async + helper = ansible_mitogen.target.run_module_async else: - helper = ansible_mitogen.helpers.run_module + helper = ansible_mitogen.target.run_module try: js = invocation.connection.call(helper, kwargs) diff --git a/ansible_mitogen/plugins/actions/mitogen_async_status.py b/ansible_mitogen/plugins/actions/mitogen_async_status.py index ad1f6384..d57a393a 100644 --- a/ansible_mitogen/plugins/actions/mitogen_async_status.py +++ b/ansible_mitogen/plugins/actions/mitogen_async_status.py @@ -28,7 +28,7 @@ import ansible.plugins.action import mitogen.core -import ansible_mitogen.helpers +import ansible_mitogen.target from mitogen.utils import cast @@ -37,7 +37,7 @@ class ActionModule(ansible.plugins.action.ActionBase): job_id = self._task.args['jid'] try: result = self._connection.call( - ansible_mitogen.helpers.get_async_result, + ansible_mitogen.target.get_async_result, cast(job_id), ) except mitogen.core.CallError, e: diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index ae6bd270..43a33489 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -29,7 +29,7 @@ """ These classes implement execution for each style of Ansible module. They are -instantiated in the target context by way of helpers.py::run_module(). +instantiated in the target context by way of target.py::run_module(). Each class in here has a corresponding Planner class in planners.py that knows how to build arguments for it, preseed related data, etc. @@ -45,7 +45,7 @@ import sys import tempfile import types -import ansible_mitogen.helpers # TODO: circular import +import ansible_mitogen.target # TODO: circular import try: from shlex import quote as shlex_quote @@ -97,7 +97,7 @@ class Runner(object): def get_temp_dir(self): if not self._temp_dir: - self._temp_dir = ansible_mitogen.helpers.make_temp_directory( + self._temp_dir = ansible_mitogen.target.make_temp_directory( self.remote_tmp, ) return self._temp_dir @@ -220,7 +220,7 @@ class ProgramRunner(Runner): """ Fetch the module binary from the master if necessary. """ - return ansible_mitogen.helpers.get_file( + return ansible_mitogen.target.get_file( context=self.service_context, path=self.path, ) @@ -241,7 +241,7 @@ class ProgramRunner(Runner): def _run(self): try: - rc, stdout, stderr = ansible_mitogen.helpers.exec_args( + rc, stdout, stderr = ansible_mitogen.target.exec_args( args=self._get_program_args(), emulate_tty=True, ) @@ -362,7 +362,7 @@ class NewStyleRunner(ScriptRunner): return self._code_by_path[self.path] except KeyError: return self._code_by_path.setdefault(self.path, compile( - source=ansible_mitogen.helpers.get_file( + source=ansible_mitogen.target.get_file( context=self.service_context, path=self.path, ), diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 1ad7099a..c70f245b 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -131,12 +131,12 @@ class StrategyMixin(object): For action plug-ins, the original class is looked up as usual, but a new subclass is created dynamically in order to mix-in - ansible_mitogen.helpers.ActionModuleMixin, which overrides many of the + ansible_mitogen.target.ActionModuleMixin, which overrides many of the methods usually inherited from ActionBase in order to replace them with pure-Python equivalents that avoid the use of shell. In particular, _execute_module() is overridden with an implementation - that uses ansible_mitogen.helpers.run_module() executed in the target + that uses ansible_mitogen.target.run_module() executed in the target Context. run_module() implements module execution by importing the module as if it were a normal Python module, and capturing its output in the remote process. Since the Mitogen module loader is active in the