ansible: rename helpers.py to target.py, to reflect its purpose

pull/178/head
David Wilson 7 years ago
parent a643f13ebe
commit d9e4781d11

@ -39,7 +39,7 @@ import ansible.plugins.connection
import mitogen.unix import mitogen.unix
import mitogen.utils import mitogen.utils
import ansible_mitogen.helpers import ansible_mitogen.target
import ansible_mitogen.process import ansible_mitogen.process
from ansible_mitogen.services import ContextService from ansible_mitogen.services import ContextService
@ -306,7 +306,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None): def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None):
""" """
Implement exec_command() by calling the corresponding Implement exec_command() by calling the corresponding
ansible_mitogen.helpers function in the target. ansible_mitogen.target function in the target.
:param str cmd: :param str cmd:
Shell command to execute. Shell command to execute.
@ -317,7 +317,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
""" """
emulate_tty = (not in_data and sudoable) emulate_tty = (not in_data and sudoable)
rc, stdout, stderr = self.call( rc, stdout, stderr = self.call(
ansible_mitogen.helpers.exec_command, ansible_mitogen.target.exec_command,
cmd=mitogen.utils.cast(cmd), cmd=mitogen.utils.cast(cmd),
in_data=mitogen.utils.cast(in_data), in_data=mitogen.utils.cast(in_data),
chdir=mitogen_chdir, chdir=mitogen_chdir,
@ -333,39 +333,39 @@ class Connection(ansible.plugins.connection.ConnectionBase):
def fetch_file(self, in_path, out_path): def fetch_file(self, in_path, out_path):
""" """
Implement fetch_file() by calling the corresponding Implement fetch_file() by calling the corresponding
ansible_mitogen.helpers function in the target. ansible_mitogen.target function in the target.
:param str in_path: :param str in_path:
Remote filesystem path to read. Remote filesystem path to read.
:param str out_path: :param str out_path:
Local filesystem path to write. Local filesystem path to write.
""" """
output = self.call(ansible_mitogen.helpers.read_path, output = self.call(ansible_mitogen.target.read_path,
mitogen.utils.cast(in_path)) mitogen.utils.cast(in_path))
ansible_mitogen.helpers.write_path(out_path, output) ansible_mitogen.target.write_path(out_path, output)
def put_data(self, out_path, data): def put_data(self, out_path, data):
""" """
Implement put_file() by caling the corresponding Implement put_file() by caling the corresponding
ansible_mitogen.helpers function in the target. ansible_mitogen.target function in the target.
:param str in_path: :param str in_path:
Local filesystem path to read. Local filesystem path to read.
:param str out_path: :param str out_path:
Remote filesystem path to write. Remote filesystem path to write.
""" """
self.call(ansible_mitogen.helpers.write_path, self.call(ansible_mitogen.target.write_path,
mitogen.utils.cast(out_path), mitogen.utils.cast(out_path),
mitogen.utils.cast(data)) mitogen.utils.cast(data))
def put_file(self, in_path, out_path): def put_file(self, in_path, out_path):
""" """
Implement put_file() by caling the corresponding Implement put_file() by caling the corresponding
ansible_mitogen.helpers function in the target. ansible_mitogen.target function in the target.
:param str in_path: :param str in_path:
Local filesystem path to read. Local filesystem path to read.
:param str out_path: :param str out_path:
Remote filesystem path to write. Remote filesystem path to write.
""" """
self.put_data(out_path, ansible_mitogen.helpers.read_path(in_path)) self.put_data(out_path, ansible_mitogen.target.read_path(in_path))

@ -1,306 +0,0 @@
# Copyright 2017, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import
import json
import logging
import operator
import os
import pwd
import random
import re
import stat
import subprocess
import tempfile
import threading
import zlib
import mitogen.core
import mitogen.service
import ansible_mitogen.runner
import ansible_mitogen.services
LOG = logging.getLogger(__name__)
#: Caching of fetched file data.
_file_cache = {}
#: Mapping of job_id<->result dict
_result_by_job_id = {}
#: Mapping of job_id<->threading.Thread
_thread_by_job_id = {}
def get_file(context, path):
"""
Basic in-memory caching module fetcher. This generates an one roundtrip for
every previously unseen module, so it is only temporary.
:param context:
Context we should direct FileService requests to. For now (and probably
forever) this is just the top-level Mitogen connection manager process.
:param path:
Path to fetch from FileService, must previously have been registered by
a privileged context using the `register` command.
:returns:
Bytestring file data.
"""
if path not in _file_cache:
_file_cache[path] = zlib.decompress(
mitogen.service.call(
context,
ansible_mitogen.services.FileService.handle,
('fetch', path)
)
)
return _file_cache[path]
def run_module(kwargs):
"""
Set up the process environment in preparation for running an Ansible
module. This monkey-patches the Ansible libraries in various places to
prevent it from trying to kill the process on completion, and to prevent it
from reading sys.stdin.
"""
runner_name = kwargs.pop('runner_name')
klass = getattr(ansible_mitogen.runner, runner_name)
impl = klass(**kwargs)
return impl.run()
def _async_main(job_id, runner_name, kwargs):
"""
Implementation for the thread that implements asynchronous module
execution.
"""
try:
rc = run_module(runner_name, kwargs)
except Exception, e:
rc = mitogen.core.CallError(e)
_result_by_job_id[job_id] = rc
def make_temp_directory(base_dir):
"""
Handle creation of `base_dir` if it is absent, in addition to a unique
temporary directory within `base_dir`.
:returns:
Newly created temporary directory.
"""
if not os.path.exists(base_dir):
os.makedirs(base_dir, mode=int('0700', 8))
return tempfile.mkdtemp(
dir=base_dir,
prefix='ansible-mitogen-tmp-',
)
def run_module_async(runner_name, kwargs):
"""
Arrange for an Ansible module to be executed in a thread of the current
process, with results available via :py:func:`get_async_result`.
"""
job_id = '%08x' % random.randint(0, 2**32-1)
_result_by_job_id[job_id] = None
_thread_by_job_id[job_id] = threading.Thread(
target=_async_main,
kwargs={
'job_id': job_id,
'runner_name': runner_name,
'kwargs': kwargs,
}
)
_thread_by_job_id[job_id].start()
return json.dumps({
'ansible_job_id': job_id,
'changed': True
})
def get_async_result(job_id):
"""
Poll for the result of an asynchronous task.
:param str job_id:
Job ID to poll for.
:returns:
``None`` if job is still running, JSON-encoded result dictionary if
execution completed normally, or :py:class:`mitogen.core.CallError` if
an exception was thrown.
"""
if not _thread_by_job_id[job_id].isAlive():
return _result_by_job_id[job_id]
def get_user_shell():
"""
For commands executed directly via an SSH command-line, SSH looks up the
user's shell via getpwuid() and only defaults to /bin/sh if that field is
missing or empty.
"""
try:
pw_shell = pwd.getpwuid(os.geteuid()).pw_shell
except KeyError:
pw_shell = None
return pw_shell or '/bin/sh'
def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
"""
Run a command in a subprocess, emulating the argument handling behaviour of
SSH.
:param list[str]:
Argument vector.
:param bytes in_data:
Optional standard input for the command.
:param bool emulate_tty:
If :data:`True`, arrange for stdout and stderr to be merged into the
stdout pipe and for LF to be translated into CRLF, emulating the
behaviour of a TTY.
:return:
(return code, stdout bytes, stderr bytes)
"""
LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir)
assert isinstance(args, list)
if emulate_tty:
stderr = subprocess.STDOUT
else:
stderr = subprocess.PIPE
proc = subprocess.Popen(
args=args,
stdout=subprocess.PIPE,
stderr=stderr,
stdin=subprocess.PIPE,
cwd=chdir,
)
stdout, stderr = proc.communicate(in_data)
if emulate_tty:
stdout = stdout.replace('\n', '\r\n')
return proc.returncode, stdout, stderr or ''
def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
"""
Run a command in a subprocess, emulating the argument handling behaviour of
SSH.
:param bytes cmd:
String command line, passed to user's shell.
:param bytes in_data:
Optional standard input for the command.
:return:
(return code, stdout bytes, stderr bytes)
"""
assert isinstance(cmd, basestring)
return exec_args(
args=[get_user_shell(), '-c', cmd],
in_data=in_data,
chdir=chdir,
shell=shell,
emulate_tty=emulate_tty,
)
def read_path(path):
"""
Fetch the contents of a filesystem `path` as bytes.
"""
return open(path, 'rb').read()
def write_path(path, s):
"""
Writes bytes `s` to a filesystem `path`.
"""
open(path, 'wb').write(s)
CHMOD_CLAUSE_PAT = re.compile(r'([uoga]*)([+\-=])([ugo]|[rwx]*)')
CHMOD_MASKS = {
'u': stat.S_IRWXU,
'g': stat.S_IRWXG,
'o': stat.S_IRWXO,
'a': (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO),
}
CHMOD_BITS = {
'u': {'r': stat.S_IRUSR, 'w': stat.S_IWUSR, 'x': stat.S_IXUSR},
'g': {'r': stat.S_IRGRP, 'w': stat.S_IWGRP, 'x': stat.S_IXGRP},
'o': {'r': stat.S_IROTH, 'w': stat.S_IWOTH, 'x': stat.S_IXOTH},
'a': {
'r': (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH),
'w': (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
'x': (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
}
}
def apply_mode_spec(spec, mode):
"""
Given a symbolic file mode change specification in the style of chmod(1)
`spec`, apply changes in the specification to the numeric file mode `mode`.
"""
for clause in spec.split(','):
match = CHMOD_CLAUSE_PAT.match(clause)
who, op, perms = match.groups()
for ch in who or 'a':
mask = CHMOD_MASKS[ch]
bits = CHMOD_BITS[ch]
cur_perm_bits = mode & mask
new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0)
mode &= ~mask
if op == '=':
mode |= new_perm_bits
elif op == '+':
mode |= new_perm_bits | cur_perm_bits
else:
mode |= cur_perm_bits & ~new_perm_bits
return mode
def set_file_mode(path, spec):
"""
Update the permissions of a file using the same syntax as chmod(1).
"""
mode = os.stat(path).st_mode
if spec.isdigit():
new_mode = int(spec, 8)
else:
new_mode = apply_mode_spec(spec, mode)
os.chmod(path, new_mode)

@ -54,7 +54,7 @@ from mitogen.utils import cast
import ansible_mitogen.connection import ansible_mitogen.connection
import ansible_mitogen.planner import ansible_mitogen.planner
import ansible_mitogen.helpers import ansible_mitogen.target
from ansible.module_utils._text import to_text from ansible.module_utils._text import to_text
@ -117,7 +117,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
""" """
Arrange for a Python function to be called in the target context, which Arrange for a Python function to be called in the target context, which
should be some function from the standard library or should be some function from the standard library or
ansible_mitogen.helpers module. This junction point exists mainly as a ansible_mitogen.target module. This junction point exists mainly as a
nice place to insert print statements during debugging. nice place to insert print statements during debugging.
""" """
return self._connection.call(func, *args, **kwargs) return self._connection.call(func, *args, **kwargs)
@ -200,7 +200,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
# The copy action plugin violates layering and grabs this attribute # The copy action plugin violates layering and grabs this attribute
# directly. # directly.
self._connection._shell.tmpdir = self.call( self._connection._shell.tmpdir = self.call(
ansible_mitogen.helpers.make_temp_directory, ansible_mitogen.target.make_temp_directory,
base_dir=self._get_remote_tmp(), base_dir=self._get_remote_tmp(),
) )
LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir)
@ -255,7 +255,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
paths, mode, sudoable) paths, mode, sudoable)
return self.fake_shell(lambda: mitogen.master.Select.all( return self.fake_shell(lambda: mitogen.master.Select.all(
self._connection.call_async( self._connection.call_async(
ansible_mitogen.helpers.set_file_mode, path, mode ansible_mitogen.target.set_file_mode, path, mode
) )
for path in paths for path in paths
)) ))
@ -299,7 +299,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
delete_remote_tmp=True, wrap_async=False): delete_remote_tmp=True, wrap_async=False):
""" """
Collect up a module's execution environment then use it to invoke Collect up a module's execution environment then use it to invoke
helpers.run_module() or helpers.run_module_async() in the target target.run_module() or helpers.run_module_async() in the target
context. context.
""" """
if module_name is None: if module_name is None:
@ -358,7 +358,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
chdir=None): chdir=None):
""" """
Override the base implementation by simply calling Override the base implementation by simply calling
helpers.exec_command() in the target context. target.exec_command() in the target context.
""" """
LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)', LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)',
cmd, type(in_data), executable, chdir) cmd, type(in_data), executable, chdir)

@ -48,7 +48,7 @@ except ImportError: # Ansible <2.4
import mitogen import mitogen
import mitogen.service import mitogen.service
import ansible_mitogen.helpers import ansible_mitogen.target
import ansible_mitogen.services import ansible_mitogen.services
@ -89,7 +89,7 @@ def parse_script_interpreter(source):
class Invocation(object): class Invocation(object):
""" """
Collect up a module's execution environment then use it to invoke Collect up a module's execution environment then use it to invoke
helpers.run_module() or helpers.run_module_async() in the target context. target.run_module() or helpers.run_module_async() in the target context.
""" """
def __init__(self, action, connection, module_name, module_args, def __init__(self, action, connection, module_name, module_args,
remote_tmp, task_vars, templar, env, wrap_async): remote_tmp, task_vars, templar, env, wrap_async):
@ -335,9 +335,9 @@ def invoke(invocation):
kwargs = planner.plan(invocation) kwargs = planner.plan(invocation)
if invocation.wrap_async: if invocation.wrap_async:
helper = ansible_mitogen.helpers.run_module_async helper = ansible_mitogen.target.run_module_async
else: else:
helper = ansible_mitogen.helpers.run_module helper = ansible_mitogen.target.run_module
try: try:
js = invocation.connection.call(helper, kwargs) js = invocation.connection.call(helper, kwargs)

@ -28,7 +28,7 @@
import ansible.plugins.action import ansible.plugins.action
import mitogen.core import mitogen.core
import ansible_mitogen.helpers import ansible_mitogen.target
from mitogen.utils import cast from mitogen.utils import cast
@ -37,7 +37,7 @@ class ActionModule(ansible.plugins.action.ActionBase):
job_id = self._task.args['jid'] job_id = self._task.args['jid']
try: try:
result = self._connection.call( result = self._connection.call(
ansible_mitogen.helpers.get_async_result, ansible_mitogen.target.get_async_result,
cast(job_id), cast(job_id),
) )
except mitogen.core.CallError, e: except mitogen.core.CallError, e:

@ -29,7 +29,7 @@
""" """
These classes implement execution for each style of Ansible module. They are These classes implement execution for each style of Ansible module. They are
instantiated in the target context by way of helpers.py::run_module(). instantiated in the target context by way of target.py::run_module().
Each class in here has a corresponding Planner class in planners.py that knows Each class in here has a corresponding Planner class in planners.py that knows
how to build arguments for it, preseed related data, etc. how to build arguments for it, preseed related data, etc.
@ -45,7 +45,7 @@ import sys
import tempfile import tempfile
import types import types
import ansible_mitogen.helpers # TODO: circular import import ansible_mitogen.target # TODO: circular import
try: try:
from shlex import quote as shlex_quote from shlex import quote as shlex_quote
@ -97,7 +97,7 @@ class Runner(object):
def get_temp_dir(self): def get_temp_dir(self):
if not self._temp_dir: if not self._temp_dir:
self._temp_dir = ansible_mitogen.helpers.make_temp_directory( self._temp_dir = ansible_mitogen.target.make_temp_directory(
self.remote_tmp, self.remote_tmp,
) )
return self._temp_dir return self._temp_dir
@ -220,7 +220,7 @@ class ProgramRunner(Runner):
""" """
Fetch the module binary from the master if necessary. Fetch the module binary from the master if necessary.
""" """
return ansible_mitogen.helpers.get_file( return ansible_mitogen.target.get_file(
context=self.service_context, context=self.service_context,
path=self.path, path=self.path,
) )
@ -241,7 +241,7 @@ class ProgramRunner(Runner):
def _run(self): def _run(self):
try: try:
rc, stdout, stderr = ansible_mitogen.helpers.exec_args( rc, stdout, stderr = ansible_mitogen.target.exec_args(
args=self._get_program_args(), args=self._get_program_args(),
emulate_tty=True, emulate_tty=True,
) )
@ -362,7 +362,7 @@ class NewStyleRunner(ScriptRunner):
return self._code_by_path[self.path] return self._code_by_path[self.path]
except KeyError: except KeyError:
return self._code_by_path.setdefault(self.path, compile( return self._code_by_path.setdefault(self.path, compile(
source=ansible_mitogen.helpers.get_file( source=ansible_mitogen.target.get_file(
context=self.service_context, context=self.service_context,
path=self.path, path=self.path,
), ),

@ -131,12 +131,12 @@ class StrategyMixin(object):
For action plug-ins, the original class is looked up as usual, but a For action plug-ins, the original class is looked up as usual, but a
new subclass is created dynamically in order to mix-in new subclass is created dynamically in order to mix-in
ansible_mitogen.helpers.ActionModuleMixin, which overrides many of the ansible_mitogen.target.ActionModuleMixin, which overrides many of the
methods usually inherited from ActionBase in order to replace them with methods usually inherited from ActionBase in order to replace them with
pure-Python equivalents that avoid the use of shell. pure-Python equivalents that avoid the use of shell.
In particular, _execute_module() is overridden with an implementation In particular, _execute_module() is overridden with an implementation
that uses ansible_mitogen.helpers.run_module() executed in the target that uses ansible_mitogen.target.run_module() executed in the target
Context. run_module() implements module execution by importing the Context. run_module() implements module execution by importing the
module as if it were a normal Python module, and capturing its output module as if it were a normal Python module, and capturing its output
in the remote process. Since the Mitogen module loader is active in the in the remote process. Since the Mitogen module loader is active in the

Loading…
Cancel
Save