Merge pull request #308 from dw/dmw

Dmw
pull/312/head
dw 6 years ago committed by GitHub
commit def848f118
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -173,6 +173,20 @@ def _connect_sudo(spec):
}
def _connect_doas(spec):
return {
'method': 'doas',
'enable_lru': True,
'kwargs': {
'username': spec['become_user'],
'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']),
'python_path': spec['python_path'],
'doas_path': spec['become_exe'],
'connect_timeout': spec['timeout'],
}
}
def _connect_mitogen_su(spec):
# su as a first-class proxied connection, not a become method.
return {
@ -202,6 +216,20 @@ def _connect_mitogen_sudo(spec):
}
def _connect_mitogen_doas(spec):
# doas as a first-class proxied connection, not a become method.
return {
'method': 'doas',
'kwargs': {
'username': spec['remote_user'],
'password': wrap_or_none(mitogen.core.Secret, spec['password']),
'python_path': spec['python_path'],
'doas_path': spec['become_exe'],
'connect_timeout': spec['timeout'],
}
}
CONNECTION_METHOD = {
'docker': _connect_docker,
'jail': _connect_jail,
@ -213,8 +241,10 @@ CONNECTION_METHOD = {
'ssh': _connect_ssh,
'su': _connect_su,
'sudo': _connect_sudo,
'doas': _connect_doas,
'mitogen_su': _connect_mitogen_su,
'mitogen_sudo': _connect_mitogen_sudo,
'mitogen_doas': _connect_mitogen_doas,
}
@ -314,8 +344,8 @@ class Connection(ansible.plugins.connection.ConnectionBase):
#: target user account.
fork_context = None
#: Only sudo and su are supported for now.
become_methods = ['sudo', 'su']
#: Only sudo, su, and doas are supported for now.
become_methods = ['sudo', 'su', 'doas']
#: Set to 'ansible_python_interpreter' by on_action_run().
python_path = None

@ -0,0 +1,43 @@
# Copyright 2017, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import os.path
import sys
try:
import ansible_mitogen.connection
except ImportError:
base_dir = os.path.dirname(__file__)
sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
del base_dir
import ansible_mitogen.connection
class Connection(ansible_mitogen.connection.Connection):
transport = 'mitogen_doas'

@ -54,7 +54,7 @@ def wrap_action_loader__get(name, *args, **kwargs):
return adorned_klass(*args, **kwargs)
def wrap_connection_loader__get(name, play_context, new_stdin, **kwargs):
def wrap_connection_loader__get(name, *args, **kwargs):
"""
While the strategy is active, rewrite connection_loader.get() calls for
some transports into requests for a compatible Mitogen transport.
@ -62,7 +62,7 @@ def wrap_connection_loader__get(name, play_context, new_stdin, **kwargs):
if name in ('docker', 'jail', 'local', 'lxc',
'lxd', 'machinectl', 'setns', 'ssh'):
name = 'mitogen_' + name
return connection_loader__get(name, play_context, new_stdin, **kwargs)
return connection_loader__get(name, *args, **kwargs)
class StrategyMixin(object):

@ -45,7 +45,7 @@ it can only ensure the module executes as quickly as possible.
* **Fewer writes to the target filesystem occur**. In typical configurations,
Ansible repeatedly rewrites and extracts ZIP files to multiple temporary
directories on the target. Security issues relating to temporarily files in
directories on the target. Security issues relating to temporary files in
cross-account scenarios are entirely avoided.
The effect is most potent on playbooks that execute many **short-lived
@ -60,13 +60,13 @@ Installation
1. Thoroughly review the :ref:`noteworthy_differences` and :ref:`changelog`.
2. Verify Ansible 2.3-2.5 and Python 2.6, 2.7 or 3.6 are listed in ``ansible
--version`` output.
3. Download and extract https://github.com/dw/mitogen/archive/stable.zip
3. Download and extract |mitogen_url| from PyPI.
4. Modify ``ansible.cfg``:
.. code-block:: dosini
.. parsed-literal::
[defaults]
strategy_plugins = /path/to/mitogen-master/ansible_mitogen/plugins/strategy
strategy_plugins = /path/to/mitogen-|mitogen_version|/ansible_mitogen/plugins/strategy
strategy = mitogen_linear
The ``strategy`` key is optional. If omitted, the
@ -90,7 +90,7 @@ concurrent to an equivalent run using the extension.
.. raw:: html
<video width="720" height="439" controls>
<source src="http://k3.botanicus.net/tmp/ansible_mitogen.mp4" type="video/mp4">
<source src="https://k3.botanicus.net/tmp/ansible_mitogen.mp4" type="video/mp4">
</video>
@ -127,8 +127,8 @@ Noteworthy Differences
precluding its use for installing Python on a target. This will be addressed
soon.
* The ``su`` and ``sudo`` become methods are available. File bugs to register
interest in more.
* The ``doas``, ``su`` and ``sudo`` become methods are available. File bugs to
register interest in more.
* The `docker <https://docs.ansible.com/ansible/2.5/plugins/connection/docker.html>`_,
`jail <https://docs.ansible.com/ansible/2.5/plugins/connection/jail.html>`_,
@ -137,9 +137,9 @@ Noteworthy Differences
`lxd <https://docs.ansible.com/ansible/2.5/plugins/connection/lxd.html>`_,
and `ssh <https://docs.ansible.com/ansible/2.5/plugins/connection/ssh.html>`_
built-in connection types are supported, along with Mitogen-specific
:ref:`machinectl <machinectl>`, :ref:`mitogen_su <su>`, :ref:`mitogen_sudo
<sudo>`, and :ref:`setns <setns>` types. File bugs to register interest in
others.
:ref:`machinectl <machinectl>`, :ref:`mitogen_doas< mitogen_doas>`,
:ref:`mitogen_su <su>`, :ref:`mitogen_sudo <sudo>`, and :ref:`setns <setns>`
types. File bugs to register interest in others.
* Local commands execute in a reuseable interpreter created identically to
interpreters on targets. Presently one interpreter per ``become_user``
@ -477,6 +477,30 @@ establishment of additional reuseable interpreters as necessary to match the
configuration of each task.
.. _doas:
Doas
~~~~
``doas`` can be used as a connection method that supports connection delegation, or
as a become method.
When used as a become method:
* ``ansible_python_interpreter``
* ``ansible_become_exe``: path to ``doas`` binary.
* ``ansible_become_user`` (default: ``root``)
* ``ansible_become_pass`` (default: assume passwordless)
* ansible.cfg: ``timeout``
When used as the ``mitogen_doas`` connection method:
* The inventory hostname has no special meaning.
* ``ansible_user``: username to use.
* ``ansible_password``: password to use.
* ``ansible_python_interpreter``
.. _method-docker:
Docker

@ -523,6 +523,31 @@ Router Class
# Use the SSH connection to create a sudo connection.
remote_root = router.sudo(username='root', via=remote_machine)
.. method:: dos (username=None, password=None, su_path=None, password_prompt=None, incorrect_prompts=None, \**kwargs)
Construct a context on the local machine over a ``su`` invocation. The
``su`` process is started in a newly allocated pseudo-terminal, and
supports typing interactive passwords.
Accepts all parameters accepted by :py:meth:`local`, in addition to:
:param str username:
Username to use, defaults to ``root``.
:param str password:
The account password to use if requested.
:param str su_path:
Filename or complete path to the ``su`` binary. ``PATH`` will be
searched if given as a filename. Defaults to ``su``.
:param bytes password_prompt:
A string that indicates ``doas`` is requesting a password. Defaults
to ``Password:``.
:param list incorrect_prompts:
List of bytestrings indicating the password is incorrect. Defaults
to `(b"doas: authentication failed")`.
:raises mitogen.su.PasswordError:
A password was requested but none was provided, the supplied
password was incorrect, or the target account did not exist.
.. method:: docker (container=None, image=None, docker_path=None, \**kwargs)
Construct a context on the local machine within an existing or
@ -616,12 +641,9 @@ Router Class
:param str su_path:
Filename or complete path to the ``su`` binary. ``PATH`` will be
searched if given as a filename. Defaults to ``su``.
:param str password_prompt:
The string to wait to that signals ``su`` is requesting a password.
Defaults to ``Password:``.
:param str password_prompt:
The string that signal a request for the password. Defaults to
``Password:``.
:param bytes password_prompt:
The string that indicates ``su`` is requesting a password. Defaults
to ``Password:``.
:param str incorrect_prompts:
Strings that signal the password is incorrect. Defaults to `("su:
sorry", "su: authentication failure")`.

@ -15,6 +15,28 @@ Release Notes
</style>
v0.2.2 (2018-07-??)
-------------------
Mitogen for Ansible
~~~~~~~~~~~~~~~~~~~
* `#299 <https://github.com/dw/mitogen/pull/299>`_: fix the ``network_cli``
connection type when the Mitogen strategy is active.
Core Library
~~~~~~~~~~~~
* `#303 <https://github.com/dw/mitogen/pull/303>`_: the ``doas`` become method
is now supported. Contributed by Mike Walker.
* `#307 <https://github.com/dw/mitogen/pull/307>`_: SSH login banner output
containing the word 'password' is no longer confused for a password prompt.
* Debug logs containing command lines are printed with the minimal quoting and
escaping required.
v0.2.1 (2018-07-10)
-------------------
@ -87,6 +109,10 @@ Mitogen for Ansible
- initech_app
- y2k_fix
* When running with ``-vvv``, log messages such as *mitogen: Router(Broker(0x7f5a48921590)): no route
for Message(..., 102, ...), my ID is ...* may be visible. These are due to a
minor race while initializing logging and can be ignored.
* Performance does not scale linearly with target count. This requires
significant additional work, as major bottlenecks exist in the surrounding
Ansible code. Performance-related bug reports for any scenario remain

@ -2,15 +2,8 @@ import os
import sys
sys.path.append('..')
def grep_version():
path = os.path.join(os.path.dirname(__file__), '../mitogen/__init__.py')
with open(path) as fp:
for line in fp:
if line.startswith('__version__'):
_, _, s = line.partition('=')
return '.'.join(map(str, eval(s)))
import mitogen
VERSION = '%s.%s.%s' % mitogen.__version__
author = u'David Wilson'
copyright = u'2018, David Wilson'
@ -31,8 +24,16 @@ language = None
master_doc = 'toc'
project = u'Mitogen'
pygments_style = 'sphinx'
release = grep_version()
release = VERSION
source_suffix = '.rst'
templates_path = ['_templates']
todo_include_todos = False
version = grep_version()
version = VERSION
rst_epilog = """
.. |mitogen_version| replace:: %(VERSION)s
.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__
""" % locals()

@ -106,6 +106,7 @@ sponsorship and outstanding future-thinking of its early adopters.
<li><a href="https://www.epartment.nl/">Epartment</a></li>
<li><a href="http://andrianaivo.org/">Fidy Andrianaivo</a> &mdash; <em>never let a human do an ansible job ;)</em></li>
<li><a href="https://www.channable.com">rkrzr</a></li>
<li>jgadling</li>
<li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
<li>KennethC</li>
<li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>

@ -125,6 +125,8 @@ class Operations(fuse.Operations): # fuse.LoggingMixIn,
self.host = host
self.root = path
self.ready = threading.Event()
if not hasattr(self, 'encoding'):
self.encoding = 'utf-8'
def init(self, path):
self.broker = mitogen.master.Broker(install_watcher=False)

@ -607,6 +607,7 @@ class Importer(object):
self._present = {'mitogen': [
'compat',
'debug',
'doas',
'docker',
'fakessh',
'fork',

@ -0,0 +1,118 @@
# Copyright 2017, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import logging
import os
import mitogen.core
import mitogen.parent
from mitogen.core import b
LOG = logging.getLogger(__name__)
class PasswordError(mitogen.core.StreamError):
pass
class Stream(mitogen.parent.Stream):
create_child = staticmethod(mitogen.parent.hybrid_tty_create_child)
child_is_immediate_subprocess = False
#: Once connected, points to the corresponding TtyLogStream, allowing it to
#: be disconnected at the same time this stream is being torn down.
tty_stream = None
username = 'root'
password = None
doas_path = 'doas'
password_prompt = b('Password:')
incorrect_prompts = (
b('doas: authentication failed'),
)
def construct(self, username=None, password=None, doas_path=None,
password_prompt=None, incorrect_prompts=None, **kwargs):
super(Stream, self).construct(**kwargs)
if username is not None:
self.username = username
if password is not None:
self.password = password
if doas_path is not None:
self.doas_path = doas_path
if password_prompt is not None:
self.password_prompt = password_prompt.lower()
if incorrect_prompts is not None:
self.incorrect_prompts = map(str.lower, incorrect_prompts)
def connect(self):
super(Stream, self).connect()
self.name = u'doas.' + mitogen.core.to_text(self.username)
def on_disconnect(self, broker):
self.tty_stream.on_disconnect(broker)
super(Stream, self).on_disconnect(broker)
def get_boot_command(self):
bits = [self.doas_path, '-u', self.username, '--']
bits = bits + super(Stream, self).get_boot_command()
LOG.debug('doas command line: %r', bits)
return bits
password_incorrect_msg = 'doas password is incorrect'
password_required_msg = 'doas password is required'
def _connect_bootstrap(self, extra_fd):
self.tty_stream = mitogen.parent.TtyLogStream(extra_fd, self)
password_sent = False
it = mitogen.parent.iter_read(
fds=[self.receive_side.fd, extra_fd],
deadline=self.connect_deadline,
)
for buf in it:
LOG.debug('%r: received %r', self, buf)
if buf.endswith(self.EC0_MARKER):
self._ec0_received()
return
if any(s in buf.lower() for s in self.incorrect_prompts):
if password_sent:
raise PasswordError(self.password_incorrect_msg)
elif self.password_prompt in buf.lower():
if self.password is None:
raise PasswordError(self.password_required_msg)
if password_sent:
raise PasswordError(self.password_incorrect_msg)
LOG.debug('sending password')
self.tty_stream.transmit_side.write(
mitogen.core.to_text(self.password + '\n').encode('utf-8')
)
password_sent = True
raise mitogen.core.StreamError('bootstrap failed')

@ -457,10 +457,16 @@ class Argv(object):
def __init__(self, argv):
self.argv = argv
must_escape = frozenset('\\$"`!')
must_escape_or_space = must_escape | frozenset(' ')
def escape(self, x):
if not self.must_escape_or_space.intersection(x):
return x
s = '"'
for c in x:
if c in '\\$"`':
if c in self.must_escape:
s += '\\'
s += c
s += '"'
@ -1240,6 +1246,9 @@ class Router(mitogen.core.Router):
self._context_by_id[context.context_id] = context
return context
def doas(self, **kwargs):
return self.connect(u'doas', **kwargs)
def docker(self, **kwargs):
return self.connect(u'docker', **kwargs)

@ -61,7 +61,18 @@ def filter_debug(stream, it):
This contains the mess of dealing with both line-oriented input, and partial
lines such as the password prompt.
Yields `(line, partial)` tuples, where `line` is the line, `partial` is
:data:`True` if no terminating newline character was present and no more
data exists in the read buffer. Consuming code can use this to unreliably
detect the presence of an interactive prompt.
"""
# The `partial` test is unreliable, but is only problematic when verbosity
# is enabled: it's possible for a combination of SSH banner, password
# prompt, verbose output, timing and OS buffering specifics to create a
# situation where an otherwise newline-terminated line appears to not be
# terminated, due to a partial read(). If something is broken when
# ssh_debug_level>0, this is the first place to look.
state = 'start_of_line'
buf = b('')
for chunk in it:
@ -86,7 +97,7 @@ def filter_debug(stream, it):
state = 'start_of_line'
elif state == 'in_plain':
line, nl, buf = buf.partition(b('\n'))
yield line + nl
yield line + nl, not (nl or buf)
if nl:
state = 'start_of_line'
@ -161,6 +172,11 @@ class Stream(mitogen.parent.Stream):
bits = [self.ssh_path]
if self.ssh_debug_level:
bits += ['-' + ('v' * min(3, self.ssh_debug_level))]
else:
# issue #307: suppress any login banner, as it may contain the
# password prompt, and there is no robust way to tell the
# difference.
bits += ['-o', 'LogLevel ERROR']
if self.username:
bits += ['-l', self.username]
if self.port is not None:
@ -232,7 +248,7 @@ class Stream(mitogen.parent.Stream):
deadline=self.connect_deadline
)
for buf in filter_debug(self, it):
for buf, partial in filter_debug(self, it):
LOG.debug('%r: received %r', self, buf)
if buf.endswith(self.EC0_MARKER):
self._router.broker.start_receive(self.tty_stream)
@ -250,7 +266,7 @@ class Stream(mitogen.parent.Stream):
raise PasswordError(self.password_incorrect_msg)
else:
raise PasswordError(self.auth_incorrect_msg)
elif PASSWORD_PROMPT in buf.lower():
elif partial and PASSWORD_PROMPT in buf.lower():
if self.password is None:
raise PasswordError(self.password_required_msg)
LOG.debug('%r: sending password', self)

@ -45,10 +45,12 @@ RUN yum clean all && \
DOCKERFILE = r"""
COPY data/001-mitogen.sudo /etc/sudoers.d/001-mitogen
COPY data/docker/ssh_login_banner.txt /etc/ssh/banner.txt
RUN \
chsh -s /bin/bash && \
mkdir -p /var/run/sshd && \
echo i-am-mitogen-test-docker-image > /etc/sentinel && \
echo "Banner /etc/ssh/banner.txt" >> /etc/ssh/sshd_config && \
groupadd mitogen__sudo_nopw && \
useradd -s /bin/bash -m mitogen__has_sudo -G SUDO_GROUP && \
useradd -s /bin/bash -m mitogen__has_sudo_pubkey -G SUDO_GROUP && \

@ -0,0 +1,21 @@
This banner tests Mitogen's ability to differentiate the word 'password'
appearing in a login banner, and 'password' appearing in a password prompt.
This system is for the use of authorized users only. Individuals using this
computer system without authority or in excess of their authority are subject
to having all of their activities on this system monitored and recorded by
system personnel.
In the course of monitoring this system with regard to any unauthorized or
improper use or in the course of system maintenance the system personnel may
have insights into regular business activity.
Anyone using this system expressly consents to such monitoring and is advised
that if such monitoring reveals possible evidence of improper activity, system
personnel may provide the evidence of such monitoring to internal Compliance
and Security Officers who will - in the case of criminal offences - relay such
incidents to law enforcement officials.
**************************************************************
NOTE: This system is connected to DOMAIN.COM,
please use your password.

@ -105,5 +105,24 @@ class SshTest(testlib.DockerMixin, unittest2.TestCase):
)
class BannerTest(testlib.DockerMixin, unittest2.TestCase):
# Verify the ability to disambiguate random spam appearing in the SSHd's
# login banner from a legitimate password prompt.
stream_class = mitogen.ssh.Stream
def test_verbose_enabled(self):
context = self.docker_ssh(
username='mitogen__has_sudo',
password='has_sudo_password',
ssh_debug_level=3,
)
name = 'ssh.%s:%s' % (
self.dockerized_ssh.get_host(),
self.dockerized_ssh.port,
)
self.assertEquals(name, context.name)
if __name__ == '__main__':
unittest2.main()

Loading…
Cancel
Save