Add pipeline-ish method using dd for file transfer over SSH (#18642)

examples/ansible.cfg

@@ -331,12 +331,20 @@
 #
 #pipelining = False
 
-# Control the mechanism for transferring files
+# Control the mechanism for transferring files (old)
 #   * smart = try sftp and then try scp [default]
 #   * True = use scp only
 #   * False = use sftp only
 #scp_if_ssh = smart
 
+# Control the mechanism for transferring files (new)
+# If set, this will override the scp_if_ssh option
+#   * sftp  = use sftp to transfer files
+#   * scp   = use scp to transfer files
+#   * piped = use 'dd' over SSH to transfer files
+#   * smart = try sftp, scp, and piped, in that order [default]
+#transfer_method = smart
+
 # if False, sftp will not use batch mode to transfer files. This may cause some
 # types of file transfer failures impossible to catch however, and should
 # only be disabled if your sftp version has problems with batch mode

lib/ansible/constants.py

@@ -218,6 +218,7 @@ DEFAULT_VAULT_PASSWORD_FILE = get_config(p, DEFAULTS, 'vault_password_file', 'AN
 DEFAULT_TRANSPORT         = get_config(p, DEFAULTS, 'transport',        'ANSIBLE_TRANSPORT',        'smart')
 DEFAULT_SCP_IF_SSH        = get_config(p, 'ssh_connection', 'scp_if_ssh',       'ANSIBLE_SCP_IF_SSH',       'smart')
 DEFAULT_SFTP_BATCH_MODE   = get_config(p, 'ssh_connection', 'sftp_batch_mode', 'ANSIBLE_SFTP_BATCH_MODE', True, value_type='boolean')
+DEFAULT_SSH_TRANSFER_METHOD = get_config(p, 'ssh_connection', 'transfer_method', 'ANSIBLE_SSH_TRANSFER_METHOD', None)
 DEFAULT_MANAGED_STR       = get_config(p, DEFAULTS, 'ansible_managed',  None,           'Ansible managed')
 DEFAULT_SYSLOG_FACILITY   = get_config(p, DEFAULTS, 'syslog_facility',  'ANSIBLE_SYSLOG_FACILITY', 'LOG_USER')
 DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBLE_KEEP_REMOTE_FILES', False, value_type='boolean')

lib/ansible/playbook/play_context.py

@@ -73,6 +73,7 @@ MAGIC_VARIABLE_MAPPING = dict(
    sftp_extra_args  = ('ansible_sftp_extra_args',),
    scp_extra_args   = ('ansible_scp_extra_args',),
    ssh_extra_args   = ('ansible_ssh_extra_args',),
+   ssh_transfer_method = ('ansible_ssh_transfer_method',),
    sudo             = ('ansible_sudo',),
    sudo_user        = ('ansible_sudo_user',),
    sudo_pass        = ('ansible_sudo_password', 'ansible_sudo_pass'),
@@ -173,6 +174,7 @@ class PlayContext(Base):
     _scp_extra_args   = FieldAttribute(isa='string')
     _ssh_extra_args   = FieldAttribute(isa='string')
     _ssh_executable   = FieldAttribute(isa='string', default=C.ANSIBLE_SSH_EXECUTABLE)
+    _ssh_transfer_method = FieldAttribute(isa='string', default=C.DEFAULT_SSH_TRANSFER_METHOD)
     _connection_lockfd= FieldAttribute(isa='int')
     _pipelining       = FieldAttribute(isa='bool', default=C.ANSIBLE_SSH_PIPELINING)
     _accelerate       = FieldAttribute(isa='bool', default=False)

lib/ansible/plugins/connection/ssh.py

@@ -34,7 +34,7 @@ from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNo
 from ansible.errors import AnsibleOptionsError
 from ansible.module_utils.basic import BOOLEANS
 from ansible.module_utils._text import to_bytes, to_native, to_text
-from ansible.plugins.connection import ConnectionBase
+from ansible.plugins.connection import ConnectionBase, BUFSIZE
 from ansible.utils.path import unfrackpath, makedirs_safe
 
 boolean = C.mk_boolean
@@ -605,44 +605,70 @@ class Connection(ConnectionBase):
         # accept them for hostnames and IPv4 addresses too.
         host = '[%s]' % self.host
 
-        # since this can be a non-bool now, we need to handle it correctly
+        # Transfer methods to try
-        scp_if_ssh = C.DEFAULT_SCP_IF_SSH
+        methods = []
-        if not isinstance(scp_if_ssh, bool):
+
-            scp_if_ssh = scp_if_ssh.lower()
+        # Use the transfer_method option if set, otherwise use scp_if_ssh
-            if scp_if_ssh in BOOLEANS:
+        ssh_transfer_method = self._play_context.ssh_transfer_method
-                scp_if_ssh = boolean(scp_if_ssh)
+        if ssh_transfer_method is not None:
-            elif scp_if_ssh != 'smart':
+            if not (ssh_transfer_method in ('smart', 'sftp', 'scp', 'piped')):
-                raise AnsibleOptionsError('scp_if_ssh needs to be one of [smart|True|False]')
+                raise AnsibleOptionsError('transfer_method needs to be one of [smart|sftp|scp|piped]')
-
+            if ssh_transfer_method == 'smart':
-        # create a list of commands to use based on config options
+                methods = ['sftp', 'scp', 'piped']
-        methods = ['sftp']
+            else:
-        if scp_if_ssh == 'smart':
+                methods = [ssh_transfer_method]
-            methods.append('scp')
+        else:
-        elif scp_if_ssh:
+            # since this can be a non-bool now, we need to handle it correctly
-            methods = ['scp']
+            scp_if_ssh = C.DEFAULT_SCP_IF_SSH
+            if not isinstance(scp_if_ssh, bool):
+                scp_if_ssh = scp_if_ssh.lower()
+                if scp_if_ssh in BOOLEANS:
+                    scp_if_ssh = boolean(scp_if_ssh)
+                elif scp_if_ssh != 'smart':
+                    raise AnsibleOptionsError('scp_if_ssh needs to be one of [smart|True|False]')
+            if scp_if_ssh == 'smart':
+                methods = ['sftp', 'scp', 'piped']
+            elif scp_if_ssh == True:
+                methods = ['scp']
+            else:
+                methods = ['sftp']
 
         success = False
         res = None
         for method in methods:
+            returncode = stdout = stderr = None
             if method == 'sftp':
                 cmd = self._build_command('sftp', to_bytes(host))
                 in_data = u"{0} {1} {2}\n".format(sftp_action, shlex_quote(in_path), shlex_quote(out_path))
+                in_data = to_bytes(in_data, nonstring='passthru')
+                (returncode, stdout, stderr) = self._run(cmd, in_data, checkrc=False)
             elif method == 'scp':
                 if sftp_action == 'get':
                     cmd = self._build_command('scp', u'{0}:{1}'.format(host, shlex_quote(in_path)), out_path)
                 else:
                     cmd = self._build_command('scp', in_path, u'{0}:{1}'.format(host, shlex_quote(out_path)))
                 in_data = None
+                (returncode, stdout, stderr) = self._run(cmd, in_data, checkrc=False)
+            elif method == 'piped':
+                if sftp_action == 'get':
+                    # we pass sudoable=False to disable pty allocation, which
+                    # would end up mixing stdout/stderr and screwing with newlines
+                    (returncode, stdout, stderr) = self._exec_command('dd if=%s bs=%s' % (in_path, BUFSIZE), sudoable=False)
+                    out_file = open(to_bytes(out_path, errors='surrogate_or_strict'), 'wb+')
+                    out_file.write(stdout)
+                    out_file.close()
+                else:
+                    in_data = open(to_bytes(in_path, errors='surrogate_or_strict'), 'rb').read()
+                    in_data = to_bytes(in_data, nonstring='passthru')
+                    (returncode, stdout, stderr) = self._exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), in_data=in_data)
 
-            in_data = to_bytes(in_data, nonstring='passthru')
-            (returncode, stdout, stderr) = self._run(cmd, in_data, checkrc=False)
             # Check the return code and rollover to next method if failed
             if returncode == 0:
                 success = True
                 break
             else:
                 # If not in smart mode, the data will be printed by the raise below
-                if scp_if_ssh == 'smart':
+                if len(methods) > 1:
                     display.warning(msg='%s transfer mechanism failed on %s. Use ANSIBLE_DEBUG=1 to see detailed information' % (method, host))
                     display.debug(msg='%s' % to_native(stdout))
                     display.debug(msg='%s' % to_native(stderr))

test/integration/targets/connection_ssh/runme.sh

@@ -6,3 +6,5 @@ set -eux
 ./posix.sh "$@"
 # scp
 ANSIBLE_SCP_IF_SSH=true ./posix.sh "$@"
+# piped
+ANSIBLE_SSH_TRANSFER_METHOD=piped ./posix.sh "$@"