Add `use_rsa_sha2_algorithms` option for paramiko (#78789)

Fixes #76737
Fixes #77673

Co-authored-by: Matt Clay <matt@mystile.com>

changelogs/fragments/76737-paramiko-rsa-sha2.yml

@@ -0,0 +1,5 @@
+bugfixes:
+- paramiko - Add a new option to allow paramiko >= 2.9 to easily work with
+  all devices now that rsa-sha2 support was added to paramiko, which
+  prevented communication with numerous platforms.
+  (https://github.com/ansible/ansible/issues/76737)

lib/ansible/module_utils/compat/paramiko.py

@@ -6,10 +6,13 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import types
+import warnings
 
 PARAMIKO_IMPORT_ERR = None
 
 try:
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', message='Blowfish has been deprecated', category=UserWarning)
         import paramiko
 # paramiko and gssapi are incompatible and raise AttributeError not ImportError
 # When running in FIPS mode, cryptography raises InternalError

lib/ansible/plugins/connection/paramiko_ssh.py

@@ -58,6 +58,20 @@ DOCUMENTATION = """
             - name: ansible_paramiko_pass
             - name: ansible_paramiko_password
               version_added: '2.5'
+      use_rsa_sha2_algorithms:
+        description:
+            - Whether or not to enable RSA SHA2 algorithms for pubkeys and hostkeys
+            - On paramiko versions older than 2.9, this only affects hostkeys
+            - For behavior matching paramiko<2.9 set this to C(False)
+        vars:
+            - name: ansible_paramiko_use_rsa_sha2_algorithms
+        ini:
+            - {key: use_rsa_sha2_algorithms, section: paramiko_connection}
+        env:
+            - {name: ANSIBLE_PARAMIKO_USE_RSA_SHA2_ALGORITHMS}
+        default: True
+        type: boolean
+        version_added: '2.14'
       host_key_auto_add:
         description: 'Automatically add host keys'
         env: [{name: ANSIBLE_PARAMIKO_HOST_KEY_AUTO_ADD}]
@@ -374,6 +388,18 @@ class Connection(ConnectionBase):
 
         ssh = paramiko.SSHClient()
 
+        # Set pubkey and hostkey algorithms to disable, the only manipulation allowed currently
+        # is keeping or omitting rsa-sha2 algorithms
+        paramiko_preferred_pubkeys = getattr(paramiko.Transport, '_preferred_pubkeys', ())
+        paramiko_preferred_hostkeys = getattr(paramiko.Transport, '_preferred_keys', ())
+        use_rsa_sha2_algorithms = self.get_option('use_rsa_sha2_algorithms')
+        disabled_algorithms = {}
+        if not use_rsa_sha2_algorithms:
+            if paramiko_preferred_pubkeys:
+                disabled_algorithms['pubkeys'] = tuple(a for a in paramiko_preferred_pubkeys if 'rsa-sha2' in a)
+            if paramiko_preferred_hostkeys:
+                disabled_algorithms['keys'] = tuple(a for a in paramiko_preferred_hostkeys if 'rsa-sha2' in a)
+
         # override paramiko's default logger name
         if self._log_channel is not None:
             ssh.set_log_channel(self._log_channel)
@@ -423,7 +449,8 @@ class Connection(ConnectionBase):
                 password=conn_password,
                 timeout=self._play_context.timeout,
                 port=port,
-                **ssh_connect_kwargs
+                disabled_algorithms=disabled_algorithms,
+                **ssh_connect_kwargs,
             )
         except paramiko.ssh_exception.BadHostKeyException as e:
             raise AnsibleConnectionFailure('host key mismatch for %s' % e.hostname)

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -1,7 +1,6 @@
 # do not add a cryptography or pyopenssl constraint to this file, they require special handling, see get_cryptography_requirements in python_requirements.py
 # do not add a coverage constraint to this file, it is handled internally by ansible-test
 packaging < 21.0 ; python_version < '3.6' # packaging 21.0 requires Python 3.6 or newer
-paramiko < 2.9.0 # paramiko 2.9.0+ requires changes to the paramiko_ssh connection plugin to work with older systems
 pywinrm >= 0.3.0 ; python_version < '3.11' # message encryption support
 pywinrm >= 0.4.3 ; python_version >= '3.11' # support for Python 3.11
 pytest < 5.0.0, >= 4.5.0 ; python_version == '2.7' # pytest 5.0.0 and later will no longer support python 2.7

test/lib/ansible_test/_internal/host_profiles.py

@@ -478,6 +478,7 @@ class NetworkRemoteProfile(RemoteProfile[NetworkRemoteConfig]):
             ansible_port=connection.port,
             ansible_user=connection.username,
             ansible_ssh_private_key_file=core_ci.ssh_key.key,
+            ansible_paramiko_use_rsa_sha2_algorithms='no',
             ansible_network_os=f'{self.config.collection}.{self.config.platform}' if self.config.collection else self.config.platform,
         )
 

test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/doc_fragments/connection_persistent.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+
+class ModuleDocFragment(object):
+
+    # Standard files documentation fragment
+    DOCUMENTATION = r"""
+options:
+  import_modules:
+    type: boolean
+    description:
+    - Reduce CPU usage and network module execution time
+      by enabling direct execution. Instead of the module being packaged
+      and executed by the shell, it will be directly executed by the Ansible
+      control node using the same python interpreter as the Ansible process.
+      Note- Incompatible with C(asynchronous mode).
+      Note- Python 3 and Ansible 2.9.16 or greater required.
+      Note- With Ansible 2.9.x fully qualified modules names are required in tasks.
+    default: true
+    ini:
+    - section: ansible_network
+      key: import_modules
+    env:
+    - name: ANSIBLE_NETWORK_IMPORT_MODULES
+    vars:
+    - name: ansible_network_import_modules
+  persistent_connect_timeout:
+    type: int
+    description:
+    - Configures, in seconds, the amount of time to wait when trying to initially
+      establish a persistent connection.  If this value expires before the connection
+      to the remote device is completed, the connection will fail.
+    default: 30
+    ini:
+    - section: persistent_connection
+      key: connect_timeout
+    env:
+    - name: ANSIBLE_PERSISTENT_CONNECT_TIMEOUT
+    vars:
+    - name: ansible_connect_timeout
+  persistent_command_timeout:
+    type: int
+    description:
+      - Configures, in seconds, the amount of time to wait for a command to
+        return from the remote device.  If this timer is exceeded before the
+        command returns, the connection plugin will raise an exception and
+        close.
+    default: 30
+    ini:
+      - section: persistent_connection
+        key: command_timeout
+    env:
+      - name: ANSIBLE_PERSISTENT_COMMAND_TIMEOUT
+    vars:
+      - name: ansible_command_timeout
+  persistent_log_messages:
+    type: boolean
+    description:
+      - This flag will enable logging the command executed and response received from
+        target device in the ansible log file. For this option to work 'log_path' ansible
+        configuration option is required to be set to a file path with write access.
+      - Be sure to fully understand the security implications of enabling this
+        option as it could create a security vulnerability by logging sensitive information in log file.
+    default: False
+    ini:
+      - section: persistent_connection
+        key: log_messages
+    env:
+      - name: ANSIBLE_PERSISTENT_LOG_MESSAGES
+    vars:
+      - name: ansible_persistent_log_messages
+"""

test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/plugin_utils/connection_base.py

@@ -0,0 +1,185 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
+# (c) 2017, Peter Sprygada <psprygad@redhat.com>
+# (c) 2017 Ansible Project
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import os
+
+from ansible import constants as C
+from ansible.plugins.connection import ConnectionBase
+from ansible.plugins.loader import connection_loader
+from ansible.utils.display import Display
+from ansible.utils.path import unfrackpath
+
+display = Display()
+
+
+__all__ = ["NetworkConnectionBase"]
+
+BUFSIZE = 65536
+
+
+class NetworkConnectionBase(ConnectionBase):
+    """
+    A base class for network-style connections.
+    """
+
+    force_persistence = True
+    # Do not use _remote_is_local in other connections
+    _remote_is_local = True
+
+    def __init__(self, play_context, new_stdin, *args, **kwargs):
+        super(NetworkConnectionBase, self).__init__(
+            play_context, new_stdin, *args, **kwargs
+        )
+        self._messages = []
+        self._conn_closed = False
+
+        self._network_os = self._play_context.network_os
+
+        self._local = connection_loader.get("local", play_context, "/dev/null")
+        self._local.set_options()
+
+        self._sub_plugin = {}
+        self._cached_variables = (None, None, None)
+
+        # reconstruct the socket_path and set instance values accordingly
+        self._ansible_playbook_pid = kwargs.get("ansible_playbook_pid")
+        self._update_connection_state()
+
+    def __getattr__(self, name):
+        try:
+            return self.__dict__[name]
+        except KeyError:
+            if not name.startswith("_"):
+                plugin = self._sub_plugin.get("obj")
+                if plugin:
+                    method = getattr(plugin, name, None)
+                    if method is not None:
+                        return method
+            raise AttributeError(
+                "'%s' object has no attribute '%s'"
+                % (self.__class__.__name__, name)
+            )
+
+    def exec_command(self, cmd, in_data=None, sudoable=True):
+        return self._local.exec_command(cmd, in_data, sudoable)
+
+    def queue_message(self, level, message):
+        """
+        Adds a message to the queue of messages waiting to be pushed back to the controller process.
+
+        :arg level: A string which can either be the name of a method in display, or 'log'. When
+            the messages are returned to task_executor, a value of log will correspond to
+            ``display.display(message, log_only=True)``, while another value will call ``display.[level](message)``
+        """
+        self._messages.append((level, message))
+
+    def pop_messages(self):
+        messages, self._messages = self._messages, []
+        return messages
+
+    def put_file(self, in_path, out_path):
+        """Transfer a file from local to remote"""
+        return self._local.put_file(in_path, out_path)
+
+    def fetch_file(self, in_path, out_path):
+        """Fetch a file from remote to local"""
+        return self._local.fetch_file(in_path, out_path)
+
+    def reset(self):
+        """
+        Reset the connection
+        """
+        if self._socket_path:
+            self.queue_message(
+                "vvvv",
+                "resetting persistent connection for socket_path %s"
+                % self._socket_path,
+            )
+            self.close()
+        self.queue_message("vvvv", "reset call on connection instance")
+
+    def close(self):
+        self._conn_closed = True
+        if self._connected:
+            self._connected = False
+
+    def get_options(self, hostvars=None):
+        options = super(NetworkConnectionBase, self).get_options(
+            hostvars=hostvars
+        )
+
+        if (
+            self._sub_plugin.get("obj")
+            and self._sub_plugin.get("type") != "external"
+        ):
+            try:
+                options.update(
+                    self._sub_plugin["obj"].get_options(hostvars=hostvars)
+                )
+            except AttributeError:
+                pass
+
+        return options
+
+    def set_options(self, task_keys=None, var_options=None, direct=None):
+        super(NetworkConnectionBase, self).set_options(
+            task_keys=task_keys, var_options=var_options, direct=direct
+        )
+        if self.get_option("persistent_log_messages"):
+            warning = (
+                "Persistent connection logging is enabled for %s. This will log ALL interactions"
+                % self._play_context.remote_addr
+            )
+            logpath = getattr(C, "DEFAULT_LOG_PATH")
+            if logpath is not None:
+                warning += " to %s" % logpath
+            self.queue_message(
+                "warning",
+                "%s and WILL NOT redact sensitive configuration like passwords. USE WITH CAUTION!"
+                % warning,
+            )
+
+        if (
+            self._sub_plugin.get("obj")
+            and self._sub_plugin.get("type") != "external"
+        ):
+            try:
+                self._sub_plugin["obj"].set_options(
+                    task_keys=task_keys, var_options=var_options, direct=direct
+                )
+            except AttributeError:
+                pass
+
+    def _update_connection_state(self):
+        """
+        Reconstruct the connection socket_path and check if it exists
+
+        If the socket path exists then the connection is active and set
+        both the _socket_path value to the path and the _connected value
+        to True.  If the socket path doesn't exist, leave the socket path
+        value to None and the _connected value to False
+        """
+        ssh = connection_loader.get("ssh", class_only=True)
+        control_path = ssh._create_control_path(
+            self._play_context.remote_addr,
+            self._play_context.port,
+            self._play_context.remote_user,
+            self._play_context.connection,
+            self._ansible_playbook_pid,
+        )
+
+        tmp_path = unfrackpath(C.PERSISTENT_CONTROL_PATH_DIR)
+        socket_path = unfrackpath(control_path % dict(directory=tmp_path))
+
+        if os.path.exists(socket_path):
+            self._connected = True
+            self._socket_path = socket_path
+
+    def _log_messages(self, message):
+        if self.get_option("persistent_log_messages"):
+            self.queue_message("log", message)