Remove PyCrypto (#74699)

* Remove PyCrypto from setup.py and packaging script
* Remove mention of pycrpto from installation docs
* Remove PyCrypto from vault
* Remove pycryto constraint and unit test requirement
* Remove PyCrypto tests from unit tests
* Add docs and fix warning message
* Remove section about cryptography library in Ansible Vault docs

changelogs/fragments/74626-remove-pycrypto.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-vault - remove support for ``PyCrypto`` (https://github.com/ansible/ansible/issues/72646)

docs/docsite/rst/installation_guide/intro_installation.rst

@@ -119,11 +119,11 @@ Installing Ansible with ``pip``
 
 	If you have Ansible 2.9 or older installed or Ansible 3, see :ref:`pip_upgrade`.
 
-Once ``pip`` is installed, you can install Ansible [1]_::
+Once ``pip`` is installed, you can install Ansible::
 
     $ python -m pip install --user ansible
 
-In order to use the ``paramiko`` connection plugin or modules that require ``paramiko``, install the required module [2]_::
+In order to use the ``paramiko`` connection plugin or modules that require ``paramiko``, install the required module [1]_::
 
     $ python -m pip install --user paramiko
 
@@ -663,5 +663,4 @@ See the `argcomplete documentation <https://kislyuk.github.io/argcomplete/>`_.
    `irc.freenode.net <http://irc.freenode.net>`_
        #ansible IRC chat channel
 
-.. [1] If you have issues with the "pycrypto" package install on macOS, then you may need to try ``CC=clang sudo -E pip install pycrypto``.
-.. [2] ``paramiko`` was included in Ansible's ``requirements.txt`` prior to 2.8.
+.. [1] ``paramiko`` was included in Ansible's ``requirements.txt`` prior to 2.8.

docs/docsite/rst/porting_guides/porting_guide_core_2.12.rst

@@ -25,7 +25,7 @@ No notable changes
 Command Line
 ============
 
-No notable changes
+* ``ansible-vault`` no longer supports ``PyCrypto`` and requires ``cryptograhpy``.
 
 
 Deprecated

docs/docsite/rst/user_guide/vault.rst

@@ -573,18 +573,6 @@ When are encrypted files made visible?
 
 In general, content you encrypt with Ansible Vault remains encrypted after execution. However, there is one exception. If you pass an encrypted file as the ``src`` argument to the :ref:`copy <copy_module>`, :ref:`template <template_module>`, :ref:`unarchive <unarchive_module>`, :ref:`script <script_module>` or :ref:`assemble <assemble_module>` module, the file will not be encrypted on the target host (assuming you supply the correct vault password when you run the play). This behavior is intended and useful. You can encrypt a configuration file or template to avoid sharing the details of your configuration, but when you copy that configuration to servers in your environment, you want it to be decrypted so local users and processes can access it.
 
-.. _speeding_up_vault:
-
-Speeding up Ansible Vault
-=========================
-
-If you have many encrypted files, decrypting them at startup may cause a perceptible delay. To speed this up, install the cryptography package:
-
-.. code-block:: bash
-
-    pip install cryptography
-
-
 .. _vault_format:
 
 Format of files encrypted with Ansible Vault

lib/ansible/executor/process/worker.py

@@ -25,19 +25,8 @@ import traceback
 
 from jinja2.exceptions import TemplateNotFound
 
-HAS_PYCRYPTO_ATFORK = False
-try:
-    from Crypto.Random import atfork
-    HAS_PYCRYPTO_ATFORK = True
-except Exception:
-    # We only need to call atfork if pycrypto is used because it will need to
-    # reinitialize its RNG.  Since old paramiko could be using pycrypto, we
-    # need to take charge of calling it.
-    pass
-
 from ansible.errors import AnsibleConnectionFailure
 from ansible.executor.task_executor import TaskExecutor
-from ansible.executor.task_result import TaskResult
 from ansible.module_utils._text import to_text
 from ansible.utils.display import Display
 from ansible.utils.multiprocessing import context as multiprocessing_context
@@ -159,9 +148,6 @@ class WorkerProcess(multiprocessing_context.Process):
         # pr = cProfile.Profile()
         # pr.enable()
 
-        if HAS_PYCRYPTO_ATFORK:
-            atfork()
-
         try:
             # execute the task and build a TaskResult from the result
             display.debug("running TaskExecutor() for %s/%s" % (self._host, self._task))

lib/ansible/parsing/vault/__init__.py

@@ -35,8 +35,6 @@ from binascii import unhexlify
 from binascii import Error as BinasciiError
 
 HAS_CRYPTOGRAPHY = False
-HAS_PYCRYPTO = False
-HAS_SOME_PYCRYPTO = False
 CRYPTOGRAPHY_BACKEND = None
 try:
     with warnings.catch_warnings():
@@ -54,24 +52,6 @@ try:
 except ImportError:
     pass
 
-try:
-    from Crypto.Cipher import AES as AES_pycrypto
-    HAS_SOME_PYCRYPTO = True
-
-    # Note: Only used for loading obsolete VaultAES files.  All files are written
-    # using the newer VaultAES256 which does not require md5
-    from Crypto.Hash import SHA256 as SHA256_pycrypto
-    from Crypto.Hash import HMAC as HMAC_pycrypto
-
-    # Counter import fails for 2.0.1, requires >= 2.6.1 from pip
-    from Crypto.Util import Counter as Counter_pycrypto
-
-    # KDF import fails for 2.0.1, requires >= 2.6.1 from pip
-    from Crypto.Protocol.KDF import PBKDF2 as PBKDF2_pycrypto
-    HAS_PYCRYPTO = True
-except ImportError:
-    pass
-
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible import constants as C
 from ansible.module_utils.six import PY3, binary_type
@@ -90,10 +70,7 @@ CIPHER_WRITE_WHITELIST = frozenset((u'AES256',))
 # See also CIPHER_MAPPING at the bottom of the file which maps cipher strings
 # (used in VaultFile header) to a cipher class
 
-NEED_CRYPTO_LIBRARY = "ansible-vault requires either the cryptography library (preferred) or"
-if HAS_SOME_PYCRYPTO:
-    NEED_CRYPTO_LIBRARY += " a newer version of"
-NEED_CRYPTO_LIBRARY += " pycrypto in order to function."
+NEED_CRYPTO_LIBRARY = "ansible-vault requires the cryptography library in order to function"
 
 
 class AnsibleVaultError(AnsibleError):
@@ -1169,7 +1146,7 @@ class VaultAES256:
     # Note: strings in this class should be byte strings by default.
 
     def __init__(self):
-        if not HAS_CRYPTOGRAPHY and not HAS_PYCRYPTO:
+        if not HAS_CRYPTOGRAPHY:
             raise AnsibleError(NEED_CRYPTO_LIBRARY)
 
     @staticmethod
@@ -1184,20 +1161,6 @@ class VaultAES256:
 
         return b_derivedkey
 
-    @staticmethod
-    def _pbkdf2_prf(p, s):
-        hash_function = SHA256_pycrypto
-        return HMAC_pycrypto.new(p, s, hash_function).digest()
-
-    @classmethod
-    def _create_key_pycrypto(cls, b_password, b_salt, key_length, iv_length):
-
-        # make two keys and one iv
-
-        b_derivedkey = PBKDF2_pycrypto(b_password, b_salt, dkLen=(2 * key_length) + iv_length,
-                                       count=10000, prf=cls._pbkdf2_prf)
-        return b_derivedkey
-
     @classmethod
     def _gen_key_initctr(cls, b_password, b_salt):
         # 16 for AES 128, 32 for AES256
@@ -1209,12 +1172,6 @@ class VaultAES256:
 
             b_derivedkey = cls._create_key_cryptography(b_password, b_salt, key_length, iv_length)
             b_iv = b_derivedkey[(key_length * 2):(key_length * 2) + iv_length]
-        elif HAS_PYCRYPTO:
-            # match the size used for counter.new to avoid extra work
-            iv_length = 16
-
-            b_derivedkey = cls._create_key_pycrypto(b_password, b_salt, key_length, iv_length)
-            b_iv = hexlify(b_derivedkey[(key_length * 2):(key_length * 2) + iv_length])
         else:
             raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in initctr)')
 
@@ -1238,34 +1195,6 @@ class VaultAES256:
 
         return to_bytes(hexlify(b_hmac), errors='surrogate_or_strict'), hexlify(b_ciphertext)
 
-    @staticmethod
-    def _encrypt_pycrypto(b_plaintext, b_key1, b_key2, b_iv):
-        # PKCS#7 PAD DATA http://tools.ietf.org/html/rfc5652#section-6.3
-        bs = AES_pycrypto.block_size
-        padding_length = (bs - len(b_plaintext) % bs) or bs
-        b_plaintext += to_bytes(padding_length * chr(padding_length), encoding='ascii', errors='strict')
-
-        # COUNTER.new PARAMETERS
-        # 1) nbits (integer) - Length of the counter, in bits.
-        # 2) initial_value (integer) - initial value of the counter. "iv" from _gen_key_initctr
-
-        ctr = Counter_pycrypto.new(128, initial_value=int(b_iv, 16))
-
-        # AES.new PARAMETERS
-        # 1) AES key, must be either 16, 24, or 32 bytes long -- "key" from _gen_key_initctr
-        # 2) MODE_CTR, is the recommended mode
-        # 3) counter=<CounterObject>
-
-        cipher = AES_pycrypto.new(b_key1, AES_pycrypto.MODE_CTR, counter=ctr)
-
-        # ENCRYPT PADDED DATA
-        b_ciphertext = cipher.encrypt(b_plaintext)
-
-        # COMBINE SALT, DIGEST AND DATA
-        hmac = HMAC_pycrypto.new(b_key2, b_ciphertext, SHA256_pycrypto)
-
-        return to_bytes(hmac.hexdigest(), errors='surrogate_or_strict'), hexlify(b_ciphertext)
-
     @classmethod
     def encrypt(cls, b_plaintext, secret):
         if secret is None:
@@ -1276,8 +1205,6 @@ class VaultAES256:
 
         if HAS_CRYPTOGRAPHY:
             b_hmac, b_ciphertext = cls._encrypt_cryptography(b_plaintext, b_key1, b_key2, b_iv)
-        elif HAS_PYCRYPTO:
-            b_hmac, b_ciphertext = cls._encrypt_pycrypto(b_plaintext, b_key1, b_key2, b_iv)
         else:
             raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in encrypt)')
 
@@ -1310,11 +1237,9 @@ class VaultAES256:
     @staticmethod
     def _is_equal(b_a, b_b):
         """
-        Comparing 2 byte arrrays in constant time
-        to avoid timing attacks.
+        Comparing 2 byte arrays in constant time to avoid timing attacks.
 
-        It would be nice if there was a library for this but
-        hey.
+        It would be nice if there were a library for this but hey.
         """
         if not (isinstance(b_a, binary_type) and isinstance(b_b, binary_type)):
             raise TypeError('_is_equal can only be used to compare two byte strings')
@@ -1331,29 +1256,6 @@ class VaultAES256:
                 result |= ord(b_x) ^ ord(b_y)
         return result == 0
 
-    @classmethod
-    def _decrypt_pycrypto(cls, b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv):
-        # EXIT EARLY IF DIGEST DOESN'T MATCH
-        hmac_decrypt = HMAC_pycrypto.new(b_key2, b_ciphertext, SHA256_pycrypto)
-        if not cls._is_equal(b_crypted_hmac, to_bytes(hmac_decrypt.hexdigest())):
-            return None
-
-        # SET THE COUNTER AND THE CIPHER
-        ctr = Counter_pycrypto.new(128, initial_value=int(b_iv, 16))
-        cipher = AES_pycrypto.new(b_key1, AES_pycrypto.MODE_CTR, counter=ctr)
-
-        # DECRYPT PADDED DATA
-        b_plaintext = cipher.decrypt(b_ciphertext)
-
-        # UNPAD DATA
-        if PY3:
-            padding_length = b_plaintext[-1]
-        else:
-            padding_length = ord(b_plaintext[-1])
-
-        b_plaintext = b_plaintext[:-padding_length]
-        return b_plaintext
-
     @classmethod
     def decrypt(cls, b_vaulttext, secret):
 
@@ -1369,8 +1271,6 @@ class VaultAES256:
 
         if HAS_CRYPTOGRAPHY:
             b_plaintext = cls._decrypt_cryptography(b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv)
-        elif HAS_PYCRYPTO:
-            b_plaintext = cls._decrypt_pycrypto(b_ciphertext, b_crypted_hmac, b_key1, b_key2, b_iv)
         else:
             raise AnsibleError(NEED_CRYPTO_LIBRARY + '(Detected in decrypt)')
 

packaging/debian/rules

@@ -7,11 +7,3 @@ DEB_PYTHON_DISTUTILS_INSTALLDIR_SKEL = /usr/lib/python3/dist-packages/
 
 include /usr/share/cdbs/1/rules/debhelper.mk
 include /usr/share/cdbs/1/class/python-distutils.mk
-
-# dist-packages for the modern distro, site-packages for the older (e.g: Ubuntu 14.04)
-ifeq ($(shell lsb_release -cs), precise)
-  export ANSIBLE_CRYPTO_BACKEND = pycrypto
-endif
-ifeq ($(shell lsb_release -cs), trusty)
-  export ANSIBLE_CRYPTO_BACKEND = pycrypto
-endif

setup.py

@@ -295,48 +295,12 @@ def read_requirements(file_name):
     return reqs
 
 
-PYCRYPTO_DIST = 'pycrypto'
-
-
-def get_crypto_req():
-    """Detect custom crypto from ANSIBLE_CRYPTO_BACKEND env var.
-
-    pycrypto or cryptography. We choose a default but allow the user to
-    override it. This translates into pip install of the sdist deciding what
-    package to install and also the runtime dependencies that pkg_resources
-    knows about.
-    """
-    crypto_backend = os.environ.get('ANSIBLE_CRYPTO_BACKEND', '').strip()
-
-    if crypto_backend == PYCRYPTO_DIST:
-        # Attempt to set version requirements
-        return '%s >= 2.6' % PYCRYPTO_DIST
-
-    return crypto_backend or None
-
-
-def substitute_crypto_to_req(req):
-    """Replace crypto requirements if customized."""
-    crypto_backend = get_crypto_req()
-
-    if crypto_backend is None:
-        return req
-
-    def is_not_crypto(r):
-        CRYPTO_LIBS = PYCRYPTO_DIST, 'cryptography'
-        return not any(r.lower().startswith(c) for c in CRYPTO_LIBS)
-
-    return [r for r in req if is_not_crypto(r)] + [crypto_backend]
-
-
 def get_dynamic_setup_params():
     """Add dynamically calculated setup params to static ones."""
     return {
         # Retrieve the long description from the README
         'long_description': read_file('README.rst'),
-        'install_requires': substitute_crypto_to_req(
-            read_requirements('requirements.txt'),
-        ),
+        'install_requires': read_requirements('requirements.txt'),
     }
 
 

test/integration/targets/vault/runme_change_pip_installed.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-# start by removing pycrypto and cryptography
-
-pip uninstall -y cryptography
-pip uninstall -y pycrypto
-
-./runme.sh
-
-# now just pycrypto
-pip install --user pycrypto
-
-./runme.sh
-
-
-# now just cryptography
-
-pip uninstall -y pycrypto
-pip install --user cryptography
-
-./runme.sh
-
-# now both
-
-pip install --user pycrypto
-
-./runme.sh

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -15,7 +15,6 @@ sphinx <= 2.1.2 ; python_version >= '2.7' # docs team hasn't  tested beyond 2.1.
 rstcheck >=3.3.1  # required for sphinx version >= 1.8
 pygments >= 2.4.0 # Pygments 2.4.0 includes bugfixes for YAML and YAML+Jinja lexers
 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
-pycrypto >= 2.6 # Need features found in 2.6 and greater
 ncclient >= 0.5.2 # Need features added in 0.5.2 and greater
 idna < 2.6, >= 2.5 # linode requires idna < 2.9, >= 2.5, requests requires idna < 2.6, but cryptography will cause the latest version to be installed instead
 paramiko < 2.4.0 ; python_version < '2.7' # paramiko 2.4.0 drops support for python 2.6

test/units/parsing/vault/test_vault.py

@@ -510,24 +510,6 @@ class TestVaultCipherAes256(unittest.TestCase):
         b_key_cryptography = self.vault_cipher._create_key_cryptography(b_password, b_salt, key_length=32, iv_length=16)
         self.assertIsInstance(b_key_cryptography, six.binary_type)
 
-    @pytest.mark.skipif(not vault.HAS_PYCRYPTO, reason='Not testing pycrypto key as pycrypto is not installed')
-    def test_create_key_pycrypto(self):
-        b_password = b'hunter42'
-        b_salt = os.urandom(32)
-
-        b_key_pycrypto = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
-        self.assertIsInstance(b_key_pycrypto, six.binary_type)
-
-    @pytest.mark.skipif(not vault.HAS_PYCRYPTO,
-                        reason='Not comparing cryptography key to pycrypto key as pycrypto is not installed')
-    def test_compare_new_keys(self):
-        b_password = b'hunter42'
-        b_salt = os.urandom(32)
-        b_key_cryptography = self.vault_cipher._create_key_cryptography(b_password, b_salt, key_length=32, iv_length=16)
-
-        b_key_pycrypto = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
-        self.assertEqual(b_key_cryptography, b_key_pycrypto)
-
     def test_create_key_known_cryptography(self):
         b_password = b'hunter42'
 
@@ -543,36 +525,6 @@ class TestVaultCipherAes256(unittest.TestCase):
         self.assertIsInstance(b_key_2, six.binary_type)
         self.assertEqual(b_key_1, b_key_2)
 
-        if vault.HAS_PYCRYPTO:
-            # And again with pycrypto
-            b_key_3 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
-            self.assertIsInstance(b_key_3, six.binary_type)
-
-            # verify we get the same answer
-            # we could potentially run a few iterations of this and time it to see if it's roughly constant time
-            #  and or that it exceeds some minimal time, but that would likely cause unreliable fails, esp in CI
-            b_key_4 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
-            self.assertIsInstance(b_key_4, six.binary_type)
-            self.assertEqual(b_key_3, b_key_4)
-            self.assertEqual(b_key_1, b_key_4)
-
-    @pytest.mark.skipif(not vault.HAS_PYCRYPTO,
-                        reason='PyCrypto is not installed')
-    def test_create_key_known_pycrypto(self):
-        b_password = b'hunter42'
-
-        # A fixed salt
-        b_salt = b'q' * 32  # q is the most random letter.
-        b_key_3 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
-        self.assertIsInstance(b_key_3, six.binary_type)
-
-        # verify we get the same answer
-        # we could potentially run a few iterations of this and time it to see if it's roughly constant time
-        #  and or that it exceeds some minimal time, but that would likely cause unreliable fails, esp in CI
-        b_key_4 = self.vault_cipher._create_key_pycrypto(b_password, b_salt, key_length=32, iv_length=16)
-        self.assertIsInstance(b_key_4, six.binary_type)
-        self.assertEqual(b_key_3, b_key_4)
-
     def test_is_equal_is_equal(self):
         self.assertTrue(self.vault_cipher._is_equal(b'abcdefghijklmnopqrstuvwxyz', b'abcdefghijklmnopqrstuvwxyz'))
 
@@ -606,19 +558,6 @@ class TestVaultCipherAes256(unittest.TestCase):
         self.assertRaises(TypeError, self.vault_cipher._is_equal, b"blue fish", 2)
 
 
-@pytest.mark.skipif(not vault.HAS_PYCRYPTO,
-                    reason="Skipping Pycrypto tests because pycrypto is not installed")
-class TestVaultCipherAes256PyCrypto(TestVaultCipherAes256):
-    def setUp(self):
-        self.has_cryptography = vault.HAS_CRYPTOGRAPHY
-        vault.HAS_CRYPTOGRAPHY = False
-        super(TestVaultCipherAes256PyCrypto, self).setUp()
-
-    def tearDown(self):
-        vault.HAS_CRYPTOGRAPHY = self.has_cryptography
-        super(TestVaultCipherAes256PyCrypto, self).tearDown()
-
-
 class TestMatchSecrets(unittest.TestCase):
     def test_empty_tuple(self):
         secrets = [tuple()]
@@ -929,16 +868,3 @@ class TestVaultLib(unittest.TestCase):
         plaintext = u"ansible"
         self.v.encrypt(plaintext)
         self.assertEqual(self.v.cipher_name, "AES256")
-
-
-@pytest.mark.skipif(not vault.HAS_PYCRYPTO,
-                    reason="Skipping Pycrypto tests because pycrypto is not installed")
-class TestVaultLibPyCrypto(TestVaultLib):
-    def setUp(self):
-        self.has_cryptography = vault.HAS_CRYPTOGRAPHY
-        vault.HAS_CRYPTOGRAPHY = False
-        super(TestVaultLibPyCrypto, self).setUp()
-
-    def tearDown(self):
-        vault.HAS_CRYPTOGRAPHY = self.has_cryptography
-        super(TestVaultLibPyCrypto, self).tearDown()

test/units/parsing/vault/test_vault_editor.py

@@ -519,16 +519,3 @@ class TestVaultEditor(unittest.TestCase):
 
         res = ve._real_path(file_link_path)
         self.assertEqual(res, file_path)
-
-
-@pytest.mark.skipif(not vault.HAS_PYCRYPTO,
-                    reason="Skipping pycrypto tests because pycrypto is not installed")
-class TestVaultEditorPyCrypto(unittest.TestCase):
-    def setUp(self):
-        self.has_cryptography = vault.HAS_CRYPTOGRAPHY
-        vault.HAS_CRYPTOGRAPHY = False
-        super(TestVaultEditorPyCrypto, self).setUp()
-
-    def tearDown(self):
-        vault.HAS_CRYPTOGRAPHY = self.has_cryptography
-        super(TestVaultEditorPyCrypto, self).tearDown()

test/units/requirements.txt

@@ -1,4 +1,3 @@
-pycrypto ; python_version < '3.10'  # pycrypto is not compatible with Python 3.10
 passlib
 pywinrm
 pytz