Fix detection of available hashlib algorithms (#79946)

* Fix detection of available hashlib algorithms

Detection of hashlib algorithms now works on Python 3.x.
The new implementation works on Python 2.7 and later.

Test coverage is provided by both integration and unit tests.

* Add additional details about hashlib in docs

changelogs/fragments/hashlib-algorithms.yml

@@ -0,0 +1,7 @@
+bugfixes:
+  - get_url module - Removed out-of-date documentation stating that ``hashlib`` is a third-party library.
+  - get_url module - Added a documentation reference to ``hashlib`` regarding algorithms,
+                     as well as a note about ``md5`` support on systems running in FIPS compliant mode.
+  - module_utils/basic.py - Fix detection of available hashing algorithms on Python 3.x.
+                            All supported algorithms are now available instead of being limited to a hard-coded list.
+                            This affects modules such as ``get_url`` which accept an arbitrary checksum algorithm.

lib/ansible/module_utils/basic.py

@@ -101,41 +101,47 @@ from ansible.module_utils.common.text.formatters import (
     SIZE_RANGES,
 )
 
-try:
-    from ansible.module_utils.common._json_compat import json
-except ImportError as e:
-    print('\n{{"msg": "Error: ansible requires the stdlib json: {0}", "failed": true}}'.format(to_native(e)))
-    sys.exit(1)
+import hashlib
 
 
-AVAILABLE_HASH_ALGORITHMS = dict()
-try:
-    import hashlib
+def _get_available_hash_algorithms():
+    """Return a dictionary of available hash function names and their associated function."""
+    try:
+        # Algorithms available in Python 2.7.9+ and Python 3.2+
+        # https://docs.python.org/2.7/library/hashlib.html#hashlib.algorithms_available
+        # https://docs.python.org/3.2/library/hashlib.html#hashlib.algorithms_available
+        algorithm_names = hashlib.algorithms_available
+    except AttributeError:
+        # Algorithms in Python 2.7.x (used only for Python 2.7.0 through 2.7.8)
+        # https://docs.python.org/2.7/library/hashlib.html#hashlib.hashlib.algorithms
+        algorithm_names = set(hashlib.algorithms)
 
-    # python 2.7.9+ and 2.7.0+
-    for attribute in ('available_algorithms', 'algorithms'):
-        algorithms = getattr(hashlib, attribute, None)
-        if algorithms:
-            break
-    if algorithms is None:
-        # python 2.5+
-        algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
-    for algorithm in algorithms:
-        AVAILABLE_HASH_ALGORITHMS[algorithm] = getattr(hashlib, algorithm)
+    algorithms = {}
 
-    # we may have been able to import md5 but it could still not be available
-    try:
-        hashlib.md5()
-    except ValueError:
-        AVAILABLE_HASH_ALGORITHMS.pop('md5', None)
-except Exception:
-    import sha
-    AVAILABLE_HASH_ALGORITHMS = {'sha1': sha.sha}
+    for algorithm_name in algorithm_names:
+        algorithm_func = getattr(hashlib, algorithm_name, None)
+
+        if algorithm_func:
             try:
-        import md5
-        AVAILABLE_HASH_ALGORITHMS['md5'] = md5.md5
+                # Make sure the algorithm is actually available for use.
+                # Not all algorithms listed as available are actually usable.
+                # For example, md5 is not available in FIPS mode.
+                algorithm_func()
             except Exception:
                 pass
+            else:
+                algorithms[algorithm_name] = algorithm_func
+
+    return algorithms
+
+
+AVAILABLE_HASH_ALGORITHMS = _get_available_hash_algorithms()
+
+try:
+    from ansible.module_utils.common._json_compat import json
+except ImportError as e:
+    print('\n{{"msg": "Error: ansible requires the stdlib json: {0}", "failed": true}}'.format(to_native(e)))
+    sys.exit(1)
 
 from ansible.module_utils.six.moves.collections_abc import (
     KeysView,

lib/ansible/modules/get_url.py

@@ -92,7 +92,9 @@ options:
         checksum="sha256:http://example.com/path/sha256sum.txt"'
       - If you worry about portability, only the sha1 algorithm is available
         on all platforms and python versions.
-      - The third party hashlib library can be installed for access to additional algorithms.
+      - The Python ``hashlib`` module is responsible for providing the available algorithms.
+        The choices vary based on Python version and OpenSSL version.
+      - On systems running in FIPS compliant mode, the ``md5`` algorithm may be unavailable.
       - Additionally, if a checksum is passed to this parameter, and the file exist under
         the C(dest) location, the I(destination_checksum) would be calculated, and if
         checksum equals I(destination_checksum), the file download would be skipped

test/integration/targets/get_url/tasks/hashlib.yml

@@ -0,0 +1,20 @@
+- name: "Set hash algorithms to test"
+  set_fact:
+    algorithms:
+      sha256: b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006
+      sha384: 298553d31087fd3f6659801d2e5cde3ff63fad609dc50ad8e194dde80bfb8a084edfa761f025928448f39d720fce55f2
+      sha512: 69b589f7775fe04244e8a9db216a3c91db1680baa33ccd0c317b8d7f0334433f7362d00c8080b3365bf08d532956ba01dbebc497b51ced8f8b05a44a66b854bf
+      sha3_256: 64e5ea73a2f799f35abd0b1242df5e70c84248c9883f89343d4cd5f6d493a139
+      sha3_384: 976edebcb496ad8be0f7fa4411cc8e2404e7e65f1088fabf7be44484458726c61d4985bdaeff8700008ed1670a9b982d
+      sha3_512: f8cca1d98e750e2c2ab44954dc9f1b6e8e35ace71ffcc1cd21c7770eb8eccfbd77d40b2d7d145120efbbb781599294ccc6148c6cda1aa66146363e5fdddd2336
+
+- name: "Verify various checksum algorithms work"
+  get_url:
+    url: 'http://localhost:{{ http_port }}/27617.txt'  # content is 'ptux'
+    dest: '{{ remote_tmp_dir }}/27617.{{ algorithm }}.txt'
+    checksum: "{{ algorithm }}:{{ algorithms[algorithm] }}"
+    force: yes
+  loop: "{{ algorithms.keys() }}"
+  loop_control:
+    loop_var: algorithm
+  when: ansible_python_version.startswith('3.') or not algorithm.startswith('sha3_')

test/integration/targets/get_url/tasks/main.yml

@@ -398,6 +398,8 @@
     port: '{{ http_port }}'
     state: started
 
+- include_tasks: hashlib.yml
+
 - name: download src with sha1 checksum url in check mode
   get_url:
     url: 'http://localhost:{{ http_port }}/27617.txt'

test/units/module_utils/basic/test_get_available_hash_algorithms.py

@@ -0,0 +1,60 @@
+"""Unit tests to provide coverage not easily obtained from integration tests."""
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+import hashlib
+import sys
+
+import pytest
+
+from ansible.module_utils.basic import _get_available_hash_algorithms
+
+
+@pytest.mark.skipif(sys.version_info < (2, 7, 9), reason="requires Python 2.7.9 or later")
+def test_unavailable_algorithm(mocker):
+    """Simulate an available algorithm that isn't."""
+    expected_algorithms = {'sha256', 'sha512'}  # guaranteed to be available
+
+    mocker.patch('hashlib.algorithms_available', expected_algorithms | {'not_actually_available'})
+
+    available_algorithms = _get_available_hash_algorithms()
+
+    assert sorted(expected_algorithms) == sorted(available_algorithms)
+
+
+@pytest.mark.skipif(sys.version_info < (2, 7, 9), reason="requires Python 2.7.9 or later")
+def test_fips_mode(mocker):
+    """Simulate running in FIPS mode on Python 2.7.9 or later."""
+    expected_algorithms = {'sha256', 'sha512'}  # guaranteed to be available
+
+    mocker.patch('hashlib.algorithms_available', expected_algorithms | {'md5'})
+    mocker.patch('hashlib.md5').side_effect = ValueError()  # using md5 in FIPS mode raises a ValueError
+
+    available_algorithms = _get_available_hash_algorithms()
+
+    assert sorted(expected_algorithms) == sorted(available_algorithms)
+
+
+@pytest.mark.skipif(sys.version_info < (2, 7, 9) or sys.version_info[:2] != (2, 7), reason="requires Python 2.7 (2.7.9 or later)")
+def test_legacy_python(mocker):
+    """Simulate behavior on Python 2.7.x earlier than Python 2.7.9."""
+    expected_algorithms = {'sha256', 'sha512'}  # guaranteed to be available
+
+    # This attribute is exclusive to Python 2.7.
+    # Since `hashlib.algorithms_available` is used on Python 2.7.9 and later, only Python 2.7.0 through 2.7.8 utilize this attribute.
+    mocker.patch('hashlib.algorithms', expected_algorithms)
+
+    saved_algorithms = hashlib.algorithms_available
+
+    # Make sure that this attribute is unavailable, to simulate running on Python 2.7.0 through 2.7.8.
+    # It will be restored immediately after performing the test.
+    del hashlib.algorithms_available
+
+    try:
+        available_algorithms = _get_available_hash_algorithms()
+    finally:
+        hashlib.algorithms_available = saved_algorithms
+
+    assert sorted(expected_algorithms) == sorted(available_algorithms)