From 9d65e122ff62b31133bce7148921f6aea9b6a394 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 8 Feb 2023 09:27:59 -0800 Subject: [PATCH] Fix detection of available hashlib algorithms (#79946) * Fix detection of available hashlib algorithms Detection of hashlib algorithms now works on Python 3.x. The new implementation works on Python 2.7 and later. Test coverage is provided by both integration and unit tests. * Add additional details about hashlib in docs --- changelogs/fragments/hashlib-algorithms.yml | 7 ++ lib/ansible/module_utils/basic.py | 66 ++++++++++--------- lib/ansible/modules/get_url.py | 4 +- .../targets/get_url/tasks/hashlib.yml | 20 ++++++ .../targets/get_url/tasks/main.yml | 2 + .../test_get_available_hash_algorithms.py | 60 +++++++++++++++++ 6 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 changelogs/fragments/hashlib-algorithms.yml create mode 100644 test/integration/targets/get_url/tasks/hashlib.yml create mode 100644 test/units/module_utils/basic/test_get_available_hash_algorithms.py diff --git a/changelogs/fragments/hashlib-algorithms.yml b/changelogs/fragments/hashlib-algorithms.yml new file mode 100644 index 00000000000..d8dfa0e0532 --- /dev/null +++ b/changelogs/fragments/hashlib-algorithms.yml @@ -0,0 +1,7 @@ +bugfixes: + - get_url module - Removed out-of-date documentation stating that ``hashlib`` is a third-party library. + - get_url module - Added a documentation reference to ``hashlib`` regarding algorithms, + as well as a note about ``md5`` support on systems running in FIPS compliant mode. + - module_utils/basic.py - Fix detection of available hashing algorithms on Python 3.x. + All supported algorithms are now available instead of being limited to a hard-coded list. + This affects modules such as ``get_url`` which accept an arbitrary checksum algorithm. diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index aa68cf4adb7..42df052a03c 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -101,42 +101,48 @@ from ansible.module_utils.common.text.formatters import ( SIZE_RANGES, ) +import hashlib + + +def _get_available_hash_algorithms(): + """Return a dictionary of available hash function names and their associated function.""" + try: + # Algorithms available in Python 2.7.9+ and Python 3.2+ + # https://docs.python.org/2.7/library/hashlib.html#hashlib.algorithms_available + # https://docs.python.org/3.2/library/hashlib.html#hashlib.algorithms_available + algorithm_names = hashlib.algorithms_available + except AttributeError: + # Algorithms in Python 2.7.x (used only for Python 2.7.0 through 2.7.8) + # https://docs.python.org/2.7/library/hashlib.html#hashlib.hashlib.algorithms + algorithm_names = set(hashlib.algorithms) + + algorithms = {} + + for algorithm_name in algorithm_names: + algorithm_func = getattr(hashlib, algorithm_name, None) + + if algorithm_func: + try: + # Make sure the algorithm is actually available for use. + # Not all algorithms listed as available are actually usable. + # For example, md5 is not available in FIPS mode. + algorithm_func() + except Exception: + pass + else: + algorithms[algorithm_name] = algorithm_func + + return algorithms + + +AVAILABLE_HASH_ALGORITHMS = _get_available_hash_algorithms() + try: from ansible.module_utils.common._json_compat import json except ImportError as e: print('\n{{"msg": "Error: ansible requires the stdlib json: {0}", "failed": true}}'.format(to_native(e))) sys.exit(1) - -AVAILABLE_HASH_ALGORITHMS = dict() -try: - import hashlib - - # python 2.7.9+ and 2.7.0+ - for attribute in ('available_algorithms', 'algorithms'): - algorithms = getattr(hashlib, attribute, None) - if algorithms: - break - if algorithms is None: - # python 2.5+ - algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') - for algorithm in algorithms: - AVAILABLE_HASH_ALGORITHMS[algorithm] = getattr(hashlib, algorithm) - - # we may have been able to import md5 but it could still not be available - try: - hashlib.md5() - except ValueError: - AVAILABLE_HASH_ALGORITHMS.pop('md5', None) -except Exception: - import sha - AVAILABLE_HASH_ALGORITHMS = {'sha1': sha.sha} - try: - import md5 - AVAILABLE_HASH_ALGORITHMS['md5'] = md5.md5 - except Exception: - pass - from ansible.module_utils.six.moves.collections_abc import ( KeysView, Mapping, MutableMapping, diff --git a/lib/ansible/modules/get_url.py b/lib/ansible/modules/get_url.py index eec24241d30..2d85d4da363 100644 --- a/lib/ansible/modules/get_url.py +++ b/lib/ansible/modules/get_url.py @@ -92,7 +92,9 @@ options: checksum="sha256:http://example.com/path/sha256sum.txt"' - If you worry about portability, only the sha1 algorithm is available on all platforms and python versions. - - The third party hashlib library can be installed for access to additional algorithms. + - The Python ``hashlib`` module is responsible for providing the available algorithms. + The choices vary based on Python version and OpenSSL version. + - On systems running in FIPS compliant mode, the ``md5`` algorithm may be unavailable. - Additionally, if a checksum is passed to this parameter, and the file exist under the C(dest) location, the I(destination_checksum) would be calculated, and if checksum equals I(destination_checksum), the file download would be skipped diff --git a/test/integration/targets/get_url/tasks/hashlib.yml b/test/integration/targets/get_url/tasks/hashlib.yml new file mode 100644 index 00000000000..cc50ad727b5 --- /dev/null +++ b/test/integration/targets/get_url/tasks/hashlib.yml @@ -0,0 +1,20 @@ +- name: "Set hash algorithms to test" + set_fact: + algorithms: + sha256: b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006 + sha384: 298553d31087fd3f6659801d2e5cde3ff63fad609dc50ad8e194dde80bfb8a084edfa761f025928448f39d720fce55f2 + sha512: 69b589f7775fe04244e8a9db216a3c91db1680baa33ccd0c317b8d7f0334433f7362d00c8080b3365bf08d532956ba01dbebc497b51ced8f8b05a44a66b854bf + sha3_256: 64e5ea73a2f799f35abd0b1242df5e70c84248c9883f89343d4cd5f6d493a139 + sha3_384: 976edebcb496ad8be0f7fa4411cc8e2404e7e65f1088fabf7be44484458726c61d4985bdaeff8700008ed1670a9b982d + sha3_512: f8cca1d98e750e2c2ab44954dc9f1b6e8e35ace71ffcc1cd21c7770eb8eccfbd77d40b2d7d145120efbbb781599294ccc6148c6cda1aa66146363e5fdddd2336 + +- name: "Verify various checksum algorithms work" + get_url: + url: 'http://localhost:{{ http_port }}/27617.txt' # content is 'ptux' + dest: '{{ remote_tmp_dir }}/27617.{{ algorithm }}.txt' + checksum: "{{ algorithm }}:{{ algorithms[algorithm] }}" + force: yes + loop: "{{ algorithms.keys() }}" + loop_control: + loop_var: algorithm + when: ansible_python_version.startswith('3.') or not algorithm.startswith('sha3_') diff --git a/test/integration/targets/get_url/tasks/main.yml b/test/integration/targets/get_url/tasks/main.yml index 09814c709e2..c26cc08b188 100644 --- a/test/integration/targets/get_url/tasks/main.yml +++ b/test/integration/targets/get_url/tasks/main.yml @@ -398,6 +398,8 @@ port: '{{ http_port }}' state: started +- include_tasks: hashlib.yml + - name: download src with sha1 checksum url in check mode get_url: url: 'http://localhost:{{ http_port }}/27617.txt' diff --git a/test/units/module_utils/basic/test_get_available_hash_algorithms.py b/test/units/module_utils/basic/test_get_available_hash_algorithms.py new file mode 100644 index 00000000000..d60f34cc8f5 --- /dev/null +++ b/test/units/module_utils/basic/test_get_available_hash_algorithms.py @@ -0,0 +1,60 @@ +"""Unit tests to provide coverage not easily obtained from integration tests.""" + +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +import hashlib +import sys + +import pytest + +from ansible.module_utils.basic import _get_available_hash_algorithms + + +@pytest.mark.skipif(sys.version_info < (2, 7, 9), reason="requires Python 2.7.9 or later") +def test_unavailable_algorithm(mocker): + """Simulate an available algorithm that isn't.""" + expected_algorithms = {'sha256', 'sha512'} # guaranteed to be available + + mocker.patch('hashlib.algorithms_available', expected_algorithms | {'not_actually_available'}) + + available_algorithms = _get_available_hash_algorithms() + + assert sorted(expected_algorithms) == sorted(available_algorithms) + + +@pytest.mark.skipif(sys.version_info < (2, 7, 9), reason="requires Python 2.7.9 or later") +def test_fips_mode(mocker): + """Simulate running in FIPS mode on Python 2.7.9 or later.""" + expected_algorithms = {'sha256', 'sha512'} # guaranteed to be available + + mocker.patch('hashlib.algorithms_available', expected_algorithms | {'md5'}) + mocker.patch('hashlib.md5').side_effect = ValueError() # using md5 in FIPS mode raises a ValueError + + available_algorithms = _get_available_hash_algorithms() + + assert sorted(expected_algorithms) == sorted(available_algorithms) + + +@pytest.mark.skipif(sys.version_info < (2, 7, 9) or sys.version_info[:2] != (2, 7), reason="requires Python 2.7 (2.7.9 or later)") +def test_legacy_python(mocker): + """Simulate behavior on Python 2.7.x earlier than Python 2.7.9.""" + expected_algorithms = {'sha256', 'sha512'} # guaranteed to be available + + # This attribute is exclusive to Python 2.7. + # Since `hashlib.algorithms_available` is used on Python 2.7.9 and later, only Python 2.7.0 through 2.7.8 utilize this attribute. + mocker.patch('hashlib.algorithms', expected_algorithms) + + saved_algorithms = hashlib.algorithms_available + + # Make sure that this attribute is unavailable, to simulate running on Python 2.7.0 through 2.7.8. + # It will be restored immediately after performing the test. + del hashlib.algorithms_available + + try: + available_algorithms = _get_available_hash_algorithms() + finally: + hashlib.algorithms_available = saved_algorithms + + assert sorted(expected_algorithms) == sorted(available_algorithms)