diff --git a/changelogs/fragments/hashlib-algorithms.yml b/changelogs/fragments/hashlib-algorithms.yml new file mode 100644 index 00000000000..d8dfa0e0532 --- /dev/null +++ b/changelogs/fragments/hashlib-algorithms.yml @@ -0,0 +1,7 @@ +bugfixes: + - get_url module - Removed out-of-date documentation stating that ``hashlib`` is a third-party library. + - get_url module - Added a documentation reference to ``hashlib`` regarding algorithms, + as well as a note about ``md5`` support on systems running in FIPS compliant mode. + - module_utils/basic.py - Fix detection of available hashing algorithms on Python 3.x. + All supported algorithms are now available instead of being limited to a hard-coded list. + This affects modules such as ``get_url`` which accept an arbitrary checksum algorithm. diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index aa68cf4adb7..42df052a03c 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -101,42 +101,48 @@ from ansible.module_utils.common.text.formatters import ( SIZE_RANGES, ) +import hashlib + + +def _get_available_hash_algorithms(): + """Return a dictionary of available hash function names and their associated function.""" + try: + # Algorithms available in Python 2.7.9+ and Python 3.2+ + # https://docs.python.org/2.7/library/hashlib.html#hashlib.algorithms_available + # https://docs.python.org/3.2/library/hashlib.html#hashlib.algorithms_available + algorithm_names = hashlib.algorithms_available + except AttributeError: + # Algorithms in Python 2.7.x (used only for Python 2.7.0 through 2.7.8) + # https://docs.python.org/2.7/library/hashlib.html#hashlib.hashlib.algorithms + algorithm_names = set(hashlib.algorithms) + + algorithms = {} + + for algorithm_name in algorithm_names: + algorithm_func = getattr(hashlib, algorithm_name, None) + + if algorithm_func: + try: + # Make sure the algorithm is actually available for use. + # Not all algorithms listed as available are actually usable. + # For example, md5 is not available in FIPS mode. + algorithm_func() + except Exception: + pass + else: + algorithms[algorithm_name] = algorithm_func + + return algorithms + + +AVAILABLE_HASH_ALGORITHMS = _get_available_hash_algorithms() + try: from ansible.module_utils.common._json_compat import json except ImportError as e: print('\n{{"msg": "Error: ansible requires the stdlib json: {0}", "failed": true}}'.format(to_native(e))) sys.exit(1) - -AVAILABLE_HASH_ALGORITHMS = dict() -try: - import hashlib - - # python 2.7.9+ and 2.7.0+ - for attribute in ('available_algorithms', 'algorithms'): - algorithms = getattr(hashlib, attribute, None) - if algorithms: - break - if algorithms is None: - # python 2.5+ - algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') - for algorithm in algorithms: - AVAILABLE_HASH_ALGORITHMS[algorithm] = getattr(hashlib, algorithm) - - # we may have been able to import md5 but it could still not be available - try: - hashlib.md5() - except ValueError: - AVAILABLE_HASH_ALGORITHMS.pop('md5', None) -except Exception: - import sha - AVAILABLE_HASH_ALGORITHMS = {'sha1': sha.sha} - try: - import md5 - AVAILABLE_HASH_ALGORITHMS['md5'] = md5.md5 - except Exception: - pass - from ansible.module_utils.six.moves.collections_abc import ( KeysView, Mapping, MutableMapping, diff --git a/lib/ansible/modules/get_url.py b/lib/ansible/modules/get_url.py index eec24241d30..2d85d4da363 100644 --- a/lib/ansible/modules/get_url.py +++ b/lib/ansible/modules/get_url.py @@ -92,7 +92,9 @@ options: checksum="sha256:http://example.com/path/sha256sum.txt"' - If you worry about portability, only the sha1 algorithm is available on all platforms and python versions. - - The third party hashlib library can be installed for access to additional algorithms. + - The Python ``hashlib`` module is responsible for providing the available algorithms. + The choices vary based on Python version and OpenSSL version. + - On systems running in FIPS compliant mode, the ``md5`` algorithm may be unavailable. - Additionally, if a checksum is passed to this parameter, and the file exist under the C(dest) location, the I(destination_checksum) would be calculated, and if checksum equals I(destination_checksum), the file download would be skipped diff --git a/test/integration/targets/get_url/tasks/hashlib.yml b/test/integration/targets/get_url/tasks/hashlib.yml new file mode 100644 index 00000000000..cc50ad727b5 --- /dev/null +++ b/test/integration/targets/get_url/tasks/hashlib.yml @@ -0,0 +1,20 @@ +- name: "Set hash algorithms to test" + set_fact: + algorithms: + sha256: b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006 + sha384: 298553d31087fd3f6659801d2e5cde3ff63fad609dc50ad8e194dde80bfb8a084edfa761f025928448f39d720fce55f2 + sha512: 69b589f7775fe04244e8a9db216a3c91db1680baa33ccd0c317b8d7f0334433f7362d00c8080b3365bf08d532956ba01dbebc497b51ced8f8b05a44a66b854bf + sha3_256: 64e5ea73a2f799f35abd0b1242df5e70c84248c9883f89343d4cd5f6d493a139 + sha3_384: 976edebcb496ad8be0f7fa4411cc8e2404e7e65f1088fabf7be44484458726c61d4985bdaeff8700008ed1670a9b982d + sha3_512: f8cca1d98e750e2c2ab44954dc9f1b6e8e35ace71ffcc1cd21c7770eb8eccfbd77d40b2d7d145120efbbb781599294ccc6148c6cda1aa66146363e5fdddd2336 + +- name: "Verify various checksum algorithms work" + get_url: + url: 'http://localhost:{{ http_port }}/27617.txt' # content is 'ptux' + dest: '{{ remote_tmp_dir }}/27617.{{ algorithm }}.txt' + checksum: "{{ algorithm }}:{{ algorithms[algorithm] }}" + force: yes + loop: "{{ algorithms.keys() }}" + loop_control: + loop_var: algorithm + when: ansible_python_version.startswith('3.') or not algorithm.startswith('sha3_') diff --git a/test/integration/targets/get_url/tasks/main.yml b/test/integration/targets/get_url/tasks/main.yml index 09814c709e2..c26cc08b188 100644 --- a/test/integration/targets/get_url/tasks/main.yml +++ b/test/integration/targets/get_url/tasks/main.yml @@ -398,6 +398,8 @@ port: '{{ http_port }}' state: started +- include_tasks: hashlib.yml + - name: download src with sha1 checksum url in check mode get_url: url: 'http://localhost:{{ http_port }}/27617.txt' diff --git a/test/units/module_utils/basic/test_get_available_hash_algorithms.py b/test/units/module_utils/basic/test_get_available_hash_algorithms.py new file mode 100644 index 00000000000..d60f34cc8f5 --- /dev/null +++ b/test/units/module_utils/basic/test_get_available_hash_algorithms.py @@ -0,0 +1,60 @@ +"""Unit tests to provide coverage not easily obtained from integration tests.""" + +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +import hashlib +import sys + +import pytest + +from ansible.module_utils.basic import _get_available_hash_algorithms + + +@pytest.mark.skipif(sys.version_info < (2, 7, 9), reason="requires Python 2.7.9 or later") +def test_unavailable_algorithm(mocker): + """Simulate an available algorithm that isn't.""" + expected_algorithms = {'sha256', 'sha512'} # guaranteed to be available + + mocker.patch('hashlib.algorithms_available', expected_algorithms | {'not_actually_available'}) + + available_algorithms = _get_available_hash_algorithms() + + assert sorted(expected_algorithms) == sorted(available_algorithms) + + +@pytest.mark.skipif(sys.version_info < (2, 7, 9), reason="requires Python 2.7.9 or later") +def test_fips_mode(mocker): + """Simulate running in FIPS mode on Python 2.7.9 or later.""" + expected_algorithms = {'sha256', 'sha512'} # guaranteed to be available + + mocker.patch('hashlib.algorithms_available', expected_algorithms | {'md5'}) + mocker.patch('hashlib.md5').side_effect = ValueError() # using md5 in FIPS mode raises a ValueError + + available_algorithms = _get_available_hash_algorithms() + + assert sorted(expected_algorithms) == sorted(available_algorithms) + + +@pytest.mark.skipif(sys.version_info < (2, 7, 9) or sys.version_info[:2] != (2, 7), reason="requires Python 2.7 (2.7.9 or later)") +def test_legacy_python(mocker): + """Simulate behavior on Python 2.7.x earlier than Python 2.7.9.""" + expected_algorithms = {'sha256', 'sha512'} # guaranteed to be available + + # This attribute is exclusive to Python 2.7. + # Since `hashlib.algorithms_available` is used on Python 2.7.9 and later, only Python 2.7.0 through 2.7.8 utilize this attribute. + mocker.patch('hashlib.algorithms', expected_algorithms) + + saved_algorithms = hashlib.algorithms_available + + # Make sure that this attribute is unavailable, to simulate running on Python 2.7.0 through 2.7.8. + # It will be restored immediately after performing the test. + del hashlib.algorithms_available + + try: + available_algorithms = _get_available_hash_algorithms() + finally: + hashlib.algorithms_available = saved_algorithms + + assert sorted(expected_algorithms) == sorted(available_algorithms)