From 3ee1694dfe6c203c49306e67700b3cca3db89534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Moser?= Date: Mon, 12 Jul 2021 20:48:27 +0200 Subject: [PATCH] [2.10] get_url: Fix checksum binary validation (#74674) * get_url: Handle same SHA sum for downloaded files (#71435) Fixes: #71420 Signed-off-by: Abhijeet Kasurde (cherry picked from commit 159544610e5a925d589198febaa5198778ea51c0) * modules: get_url: Fix checksum binary validation (#74502) From the sha512sum man page: ... The default mode is to print a line with checksum, a character indicating type ('*' for binary, ' ' for text), and name for each FILE. (cherry picked from commit 403a5d147df7261537d5fb37c2474858b96de174) Co-authored-by: Abhijeet Kasurde --- changelogs/fragments/71420_get_url.yml | 2 + .../74502-get_url-filx-checksum-binary.yml | 2 + lib/ansible/modules/get_url.py | 15 ++- .../targets/get_url/tasks/main.yml | 95 +++++++++++++++++++ 4 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 changelogs/fragments/71420_get_url.yml create mode 100644 changelogs/fragments/74502-get_url-filx-checksum-binary.yml diff --git a/changelogs/fragments/71420_get_url.yml b/changelogs/fragments/71420_get_url.yml new file mode 100644 index 00000000000..3083f66a417 --- /dev/null +++ b/changelogs/fragments/71420_get_url.yml @@ -0,0 +1,2 @@ +minor_changes: +- get_url - handle same SHA sum for checksum file (https://github.com/ansible/ansible/issues/71420). diff --git a/changelogs/fragments/74502-get_url-filx-checksum-binary.yml b/changelogs/fragments/74502-get_url-filx-checksum-binary.yml new file mode 100644 index 00000000000..76ee656bba9 --- /dev/null +++ b/changelogs/fragments/74502-get_url-filx-checksum-binary.yml @@ -0,0 +1,2 @@ +bugfixes: + - get_url - Fixed checksum validation for binary files (leading asterisk) in checksum files (https://github.com/ansible/ansible/pull/74502). diff --git a/lib/ansible/modules/get_url.py b/lib/ansible/modules/get_url.py index 501704034e0..50fcdca055d 100644 --- a/lib/ansible/modules/get_url.py +++ b/lib/ansible/modules/get_url.py @@ -502,16 +502,23 @@ def main(): with open(checksum_tmpsrc) as f: lines = [line.rstrip('\n') for line in f] os.remove(checksum_tmpsrc) - checksum_map = {} + checksum_map = [] for line in lines: - parts = line.split(None, 1) + # Split by one whitespace to keep the leading type char ' ' (whitespace) for text and '*' for binary + parts = line.split(" ", 1) if len(parts) == 2: - checksum_map[parts[0]] = parts[1] + # Remove the leading type char, we expect + if parts[1].startswith((" ", "*",)): + parts[1] = parts[1][1:] + + # Append checksum and path without potential leading './' + checksum_map.append((parts[0], parts[1].lstrip("./"))) + filename = url_filename(url) # Look through each line in the checksum file for a hash corresponding to # the filename in the url, returning the first hash that is found. - for cksum in (s for (s, f) in checksum_map.items() if f.strip('./') == filename): + for cksum in (s for (s, f) in checksum_map if f == filename): checksum = cksum break else: diff --git a/test/integration/targets/get_url/tasks/main.yml b/test/integration/targets/get_url/tasks/main.yml index 6adb3db74c7..f39c92d3824 100644 --- a/test/integration/targets/get_url/tasks/main.yml +++ b/test/integration/targets/get_url/tasks/main.yml @@ -335,11 +335,17 @@ dest: '{{ files_dir }}/27617.txt' content: "ptux" +- name: create duplicate src file + copy: + dest: '{{ files_dir }}/71420.txt' + content: "ptux" + - name: create sha1 checksum file of src copy: dest: '{{ files_dir }}/sha1sum.txt' content: | a97e6837f60cec6da4491bab387296bbcd72bdba 27617.txt + a97e6837f60cec6da4491bab387296bbcd72bdba 71420.txt 3911340502960ca33aece01129234460bfeb2791 not_target1.txt 1b4b6adf30992cedb0f6edefd6478ff0a593b2e4 not_target2.txt @@ -348,6 +354,7 @@ dest: '{{ files_dir }}/sha256sum.txt' content: | b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. 27617.txt + b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. 71420.txt 30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 not_target1.txt d0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b not_target2.txt @@ -356,9 +363,19 @@ dest: '{{ files_dir }}/sha256sum_with_dot.txt' content: | b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. ./27617.txt + b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. ./71420.txt 30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 ./not_target1.txt d0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b ./not_target2.txt +- name: create sha256 checksum file of src with a * leading path + copy: + dest: '{{ files_dir }}/sha256sum_with_asterisk.txt' + content: | + b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. *27617.txt + b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. *71420.txt + 30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 *not_target1.txt + d0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b *not_target2.txt + - copy: src: "testserver.py" dest: "{{ remote_tmp_dir }}/testserver.py" @@ -407,6 +424,17 @@ path: "{{ remote_tmp_dir }}/27617sha256_with_dot.txt" register: stat_result_sha256_with_dot +- name: download src with sha256 checksum url with asterisk leading paths + get_url: + url: 'http://localhost:{{ http_port }}/27617.txt' + dest: '{{ remote_tmp_dir }}/27617sha256_with_asterisk.txt' + checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum_with_asterisk.txt' + register: result_sha256_with_asterisk + +- stat: + path: "{{ remote_tmp_dir }}/27617sha256_with_asterisk.txt" + register: stat_result_sha256_with_asterisk + - name: download src with sha256 checksum url with file scheme get_url: url: 'http://localhost:{{ http_port }}/27617.txt' @@ -418,17 +446,84 @@ path: "{{ remote_tmp_dir }}/27617sha256_with_dot.txt" register: stat_result_sha256_with_file_scheme +- name: download 71420.txt with sha1 checksum url + get_url: + url: 'http://localhost:{{ http_port }}/71420.txt' + dest: '{{ remote_tmp_dir }}' + checksum: 'sha1:http://localhost:{{ http_port }}/sha1sum.txt' + register: result_sha1_71420 + +- stat: + path: "{{ remote_tmp_dir }}/71420.txt" + register: stat_result_sha1_71420 + +- name: download 71420.txt with sha256 checksum url + get_url: + url: 'http://localhost:{{ http_port }}/71420.txt' + dest: '{{ remote_tmp_dir }}/71420sha256.txt' + checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum.txt' + register: result_sha256_71420 + +- stat: + path: "{{ remote_tmp_dir }}/71420.txt" + register: stat_result_sha256_71420 + +- name: download 71420.txt with sha256 checksum url with dot leading paths + get_url: + url: 'http://localhost:{{ http_port }}/71420.txt' + dest: '{{ remote_tmp_dir }}/71420sha256_with_dot.txt' + checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum_with_dot.txt' + register: result_sha256_with_dot_71420 + +- stat: + path: "{{ remote_tmp_dir }}/71420sha256_with_dot.txt" + register: stat_result_sha256_with_dot_71420 + +- name: download 71420.txt with sha256 checksum url with asterisk leading paths + get_url: + url: 'http://localhost:{{ http_port }}/71420.txt' + dest: '{{ remote_tmp_dir }}/71420sha256_with_asterisk.txt' + checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum_with_asterisk.txt' + register: result_sha256_with_asterisk_71420 + +- stat: + path: "{{ remote_tmp_dir }}/71420sha256_with_asterisk.txt" + register: stat_result_sha256_with_asterisk_71420 + +- name: download 71420.txt with sha256 checksum url with file scheme + get_url: + url: 'http://localhost:{{ http_port }}/71420.txt' + dest: '{{ remote_tmp_dir }}/71420sha256_with_file_scheme.txt' + checksum: 'sha256:file://{{ files_dir }}/sha256sum.txt' + register: result_sha256_with_file_scheme_71420 + +- stat: + path: "{{ remote_tmp_dir }}/71420sha256_with_dot.txt" + register: stat_result_sha256_with_file_scheme_71420 + - name: Assert that the file was downloaded assert: that: - result_sha1 is changed - result_sha256 is changed - result_sha256_with_dot is changed + - result_sha256_with_asterisk is changed - result_sha256_with_file_scheme is changed - "stat_result_sha1.stat.exists == true" - "stat_result_sha256.stat.exists == true" - "stat_result_sha256_with_dot.stat.exists == true" + - "stat_result_sha256_with_asterisk.stat.exists == true" - "stat_result_sha256_with_file_scheme.stat.exists == true" + - result_sha1_71420 is changed + - result_sha256_71420 is changed + - result_sha256_with_dot_71420 is changed + - result_sha256_with_asterisk_71420 is changed + - result_sha256_with_file_scheme_71420 is changed + - "stat_result_sha1_71420.stat.exists == true" + - "stat_result_sha256_71420.stat.exists == true" + - "stat_result_sha256_with_dot_71420.stat.exists == true" + - "stat_result_sha256_with_asterisk_71420.stat.exists == true" + - "stat_result_sha256_with_file_scheme_71420.stat.exists == true" #https://github.com/ansible/ansible/issues/16191 - name: Test url split with no filename