[2.10] get_url: Fix checksum binary validation (#74674)

* get_url: Handle same SHA sum for downloaded files (#71435)

Fixes: #71420

Signed-off-by: Abhijeet Kasurde <akasurde@redhat.com>
(cherry picked from commit 159544610e)

* modules: get_url: Fix checksum binary validation (#74502)

From the sha512sum man page:

... The default mode is to print a line with checksum, a character indicating type ('*' for binary, ' ' for text), and name for each FILE.

(cherry picked from commit 403a5d147d)

Co-authored-by: Abhijeet Kasurde <akasurde@redhat.com>

changelogs/fragments/71420_get_url.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- get_url - handle same SHA sum for checksum file (https://github.com/ansible/ansible/issues/71420).

changelogs/fragments/74502-get_url-filx-checksum-binary.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - get_url - Fixed checksum validation for binary files (leading asterisk) in checksum files (https://github.com/ansible/ansible/pull/74502).

lib/ansible/modules/get_url.py

@@ -502,16 +502,23 @@ def main():
             with open(checksum_tmpsrc) as f:
                 lines = [line.rstrip('\n') for line in f]
             os.remove(checksum_tmpsrc)
-            checksum_map = {}
+            checksum_map = []
             for line in lines:
-                parts = line.split(None, 1)
+                # Split by one whitespace to keep the leading type char ' ' (whitespace) for text and '*' for binary
+                parts = line.split(" ", 1)
                 if len(parts) == 2:
-                    checksum_map[parts[0]] = parts[1]
+                    # Remove the leading type char, we expect
+                    if parts[1].startswith((" ", "*",)):
+                        parts[1] = parts[1][1:]
+
+                    # Append checksum and path without potential leading './'
+                    checksum_map.append((parts[0], parts[1].lstrip("./")))
+
             filename = url_filename(url)
 
             # Look through each line in the checksum file for a hash corresponding to
             # the filename in the url, returning the first hash that is found.
-            for cksum in (s for (s, f) in checksum_map.items() if f.strip('./') == filename):
+            for cksum in (s for (s, f) in checksum_map if f == filename):
                 checksum = cksum
                 break
             else:

test/integration/targets/get_url/tasks/main.yml

@@ -335,11 +335,17 @@
     dest: '{{ files_dir }}/27617.txt'
     content: "ptux"
 
+- name: create duplicate src file
+  copy:
+    dest: '{{ files_dir }}/71420.txt'
+    content: "ptux"
+
 - name: create sha1 checksum file of src
   copy:
     dest: '{{ files_dir }}/sha1sum.txt'
     content: |
       a97e6837f60cec6da4491bab387296bbcd72bdba  27617.txt
+      a97e6837f60cec6da4491bab387296bbcd72bdba  71420.txt
       3911340502960ca33aece01129234460bfeb2791  not_target1.txt
       1b4b6adf30992cedb0f6edefd6478ff0a593b2e4  not_target2.txt
 
@@ -348,6 +354,7 @@
     dest: '{{ files_dir }}/sha256sum.txt'
     content: |
       b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  27617.txt
+      b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  71420.txt
       30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  not_target1.txt
       d0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  not_target2.txt
 
@@ -356,9 +363,19 @@
     dest: '{{ files_dir }}/sha256sum_with_dot.txt'
     content: |
       b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  ./27617.txt
+      b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  ./71420.txt
       30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  ./not_target1.txt
       d0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  ./not_target2.txt
 
+- name: create sha256 checksum file of src with a * leading path
+  copy:
+    dest: '{{ files_dir }}/sha256sum_with_asterisk.txt'
+    content: |
+      b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. *27617.txt
+      b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006. *71420.txt
+      30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892 *not_target1.txt
+      d0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b *not_target2.txt
+
 - copy:
     src: "testserver.py"
     dest: "{{ remote_tmp_dir }}/testserver.py"
@@ -407,6 +424,17 @@
     path: "{{ remote_tmp_dir }}/27617sha256_with_dot.txt"
   register: stat_result_sha256_with_dot
 
+- name: download src with sha256 checksum url with asterisk leading paths
+  get_url:
+    url: 'http://localhost:{{ http_port }}/27617.txt'
+    dest: '{{ remote_tmp_dir }}/27617sha256_with_asterisk.txt'
+    checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum_with_asterisk.txt'
+  register: result_sha256_with_asterisk
+
+- stat:
+    path: "{{ remote_tmp_dir }}/27617sha256_with_asterisk.txt"
+  register: stat_result_sha256_with_asterisk
+
 - name: download src with sha256 checksum url with file scheme
   get_url:
     url: 'http://localhost:{{ http_port }}/27617.txt'
@@ -418,17 +446,84 @@
     path: "{{ remote_tmp_dir }}/27617sha256_with_dot.txt"
   register: stat_result_sha256_with_file_scheme
 
+- name: download 71420.txt with sha1 checksum url
+  get_url:
+    url: 'http://localhost:{{ http_port }}/71420.txt'
+    dest: '{{ remote_tmp_dir }}'
+    checksum: 'sha1:http://localhost:{{ http_port }}/sha1sum.txt'
+  register: result_sha1_71420
+
+- stat:
+    path: "{{ remote_tmp_dir }}/71420.txt"
+  register: stat_result_sha1_71420
+
+- name: download 71420.txt with sha256 checksum url
+  get_url:
+    url: 'http://localhost:{{ http_port }}/71420.txt'
+    dest: '{{ remote_tmp_dir }}/71420sha256.txt'
+    checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum.txt'
+  register: result_sha256_71420
+
+- stat:
+    path: "{{ remote_tmp_dir }}/71420.txt"
+  register: stat_result_sha256_71420
+
+- name: download 71420.txt with sha256 checksum url with dot leading paths
+  get_url:
+    url: 'http://localhost:{{ http_port }}/71420.txt'
+    dest: '{{ remote_tmp_dir }}/71420sha256_with_dot.txt'
+    checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum_with_dot.txt'
+  register: result_sha256_with_dot_71420
+
+- stat:
+    path: "{{ remote_tmp_dir }}/71420sha256_with_dot.txt"
+  register: stat_result_sha256_with_dot_71420
+
+- name: download 71420.txt with sha256 checksum url with asterisk leading paths
+  get_url:
+    url: 'http://localhost:{{ http_port }}/71420.txt'
+    dest: '{{ remote_tmp_dir }}/71420sha256_with_asterisk.txt'
+    checksum: 'sha256:http://localhost:{{ http_port }}/sha256sum_with_asterisk.txt'
+  register: result_sha256_with_asterisk_71420
+
+- stat:
+    path: "{{ remote_tmp_dir }}/71420sha256_with_asterisk.txt"
+  register: stat_result_sha256_with_asterisk_71420
+
+- name: download 71420.txt with sha256 checksum url with file scheme
+  get_url:
+    url: 'http://localhost:{{ http_port }}/71420.txt'
+    dest: '{{ remote_tmp_dir }}/71420sha256_with_file_scheme.txt'
+    checksum: 'sha256:file://{{ files_dir }}/sha256sum.txt'
+  register: result_sha256_with_file_scheme_71420
+
+- stat:
+    path: "{{ remote_tmp_dir }}/71420sha256_with_dot.txt"
+  register: stat_result_sha256_with_file_scheme_71420
+
 - name: Assert that the file was downloaded
   assert:
     that:
       - result_sha1 is changed
       - result_sha256 is changed
       - result_sha256_with_dot is changed
+      - result_sha256_with_asterisk is changed
       - result_sha256_with_file_scheme is changed
       - "stat_result_sha1.stat.exists == true"
       - "stat_result_sha256.stat.exists == true"
       - "stat_result_sha256_with_dot.stat.exists == true"
+      - "stat_result_sha256_with_asterisk.stat.exists == true"
       - "stat_result_sha256_with_file_scheme.stat.exists == true"
+      - result_sha1_71420 is changed
+      - result_sha256_71420 is changed
+      - result_sha256_with_dot_71420 is changed
+      - result_sha256_with_asterisk_71420 is changed
+      - result_sha256_with_file_scheme_71420 is changed
+      - "stat_result_sha1_71420.stat.exists == true"
+      - "stat_result_sha256_71420.stat.exists == true"
+      - "stat_result_sha256_with_dot_71420.stat.exists == true"
+      - "stat_result_sha256_with_asterisk_71420.stat.exists == true"
+      - "stat_result_sha256_with_file_scheme_71420.stat.exists == true"
 
 #https://github.com/ansible/ansible/issues/16191
 - name: Test url split with no filename