fix downloading collections in git repos and tar.gz artifacts (#70524) (#71000)

* Fix downloading tar files

* Fix downloading SCM collections

* changelog

(cherry picked from commit 54e2ae79e7)

changelogs/fragments/70524-fix-download-collections.yaml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-galaxy collection download - fix downloading tar.gz files and collections in git repositories (https://github.com/ansible/ansible/issues/70429)

lib/ansible/galaxy/collection.py

@@ -581,6 +581,7 @@ def build_collection(collection_path, output_path, force):
                                "the collection artifact." % to_native(collection_output))
 
     _build_collection_tar(b_collection_path, b_collection_output, collection_manifest, file_manifest)
+    return collection_output
 
 
 def download_collections(collections, output_path, apis, validate_certs, no_deps, allow_pre_release):
@@ -610,8 +611,19 @@ def download_collections(collections, output_path, apis, validate_certs, no_deps
                 requirements.append({'name': collection_filename, 'version': requirement.latest_version})
 
                 display.display("Downloading collection '%s' to '%s'" % (name, dest_path))
-                b_temp_download_path = requirement.download(b_temp_path)
-                shutil.move(b_temp_download_path, to_bytes(dest_path, errors='surrogate_or_strict'))
+
+                if requirement.api is None and requirement.b_path and os.path.isfile(requirement.b_path):
+                    shutil.copy(requirement.b_path, to_bytes(dest_path, errors='surrogate_or_strict'))
+                elif requirement.api is None and requirement.b_path:
+                    temp_path = to_text(b_temp_path, errors='surrogate_or_string')
+                    scm_build_path = os.path.join(temp_path, 'tmp_build-%s' % collection_filename)
+                    os.makedirs(to_bytes(scm_build_path, errors='surrogate_or_strict'), mode=0o0755)
+                    temp_download_path = build_collection(os.path.join(temp_path, name), scm_build_path, True)
+                    shutil.move(to_bytes(temp_download_path, errors='surrogate_or_strict'), to_bytes(dest_path, errors='surrogate_or_strict'))
+                else:
+                    b_temp_download_path = requirement.download(b_temp_path)
+                    shutil.move(b_temp_download_path, to_bytes(dest_path, errors='surrogate_or_strict'))
+
                 display.display("%s (%s) was downloaded successfully" % (name, requirement.latest_version))
 
             requirements_path = os.path.join(output_path, 'requirements.yml')

test/integration/targets/ansible-galaxy-collection-scm/tasks/download.yml

@@ -0,0 +1,36 @@
+- name: create test download dir
+  file:
+    path: '{{ galaxy_dir }}/download'
+    state: directory
+
+- name: download a git repository
+  command: 'ansible-galaxy collection download git+https://github.com/ansible-collections/amazon.aws.git,37875c5b4ba5bf3cc43e07edf29f3432fd76def5'
+  args:
+    chdir: '{{ galaxy_dir }}/download'
+  register: download_collection
+
+- name: check that the file was downloaded
+  stat:
+    path: '{{ galaxy_dir }}/download/collections/amazon-aws-1.0.0.tar.gz'
+  register: download_collection_actual
+
+- assert:
+    that:
+      - '"Downloading collection ''amazon.aws'' to" in download_collection.stdout'
+      - download_collection_actual.stat.exists
+
+- name: test the downloaded repository can be installed
+  command: 'ansible-galaxy collection install -r requirements.yml'
+  args:
+    chdir: '{{ galaxy_dir }}/download/collections/'
+
+- name: list installed collections
+  command: 'ansible-galaxy collection list'
+  register: installed_collections
+
+- assert:
+    that:
+      - "'amazon.aws' in installed_collections.stdout"
+
+- include_tasks: ./empty_installed_collections.yml
+  when: cleanup

test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml

@@ -22,6 +22,7 @@
   - include_tasks: ./multi_collection_repo_individual.yml
   - include_tasks: ./setup_recursive_scm_dependency.yml
   - include_tasks: ./scm_dependency_deduplication.yml
+  - include_tasks: ./download.yml
 
   always:
 

test/integration/targets/ansible-galaxy-collection/tasks/download.yml

@@ -104,3 +104,39 @@
   assert:
     that:
       - '"Skipping install, no requirements found" in install_no_requirements.stdout'
+
+- name: Test downloading a tar.gz collection artifact
+  block:
+
+    - name: get result of build basic collection on current directory
+      stat:
+        path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection/ansible_test-my_collection-1.0.0.tar.gz'
+      register: result
+
+    - name: create default skeleton
+      command: ansible-galaxy collection init ansible_test.my_collection {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch'
+      when: not result.stat.exists
+
+    - name: build the tar.gz
+      command:  ansible-galaxy collection build {{ galaxy_verbosity }}
+      args:
+        chdir: '{{ galaxy_dir }}/scratch/ansible_test/my_collection'
+      when: not result.stat.exists
+
+    - name: download a tar.gz file
+      command: ansible-galaxy collection download '{{ galaxy_dir }}/scratch/ansible_test/my_collection/ansible_test-my_collection-1.0.0.tar.gz'
+      args:
+        chdir: '{{ galaxy_dir }}/download'
+      register: download_collection
+
+    - name: get result of downloaded tar.gz
+      stat:
+        path: '{{ galaxy_dir }}/download/collections/ansible_test-my_collection-1.0.0.tar.gz'
+      register: download_collection_actual
+
+    - assert:
+        that:
+        - '"Downloading collection ''ansible_test.my_collection'' to" in download_collection.stdout'
+        - download_collection_actual.stat.exists