From 54e2ae79e72cd651c233bbbefc23ae53444980e7 Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Thu, 30 Jul 2020 13:55:29 -0400 Subject: [PATCH] fix downloading collections in git repos and tar.gz artifacts (#70524) * Fix downloading tar files * Fix downloading SCM collections * changelog --- .../70524-fix-download-collections.yaml | 2 ++ lib/ansible/galaxy/collection.py | 16 +++++++-- .../tasks/download.yml | 36 +++++++++++++++++++ .../tasks/main.yml | 1 + .../tasks/download.yml | 36 +++++++++++++++++++ 5 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/70524-fix-download-collections.yaml create mode 100644 test/integration/targets/ansible-galaxy-collection-scm/tasks/download.yml diff --git a/changelogs/fragments/70524-fix-download-collections.yaml b/changelogs/fragments/70524-fix-download-collections.yaml new file mode 100644 index 00000000000..ebf38d28055 --- /dev/null +++ b/changelogs/fragments/70524-fix-download-collections.yaml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-galaxy collection download - fix downloading tar.gz files and collections in git repositories (https://github.com/ansible/ansible/issues/70429) diff --git a/lib/ansible/galaxy/collection.py b/lib/ansible/galaxy/collection.py index 21d76c1244f..dca5edd3b3c 100644 --- a/lib/ansible/galaxy/collection.py +++ b/lib/ansible/galaxy/collection.py @@ -581,6 +581,7 @@ def build_collection(collection_path, output_path, force): "the collection artifact." % to_native(collection_output)) _build_collection_tar(b_collection_path, b_collection_output, collection_manifest, file_manifest) + return collection_output def download_collections(collections, output_path, apis, validate_certs, no_deps, allow_pre_release): @@ -610,8 +611,19 @@ def download_collections(collections, output_path, apis, validate_certs, no_deps requirements.append({'name': collection_filename, 'version': requirement.latest_version}) display.display("Downloading collection '%s' to '%s'" % (name, dest_path)) - b_temp_download_path = requirement.download(b_temp_path) - shutil.move(b_temp_download_path, to_bytes(dest_path, errors='surrogate_or_strict')) + + if requirement.api is None and requirement.b_path and os.path.isfile(requirement.b_path): + shutil.copy(requirement.b_path, to_bytes(dest_path, errors='surrogate_or_strict')) + elif requirement.api is None and requirement.b_path: + temp_path = to_text(b_temp_path, errors='surrogate_or_string') + scm_build_path = os.path.join(temp_path, 'tmp_build-%s' % collection_filename) + os.makedirs(to_bytes(scm_build_path, errors='surrogate_or_strict'), mode=0o0755) + temp_download_path = build_collection(os.path.join(temp_path, name), scm_build_path, True) + shutil.move(to_bytes(temp_download_path, errors='surrogate_or_strict'), to_bytes(dest_path, errors='surrogate_or_strict')) + else: + b_temp_download_path = requirement.download(b_temp_path) + shutil.move(b_temp_download_path, to_bytes(dest_path, errors='surrogate_or_strict')) + display.display("%s (%s) was downloaded successfully" % (name, requirement.latest_version)) requirements_path = os.path.join(output_path, 'requirements.yml') diff --git a/test/integration/targets/ansible-galaxy-collection-scm/tasks/download.yml b/test/integration/targets/ansible-galaxy-collection-scm/tasks/download.yml new file mode 100644 index 00000000000..fd1da68b623 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection-scm/tasks/download.yml @@ -0,0 +1,36 @@ +- name: create test download dir + file: + path: '{{ galaxy_dir }}/download' + state: directory + +- name: download a git repository + command: 'ansible-galaxy collection download git+https://github.com/ansible-collections/amazon.aws.git,37875c5b4ba5bf3cc43e07edf29f3432fd76def5' + args: + chdir: '{{ galaxy_dir }}/download' + register: download_collection + +- name: check that the file was downloaded + stat: + path: '{{ galaxy_dir }}/download/collections/amazon-aws-1.0.0.tar.gz' + register: download_collection_actual + +- assert: + that: + - '"Downloading collection ''amazon.aws'' to" in download_collection.stdout' + - download_collection_actual.stat.exists + +- name: test the downloaded repository can be installed + command: 'ansible-galaxy collection install -r requirements.yml' + args: + chdir: '{{ galaxy_dir }}/download/collections/' + +- name: list installed collections + command: 'ansible-galaxy collection list' + register: installed_collections + +- assert: + that: + - "'amazon.aws' in installed_collections.stdout" + +- include_tasks: ./empty_installed_collections.yml + when: cleanup diff --git a/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml index 500defb878d..7db7e1d6e6f 100644 --- a/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml @@ -22,6 +22,7 @@ - include_tasks: ./multi_collection_repo_individual.yml - include_tasks: ./setup_recursive_scm_dependency.yml - include_tasks: ./scm_dependency_deduplication.yml + - include_tasks: ./download.yml always: diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml index 008bb7d5976..5bb12ec12da 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml @@ -104,3 +104,39 @@ assert: that: - '"Skipping install, no requirements found" in install_no_requirements.stdout' + +- name: Test downloading a tar.gz collection artifact + block: + + - name: get result of build basic collection on current directory + stat: + path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection/ansible_test-my_collection-1.0.0.tar.gz' + register: result + + - name: create default skeleton + command: ansible-galaxy collection init ansible_test.my_collection {{ galaxy_verbosity }} + args: + chdir: '{{ galaxy_dir }}/scratch' + when: not result.stat.exists + + - name: build the tar.gz + command: ansible-galaxy collection build {{ galaxy_verbosity }} + args: + chdir: '{{ galaxy_dir }}/scratch/ansible_test/my_collection' + when: not result.stat.exists + + - name: download a tar.gz file + command: ansible-galaxy collection download '{{ galaxy_dir }}/scratch/ansible_test/my_collection/ansible_test-my_collection-1.0.0.tar.gz' + args: + chdir: '{{ galaxy_dir }}/download' + register: download_collection + + - name: get result of downloaded tar.gz + stat: + path: '{{ galaxy_dir }}/download/collections/ansible_test-my_collection-1.0.0.tar.gz' + register: download_collection_actual + + - assert: + that: + - '"Downloading collection ''ansible_test.my_collection'' to" in download_collection.stdout' + - download_collection_actual.stat.exists