From fb092a82a1a013fd38a37b90b305fc9a8fa11a13 Mon Sep 17 00:00:00 2001 From: Jordan Borean Date: Fri, 20 Nov 2020 04:54:40 +1000 Subject: [PATCH] ansible-galaxy - source deps from all servers and not just parent (#72576) * ansible-galaxy - source deps from all servers and not just parent * Added integration tests for this scenario --- changelogs/fragments/galaxy-servers.yml | 4 ++ lib/ansible/galaxy/collection/__init__.py | 2 +- .../library/reset_pulp.py | 20 +++---- .../ansible-galaxy-collection/tasks/main.yml | 54 +++++++++++++++++-- .../ansible-galaxy-collection/tasks/pulp.yml | 2 +- .../templates/ansible.cfg.j2 | 12 +++-- .../ansible-galaxy-collection/vars/main.yml | 4 ++ .../ansible_test/_internal/cloud/galaxy.py | 6 +-- 8 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 changelogs/fragments/galaxy-servers.yml diff --git a/changelogs/fragments/galaxy-servers.yml b/changelogs/fragments/galaxy-servers.yml new file mode 100644 index 00000000000..9e480ccd61b --- /dev/null +++ b/changelogs/fragments/galaxy-servers.yml @@ -0,0 +1,4 @@ +minor_changes: +- > + ansible-galaxy - find any collection dependencies in the globally configured Galaxy servers and not just the server + the parent collection is from. diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index aae8668a5a1..1038d56658d 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -1193,7 +1193,7 @@ def _build_dependency_map(collections, existing_collections, b_temp_path, apis, deps_exhausted = False for dep_name, dep_requirement in parent_info.dependencies.items(): _get_collection_info(dependency_map, existing_collections, dep_name, dep_requirement, - parent_info.api, b_temp_path, apis, validate_certs, force_deps, + source, b_temp_path, apis, validate_certs, force_deps, parent=parent, allow_pre_release=allow_pre_release) checked_parents.add(parent) diff --git a/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py b/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py index 21d13b2fdae..53c29f7704f 100644 --- a/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py +++ b/test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py @@ -33,16 +33,17 @@ options: - The password to use when authenticating against Pulp. required: yes type: str - repository: + repositories: description: - - The name of the repository to create. - - This should match C(GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH) in C(/etc/pulp/settings.py) or use the default of - C(published). + - A list of pulp repositories to create. + - Galaxy NG expects a repository that matches C(GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH) in + C(/etc/pulp/settings.py) or the default of C(published). required: yes - type: str + type: list + elements: str namespaces: description: - - A list of namespaces to create. + - A list of namespaces to create for Galaxy NG. required: yes type: list elements: str @@ -183,7 +184,7 @@ def main(): galaxy_ng_server=dict(type='str', required=True), url_username=dict(type='str', required=True), url_password=dict(type='str', required=True, no_log=True), - repository=dict(type='str', required=True), + repositories=dict(type='list', elements='str', required=True), namespaces=dict(type='list', elements='str', required=True), ) @@ -198,8 +199,9 @@ def main(): delete_pulp_orphans(module) [delete_galaxy_namespace(n, module) for n in get_galaxy_namespaces(module)] - repo_href = new_pulp_repository(module.params['repository'], module) - new_pulp_distribution(module.params['repository'], module.params['repository'], repo_href, module) + for repository in module.params['repositories']: + repo_href = new_pulp_repository(repository, module) + new_pulp_distribution(repository, repository, repo_href, module) [new_galaxy_namespace(n, module) for n in module.params['namespaces']] module.exit_json(changed=True) diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml index a18db64d4bb..fefba765a7e 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml @@ -62,9 +62,9 @@ vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}' loop: - name: pulp_v2 - server: '{{ pulp_v2_server }}' + server: '{{ pulp_server }}published/api/' - name: pulp_v3 - server: '{{ pulp_v3_server }}' + server: '{{ pulp_server }}published/api/' v3: true - name: galaxy_ng server: '{{ galaxy_ng_server }}' @@ -103,11 +103,57 @@ v3: true requires_auth: true - name: pulp_v2 - server: '{{ pulp_v2_server }}' + server: '{{ pulp_server }}published/api/' - name: pulp_v3 - server: '{{ pulp_v3_server }}' + server: '{{ pulp_server }}published/api/' v3: true +- name: publish collection with a dep on another server + setup_collections: + server: secondary + collections: + - namespace: secondary + name: name + # parent_dep.parent_collection does not exist on the secondary server + dependencies: + parent_dep.parent_collection: '*' + environment: + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + +- name: install collection with dep on another server + command: ansible-galaxy collection install secondary.name -vvv # 3 -v's will show the source in the stdout + register: install_cross_dep + environment: + ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + +- name: get result of install collection with dep on another server + slurp: + path: '{{ galaxy_dir }}/ansible_collections/{{ item.namespace }}/{{ item.name }}/MANIFEST.json' + register: install_cross_dep_actual + loop: + - namespace: secondary + name: name + - namespace: parent_dep + name: parent_collection + - namespace: child_dep + name: child_collection + - namespace: child_dep + name: child_dep2 + +- name: assert result of install collection with dep on another server + assert: + that: + - '"''secondary.name'' obtained from server secondary" in install_cross_dep.stdout' + # pulp_v2 is highest in the list so it will find it there first + - '"''parent_dep.parent_collection'' obtained from server pulp_v2" in install_cross_dep.stdout' + - '"''child_dep.child_collection'' obtained from server pulp_v2" in install_cross_dep.stdout' + - '"''child_dep.child_dep2'' obtained from server pulp_v2" in install_cross_dep.stdout' + - (install_cross_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' + - (install_cross_dep_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' + - (install_cross_dep_actual.results[2].content | b64decode | from_json).collection_info.version == '0.9.9' + - (install_cross_dep_actual.results[3].content | b64decode | from_json).collection_info.version == '1.2.2' + # fake.fake does not exist but we check the output to ensure it checked all 3 # servers defined in the config. We hardcode to -vvv as that's what level the # message is shown diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/pulp.yml b/test/integration/targets/ansible-galaxy-collection/tasks/pulp.yml index 87318cf68d1..7b6c5f86869 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/pulp.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/pulp.yml @@ -7,5 +7,5 @@ galaxy_ng_server: '{{ galaxy_ng_server }}' url_username: '{{ pulp_user }}' url_password: '{{ pulp_password }}' - repository: published + repositories: '{{ pulp_repositories }}' namespaces: '{{ collection_list|map(attribute="namespace")|unique + publish_namespaces }}' diff --git a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 index 0f52dce9357..62f3dcf901b 100644 --- a/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 +++ b/test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2 @@ -1,15 +1,15 @@ [galaxy] # Ensures subsequent unstable reruns don't use the cached information causing another failure cache_dir={{ remote_tmp_dir }}/galaxy_cache -server_list=pulp_v2,pulp_v3,galaxy_ng +server_list=pulp_v2,pulp_v3,galaxy_ng,secondary [galaxy_server.pulp_v2] -url={{ pulp_v2_server }} +url={{ pulp_server }}published/api/ username={{ pulp_user }} password={{ pulp_password }} [galaxy_server.pulp_v3] -url={{ pulp_v3_server }} +url={{ pulp_server }}published/api/ v3=true username={{ pulp_user }} password={{ pulp_password }} @@ -17,3 +17,9 @@ password={{ pulp_password }} [galaxy_server.galaxy_ng] url={{ galaxy_ng_server }} token={{ galaxy_ng_token.json.token }} + +[galaxy_server.secondary] +url={{ pulp_server }}secondary/api/ +v3=true +username={{ pulp_user }} +password={{ pulp_password }} diff --git a/test/integration/targets/ansible-galaxy-collection/vars/main.yml b/test/integration/targets/ansible-galaxy-collection/vars/main.yml index c22e8663e1f..0188efa9c07 100644 --- a/test/integration/targets/ansible-galaxy-collection/vars/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/vars/main.yml @@ -1,5 +1,9 @@ galaxy_verbosity: "{{ '' if not ansible_verbosity else '-' ~ ('v' * ansible_verbosity) }}" +pulp_repositories: + - published + - secondary + publish_namespaces: - ansible_test diff --git a/test/lib/ansible_test/_internal/cloud/galaxy.py b/test/lib/ansible_test/_internal/cloud/galaxy.py index b524078c239..c045a362c4a 100644 --- a/test/lib/ansible_test/_internal/cloud/galaxy.py +++ b/test/lib/ansible_test/_internal/cloud/galaxy.py @@ -238,16 +238,14 @@ class GalaxyEnvironment(CloudEnvironment): ansible_vars=dict( pulp_user=pulp_user, pulp_password=pulp_password, - pulp_v2_server='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port), - pulp_v3_server='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port), pulp_api='http://%s:%s' % (pulp_host, pulp_port), + pulp_server='http://%s:%s/pulp_ansible/galaxy/' % (pulp_host, pulp_port), galaxy_ng_server='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port), ), env_vars=dict( PULP_USER=pulp_user, PULP_PASSWORD=pulp_password, - PULP_V2_SERVER='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port), - PULP_V3_SERVER='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port), + PULP_SERVER='http://%s:%s/pulp_ansible/galaxy/api/' % (pulp_host, pulp_port), GALAXY_NG_SERVER='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port), ), )