ansible-galaxy - source deps from all servers and not just parent (#72576)

* ansible-galaxy - source deps from all servers and not just parent

* Added integration tests for this scenario

changelogs/fragments/galaxy-servers.yml

@@ -0,0 +1,4 @@
+minor_changes:
+- >
+  ansible-galaxy - find any collection dependencies in the globally configured Galaxy servers and not just the server
+  the parent collection is from.

lib/ansible/galaxy/collection/__init__.py

@@ -1193,7 +1193,7 @@ def _build_dependency_map(collections, existing_collections, b_temp_path, apis,
                     deps_exhausted = False
                     for dep_name, dep_requirement in parent_info.dependencies.items():
                         _get_collection_info(dependency_map, existing_collections, dep_name, dep_requirement,
-                                             parent_info.api, b_temp_path, apis, validate_certs, force_deps,
+                                             source, b_temp_path, apis, validate_certs, force_deps,
                                              parent=parent, allow_pre_release=allow_pre_release)
 
                     checked_parents.add(parent)

test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py

@@ -33,16 +33,17 @@ options:
     - The password to use when authenticating against Pulp.
     required: yes
     type: str
-  repository:
+  repositories:
     description:
-    - The name of the repository to create.
-    - This should match C(GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH) in C(/etc/pulp/settings.py) or use the default of
-      C(published).
+    - A list of pulp repositories to create.
+    - Galaxy NG expects a repository that matches C(GALAXY_API_DEFAULT_DISTRIBUTION_BASE_PATH) in
+      C(/etc/pulp/settings.py) or the default of C(published).
     required: yes
-    type: str
+    type: list
+    elements: str
   namespaces:
     description:
-    - A list of namespaces to create.
+    - A list of namespaces to create for Galaxy NG.
     required: yes
     type: list
     elements: str
@@ -183,7 +184,7 @@ def main():
         galaxy_ng_server=dict(type='str', required=True),
         url_username=dict(type='str', required=True),
         url_password=dict(type='str', required=True, no_log=True),
-        repository=dict(type='str', required=True),
+        repositories=dict(type='list', elements='str', required=True),
         namespaces=dict(type='list', elements='str', required=True),
     )
 
@@ -198,8 +199,9 @@ def main():
     delete_pulp_orphans(module)
     [delete_galaxy_namespace(n, module) for n in get_galaxy_namespaces(module)]
 
-    repo_href = new_pulp_repository(module.params['repository'], module)
-    new_pulp_distribution(module.params['repository'], module.params['repository'], repo_href, module)
+    for repository in module.params['repositories']:
+        repo_href = new_pulp_repository(repository, module)
+        new_pulp_distribution(repository, repository, repo_href, module)
     [new_galaxy_namespace(n, module) for n in module.params['namespaces']]
 
     module.exit_json(changed=True)

test/integration/targets/ansible-galaxy-collection/tasks/main.yml

@@ -62,9 +62,9 @@
     vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}'
   loop:
   - name: pulp_v2
-    server: '{{ pulp_v2_server }}'
+    server: '{{ pulp_server }}published/api/'
   - name: pulp_v3
-    server: '{{ pulp_v3_server }}'
+    server: '{{ pulp_server }}published/api/'
     v3: true
   - name: galaxy_ng
     server: '{{ galaxy_ng_server }}'
@@ -103,11 +103,57 @@
     v3: true
     requires_auth: true
   - name: pulp_v2
-    server: '{{ pulp_v2_server }}'
+    server: '{{ pulp_server }}published/api/'
   - name: pulp_v3
-    server: '{{ pulp_v3_server }}'
+    server: '{{ pulp_server }}published/api/'
     v3: true
 
+- name: publish collection with a dep on another server
+  setup_collections:
+    server: secondary
+    collections:
+    - namespace: secondary
+      name: name
+      # parent_dep.parent_collection does not exist on the secondary server
+      dependencies:
+        parent_dep.parent_collection: '*'
+  environment:
+    ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
+
+- name: install collection with dep on another server
+  command: ansible-galaxy collection install secondary.name -vvv  # 3 -v's will show the source in the stdout
+  register: install_cross_dep
+  environment:
+    ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
+    ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
+
+- name: get result of install collection with dep on another server
+  slurp:
+    path: '{{ galaxy_dir }}/ansible_collections/{{ item.namespace }}/{{ item.name }}/MANIFEST.json'
+  register: install_cross_dep_actual
+  loop:
+  - namespace: secondary
+    name: name
+  - namespace: parent_dep
+    name: parent_collection
+  - namespace: child_dep
+    name: child_collection
+  - namespace: child_dep
+    name: child_dep2
+
+- name: assert result of install collection with dep on another server
+  assert:
+    that:
+    - '"''secondary.name'' obtained from server secondary" in install_cross_dep.stdout'
+    # pulp_v2 is highest in the list so it will find it there first
+    - '"''parent_dep.parent_collection'' obtained from server pulp_v2" in install_cross_dep.stdout'
+    - '"''child_dep.child_collection'' obtained from server pulp_v2" in install_cross_dep.stdout'
+    - '"''child_dep.child_dep2'' obtained from server pulp_v2" in install_cross_dep.stdout'
+    - (install_cross_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0'
+    - (install_cross_dep_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0'
+    - (install_cross_dep_actual.results[2].content | b64decode | from_json).collection_info.version == '0.9.9'
+    - (install_cross_dep_actual.results[3].content | b64decode | from_json).collection_info.version == '1.2.2'
+
 # fake.fake does not exist but we check the output to ensure it checked all 3
 # servers defined in the config. We hardcode to -vvv as that's what level the
 # message is shown

test/integration/targets/ansible-galaxy-collection/tasks/pulp.yml

@@ -7,5 +7,5 @@
     galaxy_ng_server: '{{ galaxy_ng_server }}'
     url_username: '{{ pulp_user }}'
     url_password: '{{ pulp_password }}'
-    repository: published
+    repositories: '{{ pulp_repositories }}'
     namespaces: '{{ collection_list|map(attribute="namespace")|unique + publish_namespaces }}'

test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2

@@ -1,15 +1,15 @@
 [galaxy]
 # Ensures subsequent unstable reruns don't use the cached information causing another failure
 cache_dir={{ remote_tmp_dir }}/galaxy_cache
-server_list=pulp_v2,pulp_v3,galaxy_ng
+server_list=pulp_v2,pulp_v3,galaxy_ng,secondary
 
 [galaxy_server.pulp_v2]
-url={{ pulp_v2_server }}
+url={{ pulp_server }}published/api/
 username={{ pulp_user }}
 password={{ pulp_password }}
 
 [galaxy_server.pulp_v3]
-url={{ pulp_v3_server }}
+url={{ pulp_server }}published/api/
 v3=true
 username={{ pulp_user }}
 password={{ pulp_password }}
@@ -17,3 +17,9 @@ password={{ pulp_password }}
 [galaxy_server.galaxy_ng]
 url={{ galaxy_ng_server }}
 token={{ galaxy_ng_token.json.token }}
+
+[galaxy_server.secondary]
+url={{ pulp_server }}secondary/api/
+v3=true
+username={{ pulp_user }}
+password={{ pulp_password }}

test/integration/targets/ansible-galaxy-collection/vars/main.yml

@@ -1,5 +1,9 @@
 galaxy_verbosity: "{{ '' if not ansible_verbosity else '-' ~ ('v' * ansible_verbosity) }}"
 
+pulp_repositories:
+  - published
+  - secondary
+
 publish_namespaces:
   - ansible_test
 

test/lib/ansible_test/_internal/cloud/galaxy.py

@@ -238,16 +238,14 @@ class GalaxyEnvironment(CloudEnvironment):
             ansible_vars=dict(
                 pulp_user=pulp_user,
                 pulp_password=pulp_password,
-                pulp_v2_server='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
-                pulp_v3_server='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
                 pulp_api='http://%s:%s' % (pulp_host, pulp_port),
+                pulp_server='http://%s:%s/pulp_ansible/galaxy/' % (pulp_host, pulp_port),
                 galaxy_ng_server='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port),
             ),
             env_vars=dict(
                 PULP_USER=pulp_user,
                 PULP_PASSWORD=pulp_password,
-                PULP_V2_SERVER='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
-                PULP_V3_SERVER='http://%s:%s/pulp_ansible/galaxy/published/api/' % (pulp_host, pulp_port),
+                PULP_SERVER='http://%s:%s/pulp_ansible/galaxy/api/' % (pulp_host, pulp_port),
                 GALAXY_NG_SERVER='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port),
             ),
         )