Update galaxy_ng container for tests (#80721)

changelogs/fragments/80721-ansible-galaxy.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- ansible-galaxy - Remove internal configuration argument ``v3`` (https://github.com/ansible/ansible/pull/80721)

lib/ansible/cli/galaxy.py

@@ -10,6 +10,7 @@ __metaclass__ = type
 # ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
 from ansible.cli import CLI
 
+import argparse
 import json
 import os.path
 import pathlib
@@ -71,7 +72,7 @@ SERVER_DEF = [
     ('password', False, 'str'),
     ('token', False, 'str'),
     ('auth_url', False, 'str'),
-    ('v3', False, 'bool'),
+    ('api_version', False, 'int'),
     ('validate_certs', False, 'bool'),
     ('client_id', False, 'str'),
     ('timeout', False, 'int'),
@@ -79,7 +80,7 @@ SERVER_DEF = [
 
 # config definition fields
 SERVER_ADDITIONAL = {
-    'v3': {'default': 'False'},
+    'api_version': {'default': None, 'choices': [2, 3]},
     'validate_certs': {'cli': [{'name': 'validate_certs'}]},
     'timeout': {'default': '60', 'cli': [{'name': 'timeout'}]},
     'token': {'default': None},
@@ -240,6 +241,7 @@ class GalaxyCLI(CLI):
         # Common arguments that apply to more than 1 action
         common = opt_help.ArgumentParser(add_help=False)
         common.add_argument('-s', '--server', dest='api_server', help='The Galaxy API server URL')
+        common.add_argument('--api-version', type=int, choices=[2, 3], help=argparse.SUPPRESS)  # Hidden argument that should only be used in our tests
         common.add_argument('--token', '--api-key', dest='api_key',
                             help='The Ansible Galaxy API key which can be found at '
                                  'https://galaxy.ansible.com/me/preferences.')
@@ -644,17 +646,22 @@ class GalaxyCLI(CLI):
             client_id = server_options.pop('client_id')
             token_val = server_options['token'] or NoTokenSentinel
             username = server_options['username']
-            v3 = server_options.pop('v3')
+            api_version = server_options.pop('api_version')
             if server_options['validate_certs'] is None:
                 server_options['validate_certs'] = context.CLIARGS['resolved_validate_certs']
             validate_certs = server_options['validate_certs']
 
-            if v3:
+            # This allows a user to explicitly force use of an API version when
-                # This allows a user to explicitly indicate the server uses the /v3 API
+            # multiple versions are supported. This was added for testing
-                # This was added for testing against pulp_ansible and I'm not sure it has
+            # against pulp_ansible and I'm not sure it has a practical purpose
-                # a practical purpose outside of this use case. As such, this option is not
+            # outside of this use case. As such, this option is not documented
-                # documented as of now
+            # as of now
-                server_options['available_api_versions'] = {'v3': '/v3'}
+            if api_version:
+                display.warning(
+                    f'The specified "api_version" configuration for the galaxy server "{server_key}" is '
+                    'not a public configuration, and may be removed at any time without warning.'
+                )
+                server_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version}
 
             # default case if no auth info is provided.
             server_options['token'] = None
@@ -680,6 +687,13 @@ class GalaxyCLI(CLI):
             ))
 
         cmd_server = context.CLIARGS['api_server']
+        if context.CLIARGS['api_version']:
+            api_version = context.CLIARGS['api_version']
+            display.warning(
+                'The --api-version is not a public argument, and may be removed at any time without warning.'
+            )
+            galaxy_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version}
+
         cmd_token = GalaxyToken(token=context.CLIARGS['api_key'])
 
         validate_certs = context.CLIARGS['resolved_validate_certs']

lib/ansible/galaxy/api.py

@@ -923,10 +923,7 @@ class GalaxyAPI:
         data = self._call_galaxy(n_collection_url, error_context_msg=error_context_msg, cache=True)
         self._set_cache()
 
-        try:
+        signatures = [signature_info["signature"] for signature_info in data.get("signatures") or []]
-            signatures = data["signatures"]
+        if not signatures:
-        except KeyError:
             display.vvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}")
-            return []
+        return signatures
-        else:
-            return [signature_info["signature"] for signature_info in signatures]

test/integration/targets/ansible-galaxy-collection/library/reset_pulp.py

@@ -84,7 +84,8 @@ def invoke_api(module, url, method='GET', data=None, status_codes=None):
 
     resp, info = fetch_url(module, url, method=method, data=data, headers=headers)
     if info['status'] not in status_codes:
-        module.fail_json(url=url, **info)
+        info['url'] = url
+        module.fail_json(**info)
 
     data = to_text(resp.read())
     if data:
@@ -105,7 +106,7 @@ def delete_pulp_distribution(distribution, module):
 
 def delete_pulp_orphans(module):
     """ Deletes any orphaned pulp objects. """
-    orphan_uri = module.params['pulp_api'] + '/pulp/api/v3/orphans/'
+    orphan_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/orphans/'
     task_info = invoke_api(module, orphan_uri, method='DELETE', status_codes=[202])
     wait_pulp_task(task_info['task'], module)
 
@@ -125,25 +126,39 @@ def get_galaxy_namespaces(module):
     return [n['name'] for n in ns_info['data']]
 
 
-def get_pulp_distributions(module):
+def get_pulp_distributions(module, distribution):
     """ Gets a list of all the pulp distributions. """
-    distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/'
+    distro_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/distributions/ansible/ansible/'
-    distro_info = invoke_api(module, distro_uri)
+    distro_info = invoke_api(module, distro_uri + '?name=' + distribution)
     return [module.params['pulp_api'] + r['pulp_href'] for r in distro_info['results']]
 
 
-def get_pulp_repositories(module):
+def get_pulp_repositories(module, repository):
     """ Gets a list of all the pulp repositories. """
-    repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/'
+    repo_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/repositories/ansible/ansible/'
-    repo_info = invoke_api(module, repo_uri)
+    repo_info = invoke_api(module, repo_uri + '?name=' + repository)
     return [module.params['pulp_api'] + r['pulp_href'] for r in repo_info['results']]
 
 
+def get_repo_collections(repository, module):
+    collections_uri = module.params['galaxy_ng_server'] + 'v3/plugin/ansible/content/' + repository + '/collections/index/'
+    # status code 500 isn't really expected, an unhandled exception is causing this instead of a 404
+    # See https://issues.redhat.com/browse/AAH-2329
+    info = invoke_api(module, collections_uri + '?limit=100&offset=0', status_codes=[200, 500])
+    if not info:
+        return []
+    return [module.params['pulp_api'] + c['href'] for c in info['data']]
+
+
+def delete_repo_collection(collection, module):
+    task_info = invoke_api(module, collection, method='DELETE', status_codes=[202])
+    wait_pulp_task(task_info['task'], module)
+
+
 def new_galaxy_namespace(name, module):
     """ Creates a new namespace in Galaxy NG. """
-    ns_uri = module.params['galaxy_ng_server'] + 'v3/_ui/namespaces/'
+    ns_uri = module.params['galaxy_ng_server'] + 'v3/namespaces/ '
-    data = {'name': name, 'groups': [{'name': 'system:partner-engineers', 'object_permissions':
+    data = {'name': name, 'groups': []}
-            ['add_namespace', 'change_namespace', 'upload_to_namespace']}]}
     ns_info = invoke_api(module, ns_uri, method='POST', data=data, status_codes=[201])
 
     return ns_info['id']
@@ -151,16 +166,17 @@ def new_galaxy_namespace(name, module):
 
 def new_pulp_repository(name, module):
     """ Creates a new pulp repository. """
-    repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/'
+    repo_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/repositories/ansible/ansible/'
-    data = {'name': name}
+    # retain_repo_versions to work around https://issues.redhat.com/browse/AAH-2332
+    data = {'name': name, 'retain_repo_versions': '1024'}
     repo_info = invoke_api(module, repo_uri, method='POST', data=data, status_codes=[201])
 
-    return module.params['pulp_api'] + repo_info['pulp_href']
+    return repo_info['pulp_href']
 
 
 def new_pulp_distribution(name, base_path, repository, module):
     """ Creates a new pulp distribution for a repository. """
-    distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/'
+    distro_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/distributions/ansible/ansible/'
     data = {'name': name, 'base_path': base_path, 'repository': repository}
     task_info = invoke_api(module, distro_uri, method='POST', data=data, status_codes=[202])
     task_info = wait_pulp_task(task_info['task'], module)
@@ -194,8 +210,15 @@ def main():
     )
     module.params['force_basic_auth'] = True
 
-    [delete_pulp_distribution(d, module) for d in get_pulp_distributions(module)]
+    # It may be due to the process of cleaning up orphans, but we cannot delete the namespace
-    [delete_pulp_repository(r, module) for r in get_pulp_repositories(module)]
+    # while a collection still exists, so this is just a new safety to nuke all collections
+    # first
+    for repository in module.params['repositories']:
+        [delete_repo_collection(c, module) for c in get_repo_collections(repository, module)]
+
+    for repository in module.params['repositories']:
+        [delete_pulp_distribution(d, module) for d in get_pulp_distributions(module, repository)]
+        [delete_pulp_repository(r, module) for r in get_pulp_repositories(module, repository)]
     delete_pulp_orphans(module)
     [delete_galaxy_namespace(n, module) for n in get_galaxy_namespaces(module)]
 

test/integration/targets/ansible-galaxy-collection/library/setup_collections.py

@@ -77,6 +77,7 @@ RETURN = '''
 #
 '''
 
+import datetime
 import os
 import subprocess
 import tarfile
@@ -104,6 +105,7 @@ def publish_collection(module, collection):
     collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (namespace, name, version))
     b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict')
     os.mkdir(b_collection_dir)
+    os.mkdir(os.path.join(b_collection_dir, b'meta'))
 
     with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd:
         fd.write(b"Collection readme")
@@ -120,6 +122,8 @@ def publish_collection(module, collection):
     }
     with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
         fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))
+    with open(os.path.join(b_collection_dir, b'meta/runtime.yml'), mode='wb') as fd:
+        fd.write(b'requires_ansible: ">=1.0.0"')
 
     with tempfile.NamedTemporaryFile(mode='wb') as temp_fd:
         temp_fd.write(b"data")
@@ -241,7 +245,8 @@ def run_module():
         supports_check_mode=False
     )
 
-    result = dict(changed=True, results=[])
+    start = datetime.datetime.now()
+    result = dict(changed=True, results=[], start=str(start))
 
     pool = threading.Pool(4)
     publish_func = partial(publish_collection, module)
@@ -258,7 +263,9 @@ def run_module():
         r['build']['rc'] + r['publish']['rc'] for r in result['results']
     ))
 
-    module.exit_json(failed=failed, **result)
+    end = datetime.datetime.now()
+    delta = end - start
+    module.exit_json(failed=failed, end=str(end), delta=str(delta), **result)
 
 
 def main():

test/integration/targets/ansible-galaxy-collection/tasks/download.yml

@@ -5,7 +5,7 @@
     state: directory
 
 - name: download collection with multiple dependencies with --no-deps
-  command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 --no-deps -s pulp_v2 {{ galaxy_verbosity }}
+  command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 --no-deps -s galaxy_ng {{ galaxy_verbosity }}
   register: download_collection
   args:
     chdir: '{{ galaxy_dir }}/download'
@@ -34,7 +34,7 @@
     - (download_collection_actual.files[1].path | basename) in ['requirements.yml', 'parent_dep-parent_collection-1.0.0.tar.gz']
 
 - name: download collection with multiple dependencies
-  command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 -s pulp_v2 {{ galaxy_verbosity }}
+  command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 -s galaxy_ng {{ galaxy_verbosity }}
   register: download_collection
   args:
     chdir: '{{ galaxy_dir }}/download'

test/integration/targets/ansible-galaxy-collection/tasks/init.yml

@@ -5,6 +5,12 @@
     chdir: '{{ galaxy_dir }}/scratch'
   register: init_relative
 
+- name: create required runtime.yml
+  copy:
+    content: |
+      requires_ansible: '>=1.0.0'
+    dest: '{{ galaxy_dir }}/scratch/ansible_test/my_collection/meta/runtime.yml'
+
 - name: get result of create default skeleton
   find:
     path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection'

test/integration/targets/ansible-galaxy-collection/tasks/install.yml

@@ -168,7 +168,7 @@
 
 - name: Find artifact url for namespace3.name
   uri:
-    url: '{{ test_server }}{{ vX }}collections/namespace3/name/versions/1.0.0/'
+    url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/namespace3/name/versions/1.0.0/'
     user: '{{ pulp_user }}'
     password: '{{ pulp_password }}'
     force_basic_auth: true
@@ -218,7 +218,7 @@
     state: absent
 
 - assert:
-    that: error == expected_error
+    that: expected_error in error
   vars:
     error: "{{ result.stderr | regex_replace('\\n', ' ') }}"
     expected_error: >-
@@ -258,12 +258,14 @@
       ignore_errors: yes
       register: result
 
-    - debug: msg="Actual - {{ error }}"
+    - debug:
+        msg: "Actual - {{ error }}"
 
-    - debug: msg="Expected - {{ expected_error }}"
+    - debug:
+        msg: "Expected - {{ expected_error }}"
 
     - assert:
-        that: error == expected_error
+        that: expected_error in error
   always:
     - name: clean up collection skeleton and artifact
       file:
@@ -295,7 +297,7 @@
 
 - name: Find artifact url for namespace4.name
   uri:
-    url: '{{ test_server }}{{ vX }}collections/namespace4/name/versions/1.0.0/'
+    url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/namespace4/name/versions/1.0.0/'
     user: '{{ pulp_user }}'
     password: '{{ pulp_password }}'
     force_basic_auth: true
@@ -325,10 +327,11 @@
   environment:
     ANSIBLE_GALAXY_SERVER_LIST: undefined
 
-- when: not requires_auth
+# pulp_v2 doesn't require auth
+- when: v2|default(false)
   block:
     - name: install a collection with an empty server list - {{ test_id }}
-      command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' {{ galaxy_verbosity }}
+      command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' --api-version 2 {{ galaxy_verbosity }}
       register: install_empty_server_list
       environment:
         ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
@@ -571,7 +574,6 @@
   - namespace8
   - namespace9
 
-# SIVEL
 - name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_id }}
   assert:
     that:
@@ -697,39 +699,40 @@
     - namespace8
     - namespace9
 
-# Uncomment once pulp container is at pulp>=0.5.0
+- when: not v2|default(false)
-#- name: install cache.cache at the current latest version
+  block:
-#  command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' -vvv
+    - name: install cache.cache at the current latest version
-#  environment:
+      command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' -vvv
-#    ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
+      environment:
-#
+        ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
-#- set_fact:
+
-#    cache_version_build: '{{ (cache_version_build | int) + 1 }}'
+    - set_fact:
-#
+        cache_version_build: '{{ (cache_version_build | int) + 1 }}'
-#- name: publish update for cache.cache test
+
-#  setup_collections:
+    - name: publish update for cache.cache test
-#    server: galaxy_ng
+      setup_collections:
-#    collections:
+        server: galaxy_ng
-#    - namespace: cache
+        collections:
-#      name: cache
+        - namespace: cache
-#      version: 1.0.{{ cache_version_build }}
+          name: cache
-#
+          version: 1.0.{{ cache_version_build }}
-#- name: make sure the cache version list is ignored on a collection version change - {{ test_id }}
+
-#  command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv
+    - name: make sure the cache version list is ignored on a collection version change - {{ test_id }}
-#  register: install_cached_update
+      command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv
-#  environment:
+      register: install_cached_update
-#    ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
+      environment:
-#
+        ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
-#- name: get result of cache version list is ignored on a collection version change - {{ test_id }}
+
-#  slurp:
+    - name: get result of cache version list is ignored on a collection version change - {{ test_id }}
-#    path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json'
+      slurp:
-#  register: install_cached_update_actual
+        path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json'
-#
+      register: install_cached_update_actual
-#- name: assert cache version list is ignored on a collection version change - {{ test_id }}
+
-#  assert:
+    - name: assert cache version list is ignored on a collection version change - {{ test_id }}
-#    that:
+      assert:
-#    - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout'
+        that:
-#    - (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build
+        - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout'
+        - (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build
 
 - name: install collection with symlink - {{ test_id }}
   command: ansible-galaxy collection install symlink.symlink -s '{{ test_name }}' {{ galaxy_verbosity }}

test/integration/targets/ansible-galaxy-collection/tasks/install_offline.yml

@@ -25,6 +25,14 @@
         regexp: "^dependencies:*"
         line: "dependencies: {'ns.coll2': '>=1.0.0'}"
 
+    - name: create required runtime.yml
+      copy:
+        dest: "{{ galaxy_dir }}/offline/setup/ns/{{ item }}/meta/runtime.yml"
+        content: "requires_ansible: '>=1.0.0'"
+      loop:
+        - coll1
+        - coll2
+
     - name: build both collections
       command: ansible-galaxy collection build {{ init_dir }}/ns/{{ item }}
       args:

test/integration/targets/ansible-galaxy-collection/tasks/main.yml

@@ -72,13 +72,12 @@
   vars:
     test_name: '{{ item.name }}'
     test_server: '{{ item.server }}'
-    vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}'
+    test_api_server: '{{ item.api_server|default(item.server) }}'
   loop:
   - name: pulp_v2
-    server: '{{ pulp_server }}published/api/'
+    api_server: '{{ galaxy_ng_server }}'
-  - name: pulp_v3
+    server: '{{ pulp_server }}primary/api/'
-    server: '{{ pulp_server }}published/api/'
+    v2: true
-    v3: true
   - name: galaxy_ng
     server: '{{ galaxy_ng_server }}'
     v3: true
@@ -108,8 +107,9 @@
     test_id: '{{ item.name }}'
     test_name: '{{ item.name }}'
     test_server: '{{ item.server }}'
-    vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}'
+    test_api_server: '{{ item.api_server|default(item.server) }}'
     requires_auth: '{{ item.requires_auth|default(false) }}'
+    v2: '{{ item.v2|default(false) }}'
   args:
     apply:
       environment:
@@ -120,10 +120,9 @@
     v3: true
     requires_auth: true
   - name: pulp_v2
-    server: '{{ pulp_server }}published/api/'
+    server: '{{ pulp_server }}primary/api/'
-  - name: pulp_v3
+    api_server: '{{ galaxy_ng_server }}'
-    server: '{{ pulp_server }}published/api/'
+    v2: true
-    v3: true
 
 - name: test installing and downloading collections with the range of supported resolvelib versions
   include_tasks: supported_resolvelib.yml
@@ -176,13 +175,13 @@
       in install_cross_dep.stdout
     # pulp_v2 is highest in the list so it will find it there first
     - >-
-      "'parent_dep.parent_collection:1.0.0' obtained from server pulp_v2"
+      "'parent_dep.parent_collection:1.0.0' obtained from server galaxy_ng"
       in install_cross_dep.stdout
     - >-
-      "'child_dep.child_collection:0.9.9' obtained from server pulp_v2"
+      "'child_dep.child_collection:0.9.9' obtained from server galaxy_ng"
       in install_cross_dep.stdout
     - >-
-      "'child_dep.child_dep2:1.2.2' obtained from server pulp_v2"
+      "'child_dep.child_dep2:1.2.2' obtained from server galaxy_ng"
       in install_cross_dep.stdout
     - (install_cross_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0'
     - (install_cross_dep_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0'
@@ -204,10 +203,9 @@
         ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
         ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
   vars:
-    test_api_fallback: 'pulp_v2'
+    test_api_fallback: 'galaxy_ng'
-    test_api_fallback_versions: 'v1, v2'
+    test_api_fallback_versions: 'v3, pulp-v3, v1'
-    test_name: 'galaxy_ng'
+    test_name: 'pulp_v2'
-    test_server: '{{ galaxy_ng_server }}'
 
 - name: run ansible-galaxy collection list tests
   include_tasks: list.yml

test/integration/targets/ansible-galaxy-collection/tasks/publish.yml

@@ -5,9 +5,12 @@
     chdir: '{{ galaxy_dir }}'
   register: publish_collection
 
+- name: ensure we can download the published collection - {{ test_name }}
+  command: ansible-galaxy collection install -s {{ test_name }} -p "{{ remote_tmp_dir }}/publish/{{ test_name }}" ansible_test.my_collection==1.0.0 {{ galaxy_verbosity }}
+
 - name: get result of publish collection - {{ test_name }}
   uri:
-    url: '{{ test_server }}{{ vX }}collections/ansible_test/my_collection/versions/1.0.0/'
+    url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/ansible_test/my_collection/versions/1.0.0/'
     return_content: yes
     user: '{{ pulp_user }}'
     password: '{{ pulp_password }}'

test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml

@@ -20,11 +20,11 @@
 
     - include_tasks: install.yml
       vars:
-        test_name: pulp_v3
+        test_name: galaxy_ng
         test_id: '{{ test_name }} (resolvelib {{ resolvelib_version }})'
-        test_server: '{{ pulp_server }}published/api/'
+        test_server: '{{ galaxy_ng_server }}'
-        vX: "v3/"
+        test_api_server: '{{ galaxy_ng_server }}'
-        requires_auth: false
+        requires_auth: true
       args:
         apply:
           environment:

test/integration/targets/ansible-galaxy-collection/tasks/verify.yml

@@ -3,6 +3,11 @@
   args:
     chdir: '{{ galaxy_dir }}/scratch'
 
+- name: created required runtime.yml
+  copy:
+    content: 'requires_ansible: ">=1.0.0"'
+    dest: '{{ galaxy_dir }}/scratch/ansible_test/verify/meta/runtime.yml'
+
 - name: build the collection
   command: ansible-galaxy collection build scratch/ansible_test/verify
   args:
@@ -31,6 +36,9 @@
 - name: verify the collection against the first valid server
   command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -vvvv {{ galaxy_verbosity }}
   register: verify
+  vars:
+    # This sets a specific precedence that the tests are expecting
+    ANSIBLE_GALAXY_SERVER_LIST: offline,secondary,pulp_v2,galaxy_ng
 
 - assert:
     that:

test/integration/targets/ansible-galaxy-collection/templates/ansible.cfg.j2

@@ -1,28 +1,22 @@
 [galaxy]
 # Ensures subsequent unstable reruns don't use the cached information causing another failure
 cache_dir={{ remote_tmp_dir }}/galaxy_cache
-server_list=offline,pulp_v2,pulp_v3,galaxy_ng,secondary
+server_list=offline,galaxy_ng,secondary,pulp_v2
 
 [galaxy_server.offline]
 url={{ offline_server }}
 
 [galaxy_server.pulp_v2]
-url={{ pulp_server }}published/api/
+url={{ pulp_server }}primary/api/
-username={{ pulp_user }}
-password={{ pulp_password }}
-
-[galaxy_server.pulp_v3]
-url={{ pulp_server }}published/api/
-v3=true
 username={{ pulp_user }}
 password={{ pulp_password }}
+api_version=2
 
 [galaxy_server.galaxy_ng]
-url={{ galaxy_ng_server }}
+url={{ galaxy_ng_server }}content/primary/
 token={{ galaxy_ng_token.json.token }}
 
 [galaxy_server.secondary]
-url={{ pulp_server }}secondary/api/
+url={{ galaxy_ng_server }}content/secondary/
-v3=true
 username={{ pulp_user }}
 password={{ pulp_password }}

test/integration/targets/ansible-galaxy-collection/vars/main.yml

@@ -17,11 +17,12 @@ unsupported_resolvelib_versions:
   - "0.5.1"
 
 pulp_repositories:
-  - published
+  - primary
   - secondary
 
 publish_namespaces:
   - ansible_test
+  - secondary
 
 collection_list:
   # Scenario to test out pre-release being ignored unless explicitly set and version pagination.

test/lib/ansible_test/_internal/commands/integration/cloud/galaxy.py

@@ -10,12 +10,21 @@ from ....config import (
 
 from ....docker_util import (
     docker_cp_to,
+    docker_exec,
 )
 
 from ....containers import (
     run_support_container,
 )
 
+from ....encoding import (
+    to_text,
+)
+
+from ....util import (
+    display,
+)
+
 from . import (
     CloudEnvironment,
     CloudEnvironmentConfig,
@@ -23,53 +32,59 @@ from . import (
 )
 
 
-# We add BasicAuthentication, to make the tasks that deal with
+GALAXY_HOST_NAME = 'galaxy-pulp'
-# direct API access easier to deal with across galaxy_ng and pulp
+SETTINGS = {
-SETTINGS = '''
+    'PULP_CONTENT_ORIGIN': f'http://{GALAXY_HOST_NAME}',
-CONTENT_ORIGIN = 'http://{host}:80'
+    'PULP_ANSIBLE_API_HOSTNAME': f'http://{GALAXY_HOST_NAME}',
-ANSIBLE_API_HOSTNAME = 'http://{host}:80'
+    'PULP_GALAXY_API_PATH_PREFIX': '/api/galaxy/',
-ANSIBLE_CONTENT_HOSTNAME = 'http://{host}:80/pulp/content'
+    # These paths are unique to the container image which has an nginx location for /pulp/content to route
-TOKEN_AUTH_DISABLED = True
+    # requests to the content backend
-GALAXY_REQUIRE_CONTENT_APPROVAL = False
+    'PULP_ANSIBLE_CONTENT_HOSTNAME': f'http://{GALAXY_HOST_NAME}/pulp/content/api/galaxy/v3/artifacts/collections/',
-GALAXY_AUTHENTICATION_CLASSES = [
+    'PULP_CONTENT_PATH_PREFIX': '/pulp/content/api/galaxy/v3/artifacts/collections/',
-    "rest_framework.authentication.SessionAuthentication",
+    'PULP_GALAXY_AUTHENTICATION_CLASSES': [
-    "rest_framework.authentication.TokenAuthentication",
+        'rest_framework.authentication.SessionAuthentication',
-    "rest_framework.authentication.BasicAuthentication",
+        'rest_framework.authentication.TokenAuthentication',
-]
+        'rest_framework.authentication.BasicAuthentication',
-'''
+        'django.contrib.auth.backends.ModelBackend',
-
+    ],
-SET_ADMIN_PASSWORD = b'''#!/usr/bin/execlineb -S0
+    # This should probably be false see https://issues.redhat.com/browse/AAH-2328
-foreground {
+    'PULP_GALAXY_REQUIRE_CONTENT_APPROVAL': 'true',
-  redirfd -w 1 /dev/null
+    'PULP_GALAXY_DEPLOYMENT_MODE': 'standalone',
-  redirfd -w 2 /dev/null
+    'PULP_GALAXY_AUTO_SIGN_COLLECTIONS': 'false',
-  export DJANGO_SETTINGS_MODULE pulpcore.app.settings
+    'PULP_GALAXY_COLLECTION_SIGNING_SERVICE': 'ansible-default',
-  export PULP_CONTENT_ORIGIN localhost
+    'PULP_RH_ENTITLEMENT_REQUIRED': 'insights',
-  s6-setuidgid postgres
+    'PULP_TOKEN_AUTH_DISABLED': 'false',
-  if { /usr/local/bin/django-admin reset-admin-password --password password }
+    'PULP_TOKEN_SERVER': f'http://{GALAXY_HOST_NAME}/token/',
-  if { /usr/local/bin/pulpcore-manager create-group system:partner-engineers --users admin }
+    'PULP_TOKEN_SIGNATURE_ALGORITHM': 'ES256',
-}
+    'PULP_PUBLIC_KEY_PATH': '/src/galaxy_ng/dev/common/container_auth_public_key.pem',
-'''
+    'PULP_PRIVATE_KEY_PATH': '/src/galaxy_ng/dev/common/container_auth_private_key.pem',
-
+    'PULP_ANALYTICS': 'false',
-# There are 2 overrides here:
+    'PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_ACCESS': 'true',
-#   1. Change the gunicorn bind address from 127.0.0.1 to 0.0.0.0 now that Galaxy NG does not allow us to access the
+    'PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD': 'true',
-#      Pulp API through it.
+    'PULP_GALAXY_ENABLE_LEGACY_ROLES': 'true',
-#   2. Grant access allowing us to DELETE a namespace in Galaxy NG. This is as CI deletes and recreates repos and
+    'PULP_GALAXY_FEATURE_FLAGS__execution_environments': 'false',
-#      distributions in Pulp which now breaks the namespace in Galaxy NG. Recreating it is the "simple" fix to get it
+    'PULP_SOCIAL_AUTH_LOGIN_REDIRECT_URL': '/',
-#      working again.
+    'PULP_GALAXY_FEATURE_FLAGS__ai_deny_index': 'true',
-# These may not be needed in the future, especially if 1 becomes configurable by an env var but for now they must be
+    'PULP_DEFAULT_ADMIN_PASSWORD': 'password'
-# done.
-OVERRIDES = b'''#!/usr/bin/execlineb -S0
-foreground {
-    sed -i "0,/\\"127.0.0.1:24817\\"/s//\\"0.0.0.0:24817\\"/" /etc/services.d/pulpcore-api/run
 }
 
-# This sed calls changes the first occurrence to "allow" which is conveniently the delete operation for a namespace.
+
-# https://github.com/ansible/galaxy_ng/blob/master/galaxy_ng/app/access_control/statements/standalone.py#L9-L11.
+GALAXY_IMPORTER = b'''
-backtick NG_PREFIX { python -c "import galaxy_ng; print(galaxy_ng.__path__[0], end='')" }
+[galaxy-importer]
-importas ng_prefix NG_PREFIX
+ansible_local_tmp=~/.ansible/tmp
-foreground {
+ansible_test_local_image=false
-    sed -i "0,/\\"effect\\": \\"deny\\"/s//\\"effect\\": \\"allow\\"/" ${ng_prefix}/app/access_control/statements/standalone.py
+check_required_tags=false
-}'''
+check_runtime_yaml=false
+check_changelog=false
+infra_osd=false
+local_image_docker=false
+log_level_main=INFO
+require_v1_or_greater=false
+run_ansible_doc=false
+run_ansible_lint=false
+run_ansible_test=false
+run_flake8=false
+'''.strip()
 
 
 class GalaxyProvider(CloudProvider):
@@ -81,13 +96,9 @@ class GalaxyProvider(CloudProvider):
     def __init__(self, args: IntegrationConfig) -> None:
         super().__init__(args)
 
-        # Cannot use the latest container image as either galaxy_ng 4.2.0rc2 or pulp 0.5.0 has sporatic issues with
+        self.image = os.environ.get(
-        # dropping published collections in CI. Try running the tests multiple times when updating. Will also need to
-        # comment out the cache tests in 'test/integration/targets/ansible-galaxy-collection/tasks/install.yml' when
-        # the newer update is available.
-        self.pulp = os.environ.get(
             'ANSIBLE_PULP_CONTAINER',
-            'quay.io/ansible/pulp-galaxy-ng:b79a7be64eff'
+            'quay.io/pulp/galaxy:4.7.1'
         )
 
         self.uses_docker = True
@@ -96,46 +107,45 @@ class GalaxyProvider(CloudProvider):
         """Setup cloud resource before delegation and reg cleanup callback."""
         super().setup()
 
-        galaxy_port = 80
+        with tempfile.NamedTemporaryFile(mode='w+') as env_fd:
-        pulp_port = 24817
+            settings = '\n'.join(
-
+                f'{key}={value}' for key, value in SETTINGS.items()
-        ports = [
+            )
-            galaxy_port,
+            env_fd.write(settings)
-            pulp_port,
+            env_fd.flush()
-        ]
+            display.info(f'>>> galaxy_ng Configuration\n{settings}', verbosity=3)
-
+            descriptor = run_support_container(
-        # Create the container, don't run it, we need to inject configs before it starts
+                self.args,
-        descriptor = run_support_container(
+                self.platform,
-            self.args,
+                self.image,
-            self.platform,
+                GALAXY_HOST_NAME,
-            self.pulp,
+                [
-            'galaxy-pulp',
+                    80,
-            ports,
+                ],
-            start=False,
+                aliases=[
-        )
+                    GALAXY_HOST_NAME,
+                ],
+                start=True,
+                options=[
+                    '--env-file', env_fd.name,
+                ],
+            )
 
         if not descriptor:
             return
 
-        pulp_id = descriptor.container_id
+        injected_files = [
-
+            ('/etc/galaxy-importer/galaxy-importer.cfg', GALAXY_IMPORTER, 'galaxy-importer'),
-        injected_files = {
+        ]
-            '/etc/pulp/settings.py': SETTINGS.format(host=descriptor.name).encode(),
+        for path, content, friendly_name in injected_files:
-            '/etc/cont-init.d/111-postgres': SET_ADMIN_PASSWORD,
-            '/etc/cont-init.d/000-ansible-test-overrides': OVERRIDES,
-        }
-
-        for path, content in injected_files.items():
             with tempfile.NamedTemporaryFile() as temp_fd:
                 temp_fd.write(content)
                 temp_fd.flush()
-                docker_cp_to(self.args, pulp_id, temp_fd.name, path)
+                display.info(f'>>> {friendly_name} Configuration\n{to_text(content)}', verbosity=3)
-
+                docker_exec(self.args, descriptor.container_id, ['mkdir', '-p', os.path.dirname(path)], True)
-        descriptor.start(self.args)
+                docker_cp_to(self.args, descriptor.container_id, temp_fd.name, path)
 
-        self._set_cloud_config('PULP_HOST', descriptor.name)
+        self._set_cloud_config('PULP_HOST', GALAXY_HOST_NAME)
-        self._set_cloud_config('PULP_PORT', str(pulp_port))
-        self._set_cloud_config('GALAXY_PORT', str(galaxy_port))
         self._set_cloud_config('PULP_USER', 'admin')
         self._set_cloud_config('PULP_PASSWORD', 'password')
 
@@ -148,21 +158,19 @@ class GalaxyEnvironment(CloudEnvironment):
         pulp_user = str(self._get_cloud_config('PULP_USER'))
         pulp_password = str(self._get_cloud_config('PULP_PASSWORD'))
         pulp_host = self._get_cloud_config('PULP_HOST')
-        galaxy_port = self._get_cloud_config('GALAXY_PORT')
-        pulp_port = self._get_cloud_config('PULP_PORT')
 
         return CloudEnvironmentConfig(
             ansible_vars=dict(
                 pulp_user=pulp_user,
                 pulp_password=pulp_password,
-                pulp_api='http://%s:%s' % (pulp_host, pulp_port),
+                pulp_api=f'http://{pulp_host}',
-                pulp_server='http://%s:%s/pulp_ansible/galaxy/' % (pulp_host, pulp_port),
+                pulp_server=f'http://{pulp_host}/pulp_ansible/galaxy/',
-                galaxy_ng_server='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port),
+                galaxy_ng_server=f'http://{pulp_host}/api/galaxy/',
             ),
             env_vars=dict(
                 PULP_USER=pulp_user,
                 PULP_PASSWORD=pulp_password,
-                PULP_SERVER='http://%s:%s/pulp_ansible/galaxy/api/' % (pulp_host, pulp_port),
+                PULP_SERVER=f'http://{pulp_host}/pulp_ansible/galaxy/api/',
-                GALAXY_NG_SERVER='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port),
+                GALAXY_NG_SERVER=f'http://{pulp_host}/api/galaxy/',
             ),
         )

test/units/galaxy/test_collection.py

@@ -223,23 +223,19 @@ def test_cli_options(required_signature_count, valid, monkeypatch):
             {
                 'url': 'https://galaxy.ansible.com',
                 'validate_certs': 'False',
-                'v3': 'False',
             },
             # Expected server attributes
             {
                 'validate_certs': False,
-                '_available_api_versions': {},
             },
         ),
         (
             {
                 'url': 'https://galaxy.ansible.com',
                 'validate_certs': 'True',
-                'v3': 'True',
             },
             {
                 'validate_certs': True,
-                '_available_api_versions': {'v3': '/v3'},
             },
         ),
     ],
@@ -257,7 +253,6 @@ def test_bool_type_server_config_options(config, server, monkeypatch):
         "server_list=server1\n",
         "[galaxy_server.server1]",
         "url=%s" % config['url'],
-        "v3=%s" % config['v3'],
         "validate_certs=%s\n" % config['validate_certs'],
     ]
 
@@ -277,7 +272,6 @@ def test_bool_type_server_config_options(config, server, monkeypatch):
 
     assert galaxy_cli.api_servers[0].name == 'server1'
     assert galaxy_cli.api_servers[0].validate_certs == server['validate_certs']
-    assert galaxy_cli.api_servers[0]._available_api_versions == server['_available_api_versions']
 
 
 @pytest.mark.parametrize('global_ignore_certs', [True, False])