Update galaxy_ng container for tests (#80721)

pull/80584/head
Matt Martz 1 year ago committed by GitHub
parent af84f7a00e
commit 92d5ffda82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,2 @@
minor_changes:
- ansible-galaxy - Remove internal configuration argument ``v3`` (https://github.com/ansible/ansible/pull/80721)

@ -10,6 +10,7 @@ __metaclass__ = type
# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first # ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
from ansible.cli import CLI from ansible.cli import CLI
import argparse
import json import json
import os.path import os.path
import pathlib import pathlib
@ -71,7 +72,7 @@ SERVER_DEF = [
('password', False, 'str'), ('password', False, 'str'),
('token', False, 'str'), ('token', False, 'str'),
('auth_url', False, 'str'), ('auth_url', False, 'str'),
('v3', False, 'bool'), ('api_version', False, 'int'),
('validate_certs', False, 'bool'), ('validate_certs', False, 'bool'),
('client_id', False, 'str'), ('client_id', False, 'str'),
('timeout', False, 'int'), ('timeout', False, 'int'),
@ -79,7 +80,7 @@ SERVER_DEF = [
# config definition fields # config definition fields
SERVER_ADDITIONAL = { SERVER_ADDITIONAL = {
'v3': {'default': 'False'}, 'api_version': {'default': None, 'choices': [2, 3]},
'validate_certs': {'cli': [{'name': 'validate_certs'}]}, 'validate_certs': {'cli': [{'name': 'validate_certs'}]},
'timeout': {'default': '60', 'cli': [{'name': 'timeout'}]}, 'timeout': {'default': '60', 'cli': [{'name': 'timeout'}]},
'token': {'default': None}, 'token': {'default': None},
@ -240,6 +241,7 @@ class GalaxyCLI(CLI):
# Common arguments that apply to more than 1 action # Common arguments that apply to more than 1 action
common = opt_help.ArgumentParser(add_help=False) common = opt_help.ArgumentParser(add_help=False)
common.add_argument('-s', '--server', dest='api_server', help='The Galaxy API server URL') common.add_argument('-s', '--server', dest='api_server', help='The Galaxy API server URL')
common.add_argument('--api-version', type=int, choices=[2, 3], help=argparse.SUPPRESS) # Hidden argument that should only be used in our tests
common.add_argument('--token', '--api-key', dest='api_key', common.add_argument('--token', '--api-key', dest='api_key',
help='The Ansible Galaxy API key which can be found at ' help='The Ansible Galaxy API key which can be found at '
'https://galaxy.ansible.com/me/preferences.') 'https://galaxy.ansible.com/me/preferences.')
@ -644,17 +646,22 @@ class GalaxyCLI(CLI):
client_id = server_options.pop('client_id') client_id = server_options.pop('client_id')
token_val = server_options['token'] or NoTokenSentinel token_val = server_options['token'] or NoTokenSentinel
username = server_options['username'] username = server_options['username']
v3 = server_options.pop('v3') api_version = server_options.pop('api_version')
if server_options['validate_certs'] is None: if server_options['validate_certs'] is None:
server_options['validate_certs'] = context.CLIARGS['resolved_validate_certs'] server_options['validate_certs'] = context.CLIARGS['resolved_validate_certs']
validate_certs = server_options['validate_certs'] validate_certs = server_options['validate_certs']
if v3: # This allows a user to explicitly force use of an API version when
# This allows a user to explicitly indicate the server uses the /v3 API # multiple versions are supported. This was added for testing
# This was added for testing against pulp_ansible and I'm not sure it has # against pulp_ansible and I'm not sure it has a practical purpose
# a practical purpose outside of this use case. As such, this option is not # outside of this use case. As such, this option is not documented
# documented as of now # as of now
server_options['available_api_versions'] = {'v3': '/v3'} if api_version:
display.warning(
f'The specified "api_version" configuration for the galaxy server "{server_key}" is '
'not a public configuration, and may be removed at any time without warning.'
)
server_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version}
# default case if no auth info is provided. # default case if no auth info is provided.
server_options['token'] = None server_options['token'] = None
@ -680,6 +687,13 @@ class GalaxyCLI(CLI):
)) ))
cmd_server = context.CLIARGS['api_server'] cmd_server = context.CLIARGS['api_server']
if context.CLIARGS['api_version']:
api_version = context.CLIARGS['api_version']
display.warning(
'The --api-version is not a public argument, and may be removed at any time without warning.'
)
galaxy_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version}
cmd_token = GalaxyToken(token=context.CLIARGS['api_key']) cmd_token = GalaxyToken(token=context.CLIARGS['api_key'])
validate_certs = context.CLIARGS['resolved_validate_certs'] validate_certs = context.CLIARGS['resolved_validate_certs']

@ -923,10 +923,7 @@ class GalaxyAPI:
data = self._call_galaxy(n_collection_url, error_context_msg=error_context_msg, cache=True) data = self._call_galaxy(n_collection_url, error_context_msg=error_context_msg, cache=True)
self._set_cache() self._set_cache()
try: signatures = [signature_info["signature"] for signature_info in data.get("signatures") or []]
signatures = data["signatures"] if not signatures:
except KeyError:
display.vvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}") display.vvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}")
return [] return signatures
else:
return [signature_info["signature"] for signature_info in signatures]

@ -84,7 +84,8 @@ def invoke_api(module, url, method='GET', data=None, status_codes=None):
resp, info = fetch_url(module, url, method=method, data=data, headers=headers) resp, info = fetch_url(module, url, method=method, data=data, headers=headers)
if info['status'] not in status_codes: if info['status'] not in status_codes:
module.fail_json(url=url, **info) info['url'] = url
module.fail_json(**info)
data = to_text(resp.read()) data = to_text(resp.read())
if data: if data:
@ -105,7 +106,7 @@ def delete_pulp_distribution(distribution, module):
def delete_pulp_orphans(module): def delete_pulp_orphans(module):
""" Deletes any orphaned pulp objects. """ """ Deletes any orphaned pulp objects. """
orphan_uri = module.params['pulp_api'] + '/pulp/api/v3/orphans/' orphan_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/orphans/'
task_info = invoke_api(module, orphan_uri, method='DELETE', status_codes=[202]) task_info = invoke_api(module, orphan_uri, method='DELETE', status_codes=[202])
wait_pulp_task(task_info['task'], module) wait_pulp_task(task_info['task'], module)
@ -125,25 +126,39 @@ def get_galaxy_namespaces(module):
return [n['name'] for n in ns_info['data']] return [n['name'] for n in ns_info['data']]
def get_pulp_distributions(module): def get_pulp_distributions(module, distribution):
""" Gets a list of all the pulp distributions. """ """ Gets a list of all the pulp distributions. """
distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/' distro_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/distributions/ansible/ansible/'
distro_info = invoke_api(module, distro_uri) distro_info = invoke_api(module, distro_uri + '?name=' + distribution)
return [module.params['pulp_api'] + r['pulp_href'] for r in distro_info['results']] return [module.params['pulp_api'] + r['pulp_href'] for r in distro_info['results']]
def get_pulp_repositories(module): def get_pulp_repositories(module, repository):
""" Gets a list of all the pulp repositories. """ """ Gets a list of all the pulp repositories. """
repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/' repo_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/repositories/ansible/ansible/'
repo_info = invoke_api(module, repo_uri) repo_info = invoke_api(module, repo_uri + '?name=' + repository)
return [module.params['pulp_api'] + r['pulp_href'] for r in repo_info['results']] return [module.params['pulp_api'] + r['pulp_href'] for r in repo_info['results']]
def get_repo_collections(repository, module):
collections_uri = module.params['galaxy_ng_server'] + 'v3/plugin/ansible/content/' + repository + '/collections/index/'
# status code 500 isn't really expected, an unhandled exception is causing this instead of a 404
# See https://issues.redhat.com/browse/AAH-2329
info = invoke_api(module, collections_uri + '?limit=100&offset=0', status_codes=[200, 500])
if not info:
return []
return [module.params['pulp_api'] + c['href'] for c in info['data']]
def delete_repo_collection(collection, module):
task_info = invoke_api(module, collection, method='DELETE', status_codes=[202])
wait_pulp_task(task_info['task'], module)
def new_galaxy_namespace(name, module): def new_galaxy_namespace(name, module):
""" Creates a new namespace in Galaxy NG. """ """ Creates a new namespace in Galaxy NG. """
ns_uri = module.params['galaxy_ng_server'] + 'v3/_ui/namespaces/' ns_uri = module.params['galaxy_ng_server'] + 'v3/namespaces/ '
data = {'name': name, 'groups': [{'name': 'system:partner-engineers', 'object_permissions': data = {'name': name, 'groups': []}
['add_namespace', 'change_namespace', 'upload_to_namespace']}]}
ns_info = invoke_api(module, ns_uri, method='POST', data=data, status_codes=[201]) ns_info = invoke_api(module, ns_uri, method='POST', data=data, status_codes=[201])
return ns_info['id'] return ns_info['id']
@ -151,16 +166,17 @@ def new_galaxy_namespace(name, module):
def new_pulp_repository(name, module): def new_pulp_repository(name, module):
""" Creates a new pulp repository. """ """ Creates a new pulp repository. """
repo_uri = module.params['pulp_api'] + '/pulp/api/v3/repositories/ansible/ansible/' repo_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/repositories/ansible/ansible/'
data = {'name': name} # retain_repo_versions to work around https://issues.redhat.com/browse/AAH-2332
data = {'name': name, 'retain_repo_versions': '1024'}
repo_info = invoke_api(module, repo_uri, method='POST', data=data, status_codes=[201]) repo_info = invoke_api(module, repo_uri, method='POST', data=data, status_codes=[201])
return module.params['pulp_api'] + repo_info['pulp_href'] return repo_info['pulp_href']
def new_pulp_distribution(name, base_path, repository, module): def new_pulp_distribution(name, base_path, repository, module):
""" Creates a new pulp distribution for a repository. """ """ Creates a new pulp distribution for a repository. """
distro_uri = module.params['pulp_api'] + '/pulp/api/v3/distributions/ansible/ansible/' distro_uri = module.params['galaxy_ng_server'] + 'pulp/api/v3/distributions/ansible/ansible/'
data = {'name': name, 'base_path': base_path, 'repository': repository} data = {'name': name, 'base_path': base_path, 'repository': repository}
task_info = invoke_api(module, distro_uri, method='POST', data=data, status_codes=[202]) task_info = invoke_api(module, distro_uri, method='POST', data=data, status_codes=[202])
task_info = wait_pulp_task(task_info['task'], module) task_info = wait_pulp_task(task_info['task'], module)
@ -194,8 +210,15 @@ def main():
) )
module.params['force_basic_auth'] = True module.params['force_basic_auth'] = True
[delete_pulp_distribution(d, module) for d in get_pulp_distributions(module)] # It may be due to the process of cleaning up orphans, but we cannot delete the namespace
[delete_pulp_repository(r, module) for r in get_pulp_repositories(module)] # while a collection still exists, so this is just a new safety to nuke all collections
# first
for repository in module.params['repositories']:
[delete_repo_collection(c, module) for c in get_repo_collections(repository, module)]
for repository in module.params['repositories']:
[delete_pulp_distribution(d, module) for d in get_pulp_distributions(module, repository)]
[delete_pulp_repository(r, module) for r in get_pulp_repositories(module, repository)]
delete_pulp_orphans(module) delete_pulp_orphans(module)
[delete_galaxy_namespace(n, module) for n in get_galaxy_namespaces(module)] [delete_galaxy_namespace(n, module) for n in get_galaxy_namespaces(module)]

@ -77,6 +77,7 @@ RETURN = '''
# #
''' '''
import datetime
import os import os
import subprocess import subprocess
import tarfile import tarfile
@ -104,6 +105,7 @@ def publish_collection(module, collection):
collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (namespace, name, version)) collection_dir = os.path.join(module.tmpdir, "%s-%s-%s" % (namespace, name, version))
b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict') b_collection_dir = to_bytes(collection_dir, errors='surrogate_or_strict')
os.mkdir(b_collection_dir) os.mkdir(b_collection_dir)
os.mkdir(os.path.join(b_collection_dir, b'meta'))
with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd: with open(os.path.join(b_collection_dir, b'README.md'), mode='wb') as fd:
fd.write(b"Collection readme") fd.write(b"Collection readme")
@ -120,6 +122,8 @@ def publish_collection(module, collection):
} }
with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd: with open(os.path.join(b_collection_dir, b'galaxy.yml'), mode='wb') as fd:
fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict')) fd.write(to_bytes(yaml.safe_dump(galaxy_meta), errors='surrogate_or_strict'))
with open(os.path.join(b_collection_dir, b'meta/runtime.yml'), mode='wb') as fd:
fd.write(b'requires_ansible: ">=1.0.0"')
with tempfile.NamedTemporaryFile(mode='wb') as temp_fd: with tempfile.NamedTemporaryFile(mode='wb') as temp_fd:
temp_fd.write(b"data") temp_fd.write(b"data")
@ -241,7 +245,8 @@ def run_module():
supports_check_mode=False supports_check_mode=False
) )
result = dict(changed=True, results=[]) start = datetime.datetime.now()
result = dict(changed=True, results=[], start=str(start))
pool = threading.Pool(4) pool = threading.Pool(4)
publish_func = partial(publish_collection, module) publish_func = partial(publish_collection, module)
@ -258,7 +263,9 @@ def run_module():
r['build']['rc'] + r['publish']['rc'] for r in result['results'] r['build']['rc'] + r['publish']['rc'] for r in result['results']
)) ))
module.exit_json(failed=failed, **result) end = datetime.datetime.now()
delta = end - start
module.exit_json(failed=failed, end=str(end), delta=str(delta), **result)
def main(): def main():

@ -5,7 +5,7 @@
state: directory state: directory
- name: download collection with multiple dependencies with --no-deps - name: download collection with multiple dependencies with --no-deps
command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 --no-deps -s pulp_v2 {{ galaxy_verbosity }} command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 --no-deps -s galaxy_ng {{ galaxy_verbosity }}
register: download_collection register: download_collection
args: args:
chdir: '{{ galaxy_dir }}/download' chdir: '{{ galaxy_dir }}/download'
@ -34,7 +34,7 @@
- (download_collection_actual.files[1].path | basename) in ['requirements.yml', 'parent_dep-parent_collection-1.0.0.tar.gz'] - (download_collection_actual.files[1].path | basename) in ['requirements.yml', 'parent_dep-parent_collection-1.0.0.tar.gz']
- name: download collection with multiple dependencies - name: download collection with multiple dependencies
command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 -s pulp_v2 {{ galaxy_verbosity }} command: ansible-galaxy collection download parent_dep.parent_collection:1.0.0 -s galaxy_ng {{ galaxy_verbosity }}
register: download_collection register: download_collection
args: args:
chdir: '{{ galaxy_dir }}/download' chdir: '{{ galaxy_dir }}/download'

@ -5,6 +5,12 @@
chdir: '{{ galaxy_dir }}/scratch' chdir: '{{ galaxy_dir }}/scratch'
register: init_relative register: init_relative
- name: create required runtime.yml
copy:
content: |
requires_ansible: '>=1.0.0'
dest: '{{ galaxy_dir }}/scratch/ansible_test/my_collection/meta/runtime.yml'
- name: get result of create default skeleton - name: get result of create default skeleton
find: find:
path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection' path: '{{ galaxy_dir }}/scratch/ansible_test/my_collection'

@ -168,7 +168,7 @@
- name: Find artifact url for namespace3.name - name: Find artifact url for namespace3.name
uri: uri:
url: '{{ test_server }}{{ vX }}collections/namespace3/name/versions/1.0.0/' url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/namespace3/name/versions/1.0.0/'
user: '{{ pulp_user }}' user: '{{ pulp_user }}'
password: '{{ pulp_password }}' password: '{{ pulp_password }}'
force_basic_auth: true force_basic_auth: true
@ -218,7 +218,7 @@
state: absent state: absent
- assert: - assert:
that: error == expected_error that: expected_error in error
vars: vars:
error: "{{ result.stderr | regex_replace('\\n', ' ') }}" error: "{{ result.stderr | regex_replace('\\n', ' ') }}"
expected_error: >- expected_error: >-
@ -258,12 +258,14 @@
ignore_errors: yes ignore_errors: yes
register: result register: result
- debug: msg="Actual - {{ error }}" - debug:
msg: "Actual - {{ error }}"
- debug: msg="Expected - {{ expected_error }}" - debug:
msg: "Expected - {{ expected_error }}"
- assert: - assert:
that: error == expected_error that: expected_error in error
always: always:
- name: clean up collection skeleton and artifact - name: clean up collection skeleton and artifact
file: file:
@ -295,7 +297,7 @@
- name: Find artifact url for namespace4.name - name: Find artifact url for namespace4.name
uri: uri:
url: '{{ test_server }}{{ vX }}collections/namespace4/name/versions/1.0.0/' url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/namespace4/name/versions/1.0.0/'
user: '{{ pulp_user }}' user: '{{ pulp_user }}'
password: '{{ pulp_password }}' password: '{{ pulp_password }}'
force_basic_auth: true force_basic_auth: true
@ -325,10 +327,11 @@
environment: environment:
ANSIBLE_GALAXY_SERVER_LIST: undefined ANSIBLE_GALAXY_SERVER_LIST: undefined
- when: not requires_auth # pulp_v2 doesn't require auth
- when: v2|default(false)
block: block:
- name: install a collection with an empty server list - {{ test_id }} - name: install a collection with an empty server list - {{ test_id }}
command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' {{ galaxy_verbosity }} command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' --api-version 2 {{ galaxy_verbosity }}
register: install_empty_server_list register: install_empty_server_list
environment: environment:
ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
@ -571,7 +574,6 @@
- namespace8 - namespace8
- namespace9 - namespace9
# SIVEL
- name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_id }} - name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_id }}
assert: assert:
that: that:
@ -697,39 +699,40 @@
- namespace8 - namespace8
- namespace9 - namespace9
# Uncomment once pulp container is at pulp>=0.5.0 - when: not v2|default(false)
#- name: install cache.cache at the current latest version block:
# command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' -vvv - name: install cache.cache at the current latest version
# environment: command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' -vvv
# ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' environment:
# ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
#- set_fact:
# cache_version_build: '{{ (cache_version_build | int) + 1 }}' - set_fact:
# cache_version_build: '{{ (cache_version_build | int) + 1 }}'
#- name: publish update for cache.cache test
# setup_collections: - name: publish update for cache.cache test
# server: galaxy_ng setup_collections:
# collections: server: galaxy_ng
# - namespace: cache collections:
# name: cache - namespace: cache
# version: 1.0.{{ cache_version_build }} name: cache
# version: 1.0.{{ cache_version_build }}
#- name: make sure the cache version list is ignored on a collection version change - {{ test_id }}
# command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv - name: make sure the cache version list is ignored on a collection version change - {{ test_id }}
# register: install_cached_update command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv
# environment: register: install_cached_update
# ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' environment:
# ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections'
#- name: get result of cache version list is ignored on a collection version change - {{ test_id }}
# slurp: - name: get result of cache version list is ignored on a collection version change - {{ test_id }}
# path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json' slurp:
# register: install_cached_update_actual path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json'
# register: install_cached_update_actual
#- name: assert cache version list is ignored on a collection version change - {{ test_id }}
# assert: - name: assert cache version list is ignored on a collection version change - {{ test_id }}
# that: assert:
# - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout' that:
# - (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout'
- (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build
- name: install collection with symlink - {{ test_id }} - name: install collection with symlink - {{ test_id }}
command: ansible-galaxy collection install symlink.symlink -s '{{ test_name }}' {{ galaxy_verbosity }} command: ansible-galaxy collection install symlink.symlink -s '{{ test_name }}' {{ galaxy_verbosity }}

@ -25,6 +25,14 @@
regexp: "^dependencies:*" regexp: "^dependencies:*"
line: "dependencies: {'ns.coll2': '>=1.0.0'}" line: "dependencies: {'ns.coll2': '>=1.0.0'}"
- name: create required runtime.yml
copy:
dest: "{{ galaxy_dir }}/offline/setup/ns/{{ item }}/meta/runtime.yml"
content: "requires_ansible: '>=1.0.0'"
loop:
- coll1
- coll2
- name: build both collections - name: build both collections
command: ansible-galaxy collection build {{ init_dir }}/ns/{{ item }} command: ansible-galaxy collection build {{ init_dir }}/ns/{{ item }}
args: args:

@ -72,13 +72,12 @@
vars: vars:
test_name: '{{ item.name }}' test_name: '{{ item.name }}'
test_server: '{{ item.server }}' test_server: '{{ item.server }}'
vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}' test_api_server: '{{ item.api_server|default(item.server) }}'
loop: loop:
- name: pulp_v2 - name: pulp_v2
server: '{{ pulp_server }}published/api/' api_server: '{{ galaxy_ng_server }}'
- name: pulp_v3 server: '{{ pulp_server }}primary/api/'
server: '{{ pulp_server }}published/api/' v2: true
v3: true
- name: galaxy_ng - name: galaxy_ng
server: '{{ galaxy_ng_server }}' server: '{{ galaxy_ng_server }}'
v3: true v3: true
@ -108,8 +107,9 @@
test_id: '{{ item.name }}' test_id: '{{ item.name }}'
test_name: '{{ item.name }}' test_name: '{{ item.name }}'
test_server: '{{ item.server }}' test_server: '{{ item.server }}'
vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}' test_api_server: '{{ item.api_server|default(item.server) }}'
requires_auth: '{{ item.requires_auth|default(false) }}' requires_auth: '{{ item.requires_auth|default(false) }}'
v2: '{{ item.v2|default(false) }}'
args: args:
apply: apply:
environment: environment:
@ -120,10 +120,9 @@
v3: true v3: true
requires_auth: true requires_auth: true
- name: pulp_v2 - name: pulp_v2
server: '{{ pulp_server }}published/api/' server: '{{ pulp_server }}primary/api/'
- name: pulp_v3 api_server: '{{ galaxy_ng_server }}'
server: '{{ pulp_server }}published/api/' v2: true
v3: true
- name: test installing and downloading collections with the range of supported resolvelib versions - name: test installing and downloading collections with the range of supported resolvelib versions
include_tasks: supported_resolvelib.yml include_tasks: supported_resolvelib.yml
@ -176,13 +175,13 @@
in install_cross_dep.stdout in install_cross_dep.stdout
# pulp_v2 is highest in the list so it will find it there first # pulp_v2 is highest in the list so it will find it there first
- >- - >-
"'parent_dep.parent_collection:1.0.0' obtained from server pulp_v2" "'parent_dep.parent_collection:1.0.0' obtained from server galaxy_ng"
in install_cross_dep.stdout in install_cross_dep.stdout
- >- - >-
"'child_dep.child_collection:0.9.9' obtained from server pulp_v2" "'child_dep.child_collection:0.9.9' obtained from server galaxy_ng"
in install_cross_dep.stdout in install_cross_dep.stdout
- >- - >-
"'child_dep.child_dep2:1.2.2' obtained from server pulp_v2" "'child_dep.child_dep2:1.2.2' obtained from server galaxy_ng"
in install_cross_dep.stdout in install_cross_dep.stdout
- (install_cross_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' - (install_cross_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0'
- (install_cross_dep_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' - (install_cross_dep_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0'
@ -204,10 +203,9 @@
ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}' ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}'
ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
vars: vars:
test_api_fallback: 'pulp_v2' test_api_fallback: 'galaxy_ng'
test_api_fallback_versions: 'v1, v2' test_api_fallback_versions: 'v3, pulp-v3, v1'
test_name: 'galaxy_ng' test_name: 'pulp_v2'
test_server: '{{ galaxy_ng_server }}'
- name: run ansible-galaxy collection list tests - name: run ansible-galaxy collection list tests
include_tasks: list.yml include_tasks: list.yml

@ -5,9 +5,12 @@
chdir: '{{ galaxy_dir }}' chdir: '{{ galaxy_dir }}'
register: publish_collection register: publish_collection
- name: ensure we can download the published collection - {{ test_name }}
command: ansible-galaxy collection install -s {{ test_name }} -p "{{ remote_tmp_dir }}/publish/{{ test_name }}" ansible_test.my_collection==1.0.0 {{ galaxy_verbosity }}
- name: get result of publish collection - {{ test_name }} - name: get result of publish collection - {{ test_name }}
uri: uri:
url: '{{ test_server }}{{ vX }}collections/ansible_test/my_collection/versions/1.0.0/' url: '{{ test_api_server }}v3/plugin/ansible/content/primary/collections/index/ansible_test/my_collection/versions/1.0.0/'
return_content: yes return_content: yes
user: '{{ pulp_user }}' user: '{{ pulp_user }}'
password: '{{ pulp_password }}' password: '{{ pulp_password }}'

@ -20,11 +20,11 @@
- include_tasks: install.yml - include_tasks: install.yml
vars: vars:
test_name: pulp_v3 test_name: galaxy_ng
test_id: '{{ test_name }} (resolvelib {{ resolvelib_version }})' test_id: '{{ test_name }} (resolvelib {{ resolvelib_version }})'
test_server: '{{ pulp_server }}published/api/' test_server: '{{ galaxy_ng_server }}'
vX: "v3/" test_api_server: '{{ galaxy_ng_server }}'
requires_auth: false requires_auth: true
args: args:
apply: apply:
environment: environment:

@ -3,6 +3,11 @@
args: args:
chdir: '{{ galaxy_dir }}/scratch' chdir: '{{ galaxy_dir }}/scratch'
- name: created required runtime.yml
copy:
content: 'requires_ansible: ">=1.0.0"'
dest: '{{ galaxy_dir }}/scratch/ansible_test/verify/meta/runtime.yml'
- name: build the collection - name: build the collection
command: ansible-galaxy collection build scratch/ansible_test/verify command: ansible-galaxy collection build scratch/ansible_test/verify
args: args:
@ -31,6 +36,9 @@
- name: verify the collection against the first valid server - name: verify the collection against the first valid server
command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -vvvv {{ galaxy_verbosity }} command: ansible-galaxy collection verify ansible_test.verify:1.0.0 -vvvv {{ galaxy_verbosity }}
register: verify register: verify
vars:
# This sets a specific precedence that the tests are expecting
ANSIBLE_GALAXY_SERVER_LIST: offline,secondary,pulp_v2,galaxy_ng
- assert: - assert:
that: that:

@ -1,28 +1,22 @@
[galaxy] [galaxy]
# Ensures subsequent unstable reruns don't use the cached information causing another failure # Ensures subsequent unstable reruns don't use the cached information causing another failure
cache_dir={{ remote_tmp_dir }}/galaxy_cache cache_dir={{ remote_tmp_dir }}/galaxy_cache
server_list=offline,pulp_v2,pulp_v3,galaxy_ng,secondary server_list=offline,galaxy_ng,secondary,pulp_v2
[galaxy_server.offline] [galaxy_server.offline]
url={{ offline_server }} url={{ offline_server }}
[galaxy_server.pulp_v2] [galaxy_server.pulp_v2]
url={{ pulp_server }}published/api/ url={{ pulp_server }}primary/api/
username={{ pulp_user }}
password={{ pulp_password }}
[galaxy_server.pulp_v3]
url={{ pulp_server }}published/api/
v3=true
username={{ pulp_user }} username={{ pulp_user }}
password={{ pulp_password }} password={{ pulp_password }}
api_version=2
[galaxy_server.galaxy_ng] [galaxy_server.galaxy_ng]
url={{ galaxy_ng_server }} url={{ galaxy_ng_server }}content/primary/
token={{ galaxy_ng_token.json.token }} token={{ galaxy_ng_token.json.token }}
[galaxy_server.secondary] [galaxy_server.secondary]
url={{ pulp_server }}secondary/api/ url={{ galaxy_ng_server }}content/secondary/
v3=true
username={{ pulp_user }} username={{ pulp_user }}
password={{ pulp_password }} password={{ pulp_password }}

@ -17,11 +17,12 @@ unsupported_resolvelib_versions:
- "0.5.1" - "0.5.1"
pulp_repositories: pulp_repositories:
- published - primary
- secondary - secondary
publish_namespaces: publish_namespaces:
- ansible_test - ansible_test
- secondary
collection_list: collection_list:
# Scenario to test out pre-release being ignored unless explicitly set and version pagination. # Scenario to test out pre-release being ignored unless explicitly set and version pagination.

@ -10,12 +10,21 @@ from ....config import (
from ....docker_util import ( from ....docker_util import (
docker_cp_to, docker_cp_to,
docker_exec,
) )
from ....containers import ( from ....containers import (
run_support_container, run_support_container,
) )
from ....encoding import (
to_text,
)
from ....util import (
display,
)
from . import ( from . import (
CloudEnvironment, CloudEnvironment,
CloudEnvironmentConfig, CloudEnvironmentConfig,
@ -23,53 +32,59 @@ from . import (
) )
# We add BasicAuthentication, to make the tasks that deal with GALAXY_HOST_NAME = 'galaxy-pulp'
# direct API access easier to deal with across galaxy_ng and pulp SETTINGS = {
SETTINGS = ''' 'PULP_CONTENT_ORIGIN': f'http://{GALAXY_HOST_NAME}',
CONTENT_ORIGIN = 'http://{host}:80' 'PULP_ANSIBLE_API_HOSTNAME': f'http://{GALAXY_HOST_NAME}',
ANSIBLE_API_HOSTNAME = 'http://{host}:80' 'PULP_GALAXY_API_PATH_PREFIX': '/api/galaxy/',
ANSIBLE_CONTENT_HOSTNAME = 'http://{host}:80/pulp/content' # These paths are unique to the container image which has an nginx location for /pulp/content to route
TOKEN_AUTH_DISABLED = True # requests to the content backend
GALAXY_REQUIRE_CONTENT_APPROVAL = False 'PULP_ANSIBLE_CONTENT_HOSTNAME': f'http://{GALAXY_HOST_NAME}/pulp/content/api/galaxy/v3/artifacts/collections/',
GALAXY_AUTHENTICATION_CLASSES = [ 'PULP_CONTENT_PATH_PREFIX': '/pulp/content/api/galaxy/v3/artifacts/collections/',
"rest_framework.authentication.SessionAuthentication", 'PULP_GALAXY_AUTHENTICATION_CLASSES': [
"rest_framework.authentication.TokenAuthentication", 'rest_framework.authentication.SessionAuthentication',
"rest_framework.authentication.BasicAuthentication", 'rest_framework.authentication.TokenAuthentication',
] 'rest_framework.authentication.BasicAuthentication',
''' 'django.contrib.auth.backends.ModelBackend',
],
SET_ADMIN_PASSWORD = b'''#!/usr/bin/execlineb -S0 # This should probably be false see https://issues.redhat.com/browse/AAH-2328
foreground { 'PULP_GALAXY_REQUIRE_CONTENT_APPROVAL': 'true',
redirfd -w 1 /dev/null 'PULP_GALAXY_DEPLOYMENT_MODE': 'standalone',
redirfd -w 2 /dev/null 'PULP_GALAXY_AUTO_SIGN_COLLECTIONS': 'false',
export DJANGO_SETTINGS_MODULE pulpcore.app.settings 'PULP_GALAXY_COLLECTION_SIGNING_SERVICE': 'ansible-default',
export PULP_CONTENT_ORIGIN localhost 'PULP_RH_ENTITLEMENT_REQUIRED': 'insights',
s6-setuidgid postgres 'PULP_TOKEN_AUTH_DISABLED': 'false',
if { /usr/local/bin/django-admin reset-admin-password --password password } 'PULP_TOKEN_SERVER': f'http://{GALAXY_HOST_NAME}/token/',
if { /usr/local/bin/pulpcore-manager create-group system:partner-engineers --users admin } 'PULP_TOKEN_SIGNATURE_ALGORITHM': 'ES256',
} 'PULP_PUBLIC_KEY_PATH': '/src/galaxy_ng/dev/common/container_auth_public_key.pem',
''' 'PULP_PRIVATE_KEY_PATH': '/src/galaxy_ng/dev/common/container_auth_private_key.pem',
'PULP_ANALYTICS': 'false',
# There are 2 overrides here: 'PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_ACCESS': 'true',
# 1. Change the gunicorn bind address from 127.0.0.1 to 0.0.0.0 now that Galaxy NG does not allow us to access the 'PULP_GALAXY_ENABLE_UNAUTHENTICATED_COLLECTION_DOWNLOAD': 'true',
# Pulp API through it. 'PULP_GALAXY_ENABLE_LEGACY_ROLES': 'true',
# 2. Grant access allowing us to DELETE a namespace in Galaxy NG. This is as CI deletes and recreates repos and 'PULP_GALAXY_FEATURE_FLAGS__execution_environments': 'false',
# distributions in Pulp which now breaks the namespace in Galaxy NG. Recreating it is the "simple" fix to get it 'PULP_SOCIAL_AUTH_LOGIN_REDIRECT_URL': '/',
# working again. 'PULP_GALAXY_FEATURE_FLAGS__ai_deny_index': 'true',
# These may not be needed in the future, especially if 1 becomes configurable by an env var but for now they must be 'PULP_DEFAULT_ADMIN_PASSWORD': 'password'
# done.
OVERRIDES = b'''#!/usr/bin/execlineb -S0
foreground {
sed -i "0,/\\"127.0.0.1:24817\\"/s//\\"0.0.0.0:24817\\"/" /etc/services.d/pulpcore-api/run
} }
# This sed calls changes the first occurrence to "allow" which is conveniently the delete operation for a namespace.
# https://github.com/ansible/galaxy_ng/blob/master/galaxy_ng/app/access_control/statements/standalone.py#L9-L11. GALAXY_IMPORTER = b'''
backtick NG_PREFIX { python -c "import galaxy_ng; print(galaxy_ng.__path__[0], end='')" } [galaxy-importer]
importas ng_prefix NG_PREFIX ansible_local_tmp=~/.ansible/tmp
foreground { ansible_test_local_image=false
sed -i "0,/\\"effect\\": \\"deny\\"/s//\\"effect\\": \\"allow\\"/" ${ng_prefix}/app/access_control/statements/standalone.py check_required_tags=false
}''' check_runtime_yaml=false
check_changelog=false
infra_osd=false
local_image_docker=false
log_level_main=INFO
require_v1_or_greater=false
run_ansible_doc=false
run_ansible_lint=false
run_ansible_test=false
run_flake8=false
'''.strip()
class GalaxyProvider(CloudProvider): class GalaxyProvider(CloudProvider):
@ -81,13 +96,9 @@ class GalaxyProvider(CloudProvider):
def __init__(self, args: IntegrationConfig) -> None: def __init__(self, args: IntegrationConfig) -> None:
super().__init__(args) super().__init__(args)
# Cannot use the latest container image as either galaxy_ng 4.2.0rc2 or pulp 0.5.0 has sporatic issues with self.image = os.environ.get(
# dropping published collections in CI. Try running the tests multiple times when updating. Will also need to
# comment out the cache tests in 'test/integration/targets/ansible-galaxy-collection/tasks/install.yml' when
# the newer update is available.
self.pulp = os.environ.get(
'ANSIBLE_PULP_CONTAINER', 'ANSIBLE_PULP_CONTAINER',
'quay.io/ansible/pulp-galaxy-ng:b79a7be64eff' 'quay.io/pulp/galaxy:4.7.1'
) )
self.uses_docker = True self.uses_docker = True
@ -96,46 +107,45 @@ class GalaxyProvider(CloudProvider):
"""Setup cloud resource before delegation and reg cleanup callback.""" """Setup cloud resource before delegation and reg cleanup callback."""
super().setup() super().setup()
galaxy_port = 80 with tempfile.NamedTemporaryFile(mode='w+') as env_fd:
pulp_port = 24817 settings = '\n'.join(
f'{key}={value}' for key, value in SETTINGS.items()
ports = [ )
galaxy_port, env_fd.write(settings)
pulp_port, env_fd.flush()
] display.info(f'>>> galaxy_ng Configuration\n{settings}', verbosity=3)
# Create the container, don't run it, we need to inject configs before it starts
descriptor = run_support_container( descriptor = run_support_container(
self.args, self.args,
self.platform, self.platform,
self.pulp, self.image,
'galaxy-pulp', GALAXY_HOST_NAME,
ports, [
start=False, 80,
],
aliases=[
GALAXY_HOST_NAME,
],
start=True,
options=[
'--env-file', env_fd.name,
],
) )
if not descriptor: if not descriptor:
return return
pulp_id = descriptor.container_id injected_files = [
('/etc/galaxy-importer/galaxy-importer.cfg', GALAXY_IMPORTER, 'galaxy-importer'),
injected_files = { ]
'/etc/pulp/settings.py': SETTINGS.format(host=descriptor.name).encode(), for path, content, friendly_name in injected_files:
'/etc/cont-init.d/111-postgres': SET_ADMIN_PASSWORD,
'/etc/cont-init.d/000-ansible-test-overrides': OVERRIDES,
}
for path, content in injected_files.items():
with tempfile.NamedTemporaryFile() as temp_fd: with tempfile.NamedTemporaryFile() as temp_fd:
temp_fd.write(content) temp_fd.write(content)
temp_fd.flush() temp_fd.flush()
docker_cp_to(self.args, pulp_id, temp_fd.name, path) display.info(f'>>> {friendly_name} Configuration\n{to_text(content)}', verbosity=3)
docker_exec(self.args, descriptor.container_id, ['mkdir', '-p', os.path.dirname(path)], True)
descriptor.start(self.args) docker_cp_to(self.args, descriptor.container_id, temp_fd.name, path)
self._set_cloud_config('PULP_HOST', descriptor.name) self._set_cloud_config('PULP_HOST', GALAXY_HOST_NAME)
self._set_cloud_config('PULP_PORT', str(pulp_port))
self._set_cloud_config('GALAXY_PORT', str(galaxy_port))
self._set_cloud_config('PULP_USER', 'admin') self._set_cloud_config('PULP_USER', 'admin')
self._set_cloud_config('PULP_PASSWORD', 'password') self._set_cloud_config('PULP_PASSWORD', 'password')
@ -148,21 +158,19 @@ class GalaxyEnvironment(CloudEnvironment):
pulp_user = str(self._get_cloud_config('PULP_USER')) pulp_user = str(self._get_cloud_config('PULP_USER'))
pulp_password = str(self._get_cloud_config('PULP_PASSWORD')) pulp_password = str(self._get_cloud_config('PULP_PASSWORD'))
pulp_host = self._get_cloud_config('PULP_HOST') pulp_host = self._get_cloud_config('PULP_HOST')
galaxy_port = self._get_cloud_config('GALAXY_PORT')
pulp_port = self._get_cloud_config('PULP_PORT')
return CloudEnvironmentConfig( return CloudEnvironmentConfig(
ansible_vars=dict( ansible_vars=dict(
pulp_user=pulp_user, pulp_user=pulp_user,
pulp_password=pulp_password, pulp_password=pulp_password,
pulp_api='http://%s:%s' % (pulp_host, pulp_port), pulp_api=f'http://{pulp_host}',
pulp_server='http://%s:%s/pulp_ansible/galaxy/' % (pulp_host, pulp_port), pulp_server=f'http://{pulp_host}/pulp_ansible/galaxy/',
galaxy_ng_server='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port), galaxy_ng_server=f'http://{pulp_host}/api/galaxy/',
), ),
env_vars=dict( env_vars=dict(
PULP_USER=pulp_user, PULP_USER=pulp_user,
PULP_PASSWORD=pulp_password, PULP_PASSWORD=pulp_password,
PULP_SERVER='http://%s:%s/pulp_ansible/galaxy/api/' % (pulp_host, pulp_port), PULP_SERVER=f'http://{pulp_host}/pulp_ansible/galaxy/api/',
GALAXY_NG_SERVER='http://%s:%s/api/galaxy/' % (pulp_host, galaxy_port), GALAXY_NG_SERVER=f'http://{pulp_host}/api/galaxy/',
), ),
) )

@ -223,23 +223,19 @@ def test_cli_options(required_signature_count, valid, monkeypatch):
{ {
'url': 'https://galaxy.ansible.com', 'url': 'https://galaxy.ansible.com',
'validate_certs': 'False', 'validate_certs': 'False',
'v3': 'False',
}, },
# Expected server attributes # Expected server attributes
{ {
'validate_certs': False, 'validate_certs': False,
'_available_api_versions': {},
}, },
), ),
( (
{ {
'url': 'https://galaxy.ansible.com', 'url': 'https://galaxy.ansible.com',
'validate_certs': 'True', 'validate_certs': 'True',
'v3': 'True',
}, },
{ {
'validate_certs': True, 'validate_certs': True,
'_available_api_versions': {'v3': '/v3'},
}, },
), ),
], ],
@ -257,7 +253,6 @@ def test_bool_type_server_config_options(config, server, monkeypatch):
"server_list=server1\n", "server_list=server1\n",
"[galaxy_server.server1]", "[galaxy_server.server1]",
"url=%s" % config['url'], "url=%s" % config['url'],
"v3=%s" % config['v3'],
"validate_certs=%s\n" % config['validate_certs'], "validate_certs=%s\n" % config['validate_certs'],
] ]
@ -277,7 +272,6 @@ def test_bool_type_server_config_options(config, server, monkeypatch):
assert galaxy_cli.api_servers[0].name == 'server1' assert galaxy_cli.api_servers[0].name == 'server1'
assert galaxy_cli.api_servers[0].validate_certs == server['validate_certs'] assert galaxy_cli.api_servers[0].validate_certs == server['validate_certs']
assert galaxy_cli.api_servers[0]._available_api_versions == server['_available_api_versions']
@pytest.mark.parametrize('global_ignore_certs', [True, False]) @pytest.mark.parametrize('global_ignore_certs', [True, False])

Loading…
Cancel
Save