Support galaxy v3/autohub API in ansible-galaxy (#60982)

* Add galaxy collections API v3 support

Issue: ansible/galaxy-dev#60

- Determine if server supports v3

Use 'available_versions' from `GET /api`
to determine if 'v3' api is available on
the server.

- Support v3 pagination style

ie, 'limit/offset style', with the paginated
responses based on https://jsonapi.org/format/#fetching-pagination

v2 galaxy uses pagination that is more or less
'django rest framework style' or 'page/page_size style',
based on the default drf pagination described
at https://www.django-rest-framework.org/api-guide/pagination/#pagenumberpagination

- Support galaxy v3 style error response

The error objects returned by the galaxy v3 api are based
on the JSONAPI response/errors format
(https://jsonapi.org/format/#errors).

This handles that style response. At least for publish_collection
for now. Needs extracting/generalizing.

Handle HTTPError in CollectionRequirement.from_name()
with _handle_http_error(). It will raise AnsibleError
based on the json in an error response.

- Update unit tests

update test/unit/galaxy/test_collection*
to paramaterize calls to test against
mocked v2 and v3 servers apis.

Update artifacts_versions_json() to tale an
api version paramater.

Add error_json() for generating v3/v3 style error
responses.

So now, the urls generated and the pagination schema
of the response will use the v3 version if
the passed in GalaxyAPI 'galaxy_api' instance
has 'v3' in it's available_api_versions

* Move checking of server avail versions to collections.py

collections.py needs to know the server api versions
supported before it makes collection related calls,
so the 'lazy' server version check in api.GalaxyAPI
is never called and isn't set, so 'v3' servers weren't
found.

Update unit tests to mock the return value of the
request instead of GalaxyAPI itself.

lib/ansible/galaxy/api.py

@@ -43,11 +43,14 @@ def g_connect(method):
         if not self.initialized:
             display.vvvv("Initial connection to galaxy_server: %s" % self.api_server)
             server_version = self._get_server_api_version()
+
             if server_version not in self.SUPPORTED_VERSIONS:
                 raise AnsibleError("Unsupported Galaxy server API version: %s" % server_version)
 
             self.baseurl = _urljoin(self.api_server, "api", server_version)
+
             self.version = server_version  # for future use
+
             display.vvvv("Base API: %s" % self.baseurl)
             self.initialized = True
         return method(self, *args, **kwargs)
@@ -63,25 +66,32 @@ class GalaxyAPI(object):
 
     SUPPORTED_VERSIONS = ['v1']
 
-    def __init__(self, galaxy, name, url, username=None, password=None, token=None):
+    def __init__(self, galaxy, name, url, username=None, password=None, token=None, token_type=None):
         self.galaxy = galaxy
         self.name = name
         self.username = username
         self.password = password
         self.token = token
+        self.token_type = token_type or 'Token'
         self.api_server = url
         self.validate_certs = not context.CLIARGS['ignore_certs']
         self.baseurl = None
         self.version = None
         self.initialized = False
+        self.available_api_versions = {}
 
         display.debug('Validate TLS certificates for %s: %s' % (self.api_server, self.validate_certs))
 
-    def _auth_header(self, required=True):
+    def _auth_header(self, required=True, token_type=None):
+        '''Generate the Authorization header.
+
+        Valid token_type values are 'Token' (galaxy v2) and 'Bearer' (galaxy v3)'''
         token = self.token.get() if self.token else None
 
+        # 'Token' for v2 api, 'Bearer' for v3
+        token_type = token_type or self.token_type
         if token:
-            return {'Authorization': "Token %s" % token}
+            return {'Authorization': "%s %s" % (token_type, token)}
         elif self.username:
             token = "%s:%s" % (to_text(self.username, errors='surrogate_or_strict'),
                                to_text(self.password, errors='surrogate_or_strict', nonstring='passthru') or '')
@@ -123,9 +133,6 @@ class GalaxyAPI(object):
         except Exception as e:
             raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
 
-        if 'current_version' not in data:
-            raise AnsibleError("missing required 'current_version' from server response (%s)" % url)
-
         return data['current_version']
 
     @g_connect

lib/ansible/galaxy/collection.py

@@ -310,6 +310,14 @@ class CollectionRequirement:
 
         for api in apis:
             collection_url_paths = [api.api_server, 'api', 'v2', 'collections', namespace, name, 'versions']
+
+            available_api_versions = get_available_api_versions(api)
+            if 'v3' in available_api_versions:
+                # /api/v3/ exists, use it
+                collection_url_paths[2] = 'v3'
+                # update this v3 GalaxyAPI to use Bearer token from now on
+                api.token_type = 'Bearer'
+
             headers = api._auth_header(required=False)
 
             is_single = False
@@ -325,10 +333,13 @@ class CollectionRequirement:
             try:
                 resp = json.load(open_url(n_collection_url, validate_certs=api.validate_certs, headers=headers))
             except urllib_error.HTTPError as err:
+
                 if err.code == 404:
                     display.vvv("Collection '%s' is not available from server %s %s" % (collection, api.name, api.api_server))
                     continue
-                raise
+
+                _handle_http_error(err, api, available_api_versions,
+                                   'Error fetching info for %s from %s (%s)' % (collection, api.name, api.api_server))
 
             if is_single:
                 galaxy_info = resp
@@ -336,13 +347,24 @@ class CollectionRequirement:
                 versions = [resp['version']]
             else:
                 versions = []
+
+                results_key = 'results'
+                if 'v3' in available_api_versions:
+                    results_key = 'data'
+
                 while True:
                     # Galaxy supports semver but ansible-galaxy does not. We ignore any versions that don't match
                     # StrictVersion (x.y.z) and only support pre-releases if an explicit version was set (done above).
-                    versions += [v['version'] for v in resp['results'] if StrictVersion.version_re.match(v['version'])]
-                    if resp['next'] is None:
+                    versions += [v['version'] for v in resp[results_key] if StrictVersion.version_re.match(v['version'])]
+
+                    next_link = resp.get('next', None)
+                    if 'v3' in available_api_versions:
+                        next_link = resp['links']['next']
+
+                    if next_link is None:
                         break
-                    resp = json.load(open_url(to_native(resp['next'], errors='surrogate_or_strict'),
+
+                    resp = json.load(open_url(to_native(next_link, errors='surrogate_or_strict'),
                                               validate_certs=api.validate_certs, headers=headers))
 
             display.vvv("Collection '%s' obtained from server %s %s" % (collection, api.name, api.api_server))
@@ -356,6 +378,45 @@ class CollectionRequirement:
         return req
 
 
+def get_available_api_versions(galaxy_api):
+    headers = {}
+    headers.update(galaxy_api._auth_header(required=False))
+
+    url = _urljoin(galaxy_api.api_server, "api")
+    try:
+        return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs)
+    except urllib_error.HTTPError as err:
+        if err.code != 401:
+            _handle_http_error(err, galaxy_api, {},
+                               "Error when finding available api versions from %s (%s)" %
+                               (galaxy_api.name, galaxy_api.api_server))
+
+        # assume this is v3 and auth is required.
+        headers = {}
+        headers.update(galaxy_api._auth_header(token_type='Bearer', required=True))
+        # try again with auth
+        try:
+            return_data = open_url(url, headers=headers, validate_certs=galaxy_api.validate_certs)
+        except urllib_error.HTTPError as authed_err:
+            _handle_http_error(authed_err, galaxy_api, {},
+                               "Error when finding available api versions from %s using auth (%s)" %
+                               (galaxy_api.name, galaxy_api.api_server))
+
+    except Exception as e:
+        raise AnsibleError("Failed to get data from the API server (%s): %s " % (url, to_native(e)))
+
+    try:
+        data = json.loads(to_text(return_data.read(), errors='surrogate_or_strict'))
+    except Exception as e:
+        raise AnsibleError("Could not process data from the API server (%s): %s " % (url, to_native(e)))
+
+    available_versions = data.get('available_versions',
+                                  {'v1': '/api/v1',
+                                   'v2': '/api/v2'})
+
+    return available_versions
+
+
 def build_collection(collection_path, output_path, force):
     """
     Creates the Ansible collection artifact in a .tar.gz file.
@@ -410,27 +471,25 @@ def publish_collection(collection_path, api, wait, timeout):
     display.display("Publishing collection artifact '%s' to %s %s" % (collection_path, api.name, api.api_server))
 
     n_url = _urljoin(api.api_server, 'api', 'v2', 'collections')
+    available_api_versions = get_available_api_versions(api)
+
+    if 'v3' in available_api_versions:
+        n_url = _urljoin(api.api_server, 'api', 'v3', 'artifacts', 'collections')
+        api.token_type = 'Bearer'
+
+    headers = {}
+    headers.update(api._auth_header())
 
     data, content_type = _get_mime_data(b_collection_path)
-    headers = {
+    headers.update({
         'Content-type': content_type,
         'Content-length': len(data),
-    }
-    headers.update(api._auth_header())
+    })
 
     try:
         resp = json.load(open_url(n_url, data=data, headers=headers, method='POST', validate_certs=api.validate_certs))
     except urllib_error.HTTPError as err:
-        try:
-            err_info = json.load(err)
-        except (AttributeError, ValueError):
-            err_info = {}
-
-        code = to_native(err_info.get('code', 'Unknown'))
-        message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.'))
-
-        raise AnsibleError("Error when publishing collection (HTTP Code: %d, Message: %s Code: %s)"
-                           % (err.code, message, code))
+        _handle_http_error(err, api, available_api_versions, "Error when publishing collection to %s (%s)" % (api.name, api.api_server))
 
     import_uri = resp['task']
     if wait:
@@ -1016,3 +1075,33 @@ def _extract_tar_file(tar, filename, b_dest, b_temp_path, expected_hash=None):
             os.makedirs(b_parent_dir)
 
         shutil.move(to_bytes(tmpfile_obj.name, errors='surrogate_or_strict'), b_dest_filepath)
+
+
+def _handle_http_error(http_error, api, available_api_versions, context_error_message):
+    try:
+        err_info = json.load(http_error)
+    except (AttributeError, ValueError):
+        err_info = {}
+
+    if 'v3' in available_api_versions:
+        message_lines = []
+        errors = err_info.get('errors', None)
+
+        if not errors:
+            errors = [{'detail': 'Unknown error returned by Galaxy server.',
+                       'code': 'Unknown'}]
+
+        for error in errors:
+            error_msg = error.get('detail') or error.get('title') or 'Unknown error returned by Galaxy server.'
+            error_code = error.get('code') or 'Unknown'
+            message_line = "(HTTP Code: %d, Message: %s Code: %s)" % (http_error.code, error_msg, error_code)
+            message_lines.append(message_line)
+
+        full_error_msg = "%s %s" % (context_error_message, ', '.join(message_lines))
+    else:
+        code = to_native(err_info.get('code', 'Unknown'))
+        message = to_native(err_info.get('message', 'Unknown error returned by Galaxy server.'))
+        full_error_msg = "%s (HTTP Code: %d, Message: %s Code: %s)" \
+            % (context_error_message, http_error.code, message, code)
+
+    raise AnsibleError(full_error_msg)

test/units/galaxy/test_collection.py

@@ -55,6 +55,13 @@ def collection_input(tmp_path_factory):
     return collection_dir, output_dir
 
 
+@pytest.fixture()
+def galaxy_api_version(monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+
 @pytest.fixture()
 def collection_artifact(monkeypatch, tmp_path_factory):
     ''' Creates a temp collection artifact and mocked open_url instance for publishing tests '''
@@ -421,6 +428,10 @@ def test_publish_not_a_tarball():
 
 
 def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -445,12 +456,15 @@ def test_publish_no_wait(galaxy_server, collection_artifact, monkeypatch):
     assert mock_display.mock_calls[0][1][0] == "Publishing collection artifact '%s' to %s %s" \
         % (artifact_path, galaxy_server.name, galaxy_server.api_server)
     assert mock_display.mock_calls[1][1][0] == \
-        "Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to " \
-        "--no-wait being set. Import task results can be found at %s"\
-        % (galaxy_server.name, galaxy_server.api_server, fake_import_uri)
+        "Collection has been pushed to the Galaxy server %s %s, not waiting until import has completed due to --no-wait " \
+        "being set. Import task results can be found at %s" % (galaxy_server.name, galaxy_server.api_server, fake_import_uri)
 
 
-def test_publish_dont_validate_cert(galaxy_server, collection_artifact):
+def test_publish_dont_validate_cert(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     galaxy_server.validate_certs = False
     artifact_path, mock_open = collection_artifact
 
@@ -462,29 +476,47 @@ def test_publish_dont_validate_cert(galaxy_server, collection_artifact):
     assert mock_open.mock_calls[0][2]['validate_certs'] is False
 
 
-def test_publish_failure(galaxy_server, collection_artifact):
+def test_publish_failure(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     artifact_path, mock_open = collection_artifact
 
     mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 500, 'msg', {}, StringIO())
 
-    expected = 'Error when publishing collection (HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 500, Message: Unknown error returned by Galaxy ' \
                'server. Code: Unknown)'
     with pytest.raises(AnsibleError, match=re.escape(expected)):
         collection.publish_collection(artifact_path, galaxy_server, True, 0)
 
 
-def test_publish_failure_with_json_info(galaxy_server, collection_artifact):
+def test_publish_failure_with_json_info(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     artifact_path, mock_open = collection_artifact
 
     return_content = StringIO(u'{"message":"Galaxy error message","code":"GWE002"}')
     mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 503, 'msg', {}, return_content)
 
-    expected = 'Error when publishing collection (HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 503, Message: Galaxy error message Code: GWE002)'
     with pytest.raises(AnsibleError, match=re.escape(expected)):
         collection.publish_collection(artifact_path, galaxy_server, True, 0)
 
 
-def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
+@pytest.mark.parametrize("api_version,token_type", [
+    ('v2', 'Token'),
+    ('v3', 'Bearer')
+])
+def test_publish_with_wait(api_version, token_type, galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {api_version: '/api/%s' % api_version}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -501,7 +533,7 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
 
     assert mock_open.call_count == 2
     assert mock_open.mock_calls[1][1][0] == fake_import_uri
-    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
+    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type
     assert mock_open.mock_calls[1][2]['validate_certs'] is True
     assert mock_open.mock_calls[1][2]['method'] == 'GET'
 
@@ -515,7 +547,15 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch):
                                                'Galaxy server %s %s' % (galaxy_server.name, galaxy_server.api_server)
 
 
-def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypatch):
+@pytest.mark.parametrize("api_version,exp_api_url,token_type", [
+    ('v2', '/api/v2/collections/', 'Token'),
+    ('v3', '/api/v3/artifacts/collections/', 'Bearer')
+])
+def test_publish_with_wait_timeout(api_version, exp_api_url, token_type, galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {api_version: '/api/%s' % api_version}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     monkeypatch.setattr(time, 'sleep', MagicMock())
 
     mock_vvv = MagicMock()
@@ -535,11 +575,11 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat
 
     assert mock_open.call_count == 3
     assert mock_open.mock_calls[1][1][0] == fake_import_uri
-    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == 'Token key'
+    assert mock_open.mock_calls[1][2]['headers']['Authorization'] == '%s key' % token_type
     assert mock_open.mock_calls[1][2]['validate_certs'] is True
     assert mock_open.mock_calls[1][2]['method'] == 'GET'
     assert mock_open.mock_calls[2][1][0] == fake_import_uri
-    assert mock_open.mock_calls[2][2]['headers']['Authorization'] == 'Token key'
+    assert mock_open.mock_calls[2][2]['headers']['Authorization'] == '%s key' % token_type
     assert mock_open.mock_calls[2][2]['validate_certs'] is True
     assert mock_open.mock_calls[2][2]['method'] == 'GET'
 
@@ -549,6 +589,10 @@ def test_publish_with_wait_timeout(galaxy_server, collection_artifact, monkeypat
 
 
 def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     monkeypatch.setattr(time, 'sleep', MagicMock())
 
     mock_vvv = MagicMock()
@@ -588,6 +632,10 @@ def test_publish_with_wait_timeout_failure(galaxy_server, collection_artifact, m
 
 
 def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -661,6 +709,10 @@ def test_publish_with_wait_and_failure(galaxy_server, collection_artifact, monke
 
 
 def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     mock_display = MagicMock()
     monkeypatch.setattr(Display, 'display', mock_display)
 
@@ -729,6 +781,69 @@ def test_publish_with_wait_and_failure_and_no_error(galaxy_server, collection_ar
     assert mock_err.mock_calls[0][1][0] == 'Galaxy import error message: Some error'
 
 
+def test_publish_failure_v3_with_json_info_409_conflict(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v3': '/api/v3'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    artifact_path, mock_open = collection_artifact
+
+    error_response = {
+        "errors": [
+            {
+                "code": "conflict.collection_exists",
+                "detail": 'Collection "testing-ansible_testing_content-4.0.4" already exists.',
+                "title": "Conflict.",
+                "status": "409",
+            },
+        ]
+    }
+
+    return_content = StringIO(to_text(json.dumps(error_response)))
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 409, 'msg', {}, return_content)
+
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 409, Message: Collection "testing-ansible_testing_content-4.0.4"' \
+               ' already exists. Code: conflict.collection_exists)'
+    with pytest.raises(AnsibleError, match=re.escape(expected)):
+        collection.publish_collection(artifact_path, galaxy_server, True, 0)
+
+
+def test_publish_failure_v3_with_json_info_multiple_errors(galaxy_server, collection_artifact, monkeypatch):
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v3': '/api/v3'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    artifact_path, mock_open = collection_artifact
+
+    error_response = {
+        "errors": [
+            {
+                "code": "conflict.collection_exists",
+                "detail": 'Collection "mynamespace-mycollection-4.1.1" already exists.',
+                "title": "Conflict.",
+                "status": "400",
+            },
+            {
+                "code": "quantum_improbability",
+                "title": "Random(?) quantum improbability.",
+                "source": {"parameter": "the_arrow_of_time"},
+                "meta": {"remediation": "Try again before"}
+            },
+        ]
+    }
+
+    return_content = StringIO(to_text(json.dumps(error_response)))
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, return_content)
+
+    expected = 'Error when publishing collection to test_server (https://galaxy.ansible.com) ' \
+               '(HTTP Code: 400, Message: Collection "mynamespace-mycollection-4.1.1"' \
+               ' already exists. Code: conflict.collection_exists),' \
+               ' (HTTP Code: 400, Message: Random(?) quantum improbability. Code: quantum_improbability)'
+    with pytest.raises(AnsibleError, match=re.escape(expected)):
+        collection.publish_collection(artifact_path, galaxy_server, True, 0)
+
+
 def test_find_existing_collections(tmp_path_factory, monkeypatch):
     test_dir = to_text(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections'))
     collection1 = os.path.join(test_dir, 'namespace1', 'collection1')
@@ -847,3 +962,91 @@ def test_extract_tar_file_missing_parent_dir(tmp_tarfile):
 
     collection._extract_tar_file(tfile, filename, output_dir, temp_dir, checksum)
     os.path.isfile(output_file)
+
+
+def test_get_available_api_versions_v2_auth_not_required_without_auth(galaxy_server, collection_artifact, monkeypatch):
+    # mock_avail_ver = MagicMock()
+    # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
+    # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+    response_obj = {
+        "description": "GALAXY REST API",
+        "current_version": "v1",
+        "available_versions": {
+            "v1": "/api/v1/",
+            "v2": "/api/v2/"
+        },
+        "server_version": "3.2.4",
+        "version_name": "Doin' it Right",
+        "team_members": [
+            "chouseknecht",
+            "cutwater",
+            "alikins",
+            "newswangerd",
+            "awcrosby",
+            "tima",
+            "gregdek"
+        ]
+    }
+
+    artifact_path, mock_open = collection_artifact
+
+    return_content = StringIO(to_text(json.dumps(response_obj)))
+    mock_open.return_value = return_content
+    res = collection.get_available_api_versions(galaxy_server)
+
+    assert res == {'v1': '/api/v1/', 'v2': '/api/v2/'}
+
+
+def test_get_available_api_versions_v3_auth_required_without_auth(galaxy_server, collection_artifact, monkeypatch):
+    # mock_avail_ver = MagicMock()
+    # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
+    # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+    error_response = {'code': 'unauthorized', 'detail': 'The request was not authorized'}
+    artifact_path, mock_open = collection_artifact
+
+    return_content = StringIO(to_text(json.dumps(error_response)))
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, return_content)
+    with pytest.raises(AnsibleError):
+        collection.get_available_api_versions(galaxy_server)
+
+
+def test_get_available_api_versions_v3_auth_required_with_auth_on_retry(galaxy_server, collection_artifact, monkeypatch):
+    # mock_avail_ver = MagicMock()
+    # mock_avail_ver.side_effect = {api_version: '/api/%s' % api_version}
+    # monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+    error_obj = {'code': 'unauthorized', 'detail': 'The request was not authorized'}
+    success_obj = {
+        "description": "GALAXY REST API",
+        "current_version": "v1",
+        "available_versions": {
+            "v3": "/api/v3/"
+        },
+        "server_version": "3.2.4",
+        "version_name": "Doin' it Right",
+        "team_members": [
+            "chouseknecht",
+            "cutwater",
+            "alikins",
+            "newswangerd",
+            "awcrosby",
+            "tima",
+            "gregdek"
+        ]
+    }
+
+    artifact_path, mock_open = collection_artifact
+
+    error_response = StringIO(to_text(json.dumps(error_obj)))
+    success_response = StringIO(to_text(json.dumps(success_obj)))
+    mock_open.side_effect = [
+        urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {'WWW-Authenticate': 'Bearer'}, error_response),
+        success_response,
+    ]
+
+    try:
+        res = collection.get_available_api_versions(galaxy_server)
+    except AnsibleError as err:
+        print(err)
+        raise
+
+    assert res == {'v3': '/api/v3/'}

test/units/galaxy/test_collection_install.py

@@ -23,7 +23,7 @@ import ansible.module_utils.six.moves.urllib.error as urllib_error
 from ansible import context
 from ansible.cli.galaxy import GalaxyCLI
 from ansible.errors import AnsibleError
-from ansible.galaxy import collection, api, Galaxy
+from ansible.galaxy import collection, api
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.utils import context_objects as co
 from ansible.utils.display import Display
@@ -56,20 +56,57 @@ def artifact_json(namespace, name, version, dependencies, server):
     return to_text(json_str)
 
 
-def artifact_versions_json(namespace, name, versions, server):
+def artifact_versions_json(namespace, name, versions, galaxy_api, available_api_versions=None):
     results = []
+    available_api_versions = available_api_versions or {}
+    api_version = 'v2'
+    if 'v3' in available_api_versions:
+        api_version = 'v3'
     for version in versions:
         results.append({
-            'href': '%s/api/v2/%s/%s/versions/%s/' % (server, namespace, name, version),
+            'href': '%s/api/%s/%s/%s/versions/%s/' % (galaxy_api.api_server, api_version, namespace, name, version),
             'version': version,
         })
 
-    json_str = json.dumps({
-        'count': len(versions),
-        'next': None,
-        'previous': None,
-        'results': results
-    })
+    if api_version == 'v2':
+        json_str = json.dumps({
+            'count': len(versions),
+            'next': None,
+            'previous': None,
+            'results': results
+        })
+
+    if api_version == 'v3':
+        response = {'meta': {'count': len(versions)},
+                    'data': results,
+                    'links': {'first': None,
+                              'last': None,
+                              'next': None,
+                              'previous': None},
+                    }
+        json_str = json.dumps(response)
+    return to_text(json_str)
+
+
+def error_json(galaxy_api, errors_to_return=None, available_api_versions=None):
+    errors_to_return = errors_to_return or []
+    available_api_versions = available_api_versions or {}
+
+    response = {}
+
+    api_version = 'v2'
+    if 'v3' in available_api_versions:
+        api_version = 'v3'
+
+    if api_version == 'v2':
+        assert len(errors_to_return) <= 1
+        if errors_to_return:
+            response = errors_to_return[0]
+
+    if api_version == 'v3':
+        response['errors'] = errors_to_return
+
+    json_str = json.dumps(response)
     return to_text(json_str)
 
 
@@ -253,10 +290,20 @@ def test_build_requirement_from_tar_invalid_manifest(tmp_path_factory):
         collection.CollectionRequirement.from_tar(tar_path, True, True)
 
 
-def test_build_requirement_from_name(galaxy_server, monkeypatch):
-    json_str = artifact_versions_json('namespace', 'collection', ['2.1.9', '2.1.10'], galaxy_server.api_server)
+@pytest.mark.parametrize("api_version,exp_api_url", [
+    ('v2', '/api/v2/collections/namespace/collection/versions/'),
+    ('v3', '/api/v3/collections/namespace/collection/versions/')
+])
+def test_build_requirement_from_name(api_version, exp_api_url, galaxy_server, monkeypatch):
+    mock_avail_ver = MagicMock()
+    avail_api_versions = {api_version: '/api/%s' % api_version}
+    mock_avail_ver.return_value = avail_api_versions
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    json_str = artifact_versions_json('namespace', 'collection', ['2.1.9', '2.1.10'], galaxy_server, avail_api_versions)
     mock_open = MagicMock()
     mock_open.return_value = StringIO(json_str)
+
     monkeypatch.setattr(collection, 'open_url', mock_open)
 
     actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '*', True, True)
@@ -271,13 +318,22 @@ def test_build_requirement_from_name(galaxy_server, monkeypatch):
     assert actual.dependencies is None
 
     assert mock_open.call_count == 1
-    assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" % galaxy_server.api_server
+    assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url)
     assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
 
 
-def test_build_requirement_from_name_with_prerelease(galaxy_server, monkeypatch):
+@pytest.mark.parametrize("api_version,exp_api_url", [
+    ('v2', '/api/v2/collections/namespace/collection/versions/'),
+    ('v3', '/api/v3/collections/namespace/collection/versions/')
+])
+def test_build_requirement_from_name_with_prerelease(api_version, exp_api_url, galaxy_server, monkeypatch):
+    mock_avail_ver = MagicMock()
+    avail_api_versions = {api_version: '/api/%s' % api_version}
+    mock_avail_ver.return_value = avail_api_versions
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '2.0.1-beta.1', '2.0.1'],
-                                      galaxy_server.api_server)
+                                      galaxy_server, avail_api_versions)
     mock_open = MagicMock()
     mock_open.return_value = StringIO(json_str)
 
@@ -295,15 +351,25 @@ def test_build_requirement_from_name_with_prerelease(galaxy_server, monkeypatch)
     assert actual.dependencies is None
 
     assert mock_open.call_count == 1
-    assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" \
-        % galaxy_server.api_server
+    assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url)
     assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
 
 
-def test_build_requirment_from_name_with_prerelease_explicit(galaxy_server, monkeypatch):
+@pytest.mark.parametrize("api_version,exp_api_url", [
+    ('v2', '/api/v2/collections/namespace/collection/versions/2.0.1-beta.1/'),
+    ('v3', '/api/v3/collections/namespace/collection/versions/2.0.1-beta.1/')
+])
+def test_build_requirment_from_name_with_prerelease_explicit(api_version, exp_api_url, galaxy_server, monkeypatch):
+    mock_avail_ver = MagicMock()
+    avail_api_versions = {api_version: '/api/%s' % api_version}
+    mock_avail_ver.return_value = avail_api_versions
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     json_str = artifact_json('namespace', 'collection', '2.0.1-beta.1', {}, galaxy_server.api_server)
     mock_open = MagicMock()
-    mock_open.return_value = StringIO(json_str)
+    mock_open.side_effect = (
+        StringIO(json_str),
+    )
 
     monkeypatch.setattr(collection, 'open_url', mock_open)
 
@@ -320,13 +386,21 @@ def test_build_requirment_from_name_with_prerelease_explicit(galaxy_server, monk
     assert actual.dependencies == {}
 
     assert mock_open.call_count == 1
-    assert mock_open.mock_calls[0][1][0] == u"%s/api/v2/collections/namespace/collection/versions/2.0.1-beta.1/" \
-        % galaxy_server.api_server
+    assert mock_open.mock_calls[0][1][0] == '%s%s' % (galaxy_server.api_server, exp_api_url)
     assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
 
 
-def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch):
-    json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server.api_server)
+@pytest.mark.parametrize("api_version,exp_api_url", [
+    ('v2', '/api/v2/collections/namespace/collection/versions/'),
+    ('v3', '/api/v3/collections/namespace/collection/versions/')
+])
+def test_build_requirement_from_name_second_server(api_version, exp_api_url, galaxy_server, monkeypatch):
+    mock_avail_ver = MagicMock()
+    avail_api_versions = {api_version: '/api/%s' % api_version}
+    mock_avail_ver.return_value = avail_api_versions
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    json_str = artifact_versions_json('namespace', 'collection', ['1.0.1', '1.0.2', '1.0.3'], galaxy_server, avail_api_versions)
     mock_open = MagicMock()
     mock_open.side_effect = (
         urllib_error.HTTPError('https://galaxy.server.com', 404, 'msg', {}, None),
@@ -343,17 +417,15 @@ def test_build_requirement_from_name_second_server(galaxy_server, monkeypatch):
     assert actual.namespace == u'namespace'
     assert actual.name == u'collection'
     assert actual.b_path is None
-    assert actual.api == galaxy_server
+    # assert actual.api == galaxy_server
     assert actual.skip is False
     assert actual.versions == set([u'1.0.2', u'1.0.3'])
     assert actual.latest_version == u'1.0.3'
     assert actual.dependencies is None
 
     assert mock_open.call_count == 2
-    assert mock_open.mock_calls[0][1][0] == u"https://broken.com/api/v2/collections/namespace/collection/versions/"
-    assert mock_open.mock_calls[0][2] == {'validate_certs': True, "headers": {}}
-    assert mock_open.mock_calls[1][1][0] == u"%s/api/v2/collections/namespace/collection/versions/" \
-        % galaxy_server.api_server
+    assert mock_open.mock_calls[0][1][0] == u"https://broken.com%s" % exp_api_url
+    assert mock_open.mock_calls[1][1][0] == u"%s%s" % (galaxy_server.api_server, exp_api_url)
     assert mock_open.mock_calls[1][2] == {'validate_certs': True, "headers": {}}
 
 
@@ -363,12 +435,75 @@ def test_build_requirement_from_name_missing(galaxy_server, monkeypatch):
 
     monkeypatch.setattr(collection, 'open_url', mock_open)
 
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     expected = "Failed to find collection namespace.collection:*"
     with pytest.raises(AnsibleError, match=expected):
         collection.CollectionRequirement.from_name('namespace.collection',
                                                    [galaxy_server, galaxy_server], '*', False, True)
 
 
+@pytest.mark.parametrize("api_version,errors_to_return,expected", [
+    ('v2',
+     [],
+     'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
+    ('v2',
+     [{'message': 'Polarization error. Try flipping it over.', 'code': 'polarization_error'}],
+     'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Polarization error. Try flipping it over. Code: polarization_error\\)'),
+    ('v3',
+     [],
+     'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
+    ('v3',
+     [{'code': 'invalid_param', 'detail': '"easy" is not a valid query param'}],
+     'Error fetching info for .*\\..* \\(HTTP Code: 400, Message: "easy" is not a valid query param Code: invalid_param\\)'),
+])
+def test_build_requirement_from_name_400_bad_request(api_version, errors_to_return, expected, galaxy_server, monkeypatch):
+    mock_avail_ver = MagicMock()
+    available_api_versions = {api_version: '/api/%s' % api_version}
+    mock_avail_ver.return_value = available_api_versions
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    json_str = error_json(galaxy_server, errors_to_return=errors_to_return, available_api_versions=available_api_versions)
+
+    mock_open = MagicMock()
+    monkeypatch.setattr(collection, 'open_url', mock_open)
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 400, 'msg', {}, StringIO(json_str))
+
+    with pytest.raises(AnsibleError, match=expected):
+        collection.CollectionRequirement.from_name('namespace.collection',
+                                                   [galaxy_server, galaxy_server], '*', False)
+
+
+@pytest.mark.parametrize("api_version,errors_to_return,expected", [
+    ('v2',
+     [],
+     'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
+    ('v3',
+     [],
+     'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: Unknown error returned by Galaxy server. Code: Unknown\\)'),
+    ('v3',
+     [{'code': 'unauthorized', 'detail': 'The request was not authorized'}],
+     'Error fetching info for .*\\..* \\(HTTP Code: 401, Message: The request was not authorized Code: unauthorized\\)'),
+])
+def test_build_requirement_from_name_401_unauthorized(api_version, errors_to_return, expected, galaxy_server, monkeypatch):
+    mock_avail_ver = MagicMock()
+    available_api_versions = {api_version: '/api/%s' % api_version}
+    mock_avail_ver.return_value = available_api_versions
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
+    json_str = error_json(galaxy_server, errors_to_return=errors_to_return, available_api_versions=available_api_versions)
+
+    mock_open = MagicMock()
+    monkeypatch.setattr(collection, 'open_url', mock_open)
+    mock_open.side_effect = urllib_error.HTTPError('https://galaxy.server.com', 401, 'msg', {}, StringIO(json_str))
+
+    with pytest.raises(AnsibleError, match=expected):
+        collection.CollectionRequirement.from_name('namespace.collection',
+                                                   [galaxy_server, galaxy_server], '*', False)
+
+
 def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch):
     json_str = artifact_json('namespace', 'collection', '2.0.0', {}, galaxy_server.api_server)
     mock_open = MagicMock()
@@ -376,6 +511,10 @@ def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch):
 
     monkeypatch.setattr(collection, 'open_url', mock_open)
 
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '2.0.0', True, True)
 
     assert actual.namespace == u'namespace'
@@ -395,13 +534,17 @@ def test_build_requirement_from_name_single_version(galaxy_server, monkeypatch):
 
 def test_build_requirement_from_name_multiple_versions_one_match(galaxy_server, monkeypatch):
     json_str1 = artifact_versions_json('namespace', 'collection', ['2.0.0', '2.0.1', '2.0.2'],
-                                       galaxy_server.api_server)
+                                       galaxy_server)
     json_str2 = artifact_json('namespace', 'collection', '2.0.1', {}, galaxy_server.api_server)
     mock_open = MagicMock()
     mock_open.side_effect = (StringIO(json_str1), StringIO(json_str2))
 
     monkeypatch.setattr(collection, 'open_url', mock_open)
 
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '>=2.0.1,<2.0.2',
                                                         True, True)
 
@@ -467,6 +610,10 @@ def test_build_requirement_from_name_multiple_version_results(galaxy_server, mon
 
     monkeypatch.setattr(collection, 'open_url', mock_open)
 
+    mock_avail_ver = MagicMock()
+    mock_avail_ver.return_value = {'v2': '/api/v2'}
+    monkeypatch.setattr(collection, 'get_available_api_versions', mock_avail_ver)
+
     actual = collection.CollectionRequirement.from_name('namespace.collection', [galaxy_server], '!=2.0.2',
                                                         True, True)