Fix KeyError for ansible-galaxy when caching paginated responses from v3 (#78325)

* Fix KeyError for ansible-galaxy when caching paginated responses from v3

* changelog

* generate responses in loop for test

Co-authored-by: Matt Martz <matt@sivel.net>

changelogs/fragments/78325-ansible-galaxy-fix-caching-paginated-responses-from-v3-servers.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-galaxy - fix setting the cache for paginated responses from Galaxy NG/AH (https://github.com/ansible/ansible/issues/77911).

lib/ansible/galaxy/api.py

@@ -330,25 +330,27 @@ class GalaxyAPI:
         should_retry_error=is_rate_limit_exception
     )
     def _call_galaxy(self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None,
-                     cache=False):
+                     cache=False, cache_key=None):
         url_info = urlparse(url)
         cache_id = get_cache_id(url)
+        if not cache_key:
+            cache_key = url_info.path
         query = parse_qs(url_info.query)
         if cache and self._cache:
             server_cache = self._cache.setdefault(cache_id, {})
             iso_datetime_format = '%Y-%m-%dT%H:%M:%SZ'
 
             valid = False
-            if url_info.path in server_cache:
-                expires = datetime.datetime.strptime(server_cache[url_info.path]['expires'], iso_datetime_format)
+            if cache_key in server_cache:
+                expires = datetime.datetime.strptime(server_cache[cache_key]['expires'], iso_datetime_format)
                 valid = datetime.datetime.utcnow() < expires
 
             is_paginated_url = 'page' in query or 'offset' in query
             if valid and not is_paginated_url:
                 # Got a hit on the cache and we aren't getting a paginated response
-                path_cache = server_cache[url_info.path]
+                path_cache = server_cache[cache_key]
                 if path_cache.get('paginated'):
-                    if '/v3/' in url_info.path:
+                    if '/v3/' in cache_key:
                         res = {'links': {'next': None}}
                     else:
                         res = {'next': None}
@@ -368,7 +370,7 @@ class GalaxyAPI:
                 # The cache entry had expired or does not exist, start a new blank entry to be filled later.
                 expires = datetime.datetime.utcnow()
                 expires += datetime.timedelta(days=1)
-                server_cache[url_info.path] = {
+                server_cache[cache_key] = {
                     'expires': expires.strftime(iso_datetime_format),
                     'paginated': False,
                 }
@@ -393,7 +395,7 @@ class GalaxyAPI:
                                % (resp.url, to_native(resp_data)))
 
         if cache and self._cache:
-            path_cache = self._cache[cache_id][url_info.path]
+            path_cache = self._cache[cache_id][cache_key]
 
             # v3 can return data or results for paginated results. Scan the result so we can determine what to cache.
             paginated_key = None
@@ -815,6 +817,7 @@ class GalaxyAPI:
         page_size_name = 'limit' if 'v3' in self.available_api_versions else 'page_size'
         versions_url = _urljoin(self.api_server, api_path, 'collections', namespace, name, 'versions', '/?%s=%d' % (page_size_name, COLLECTION_PAGE_SIZE))
         versions_url_info = urlparse(versions_url)
+        cache_key = versions_url_info.path
 
         # We should only rely on the cache if the collection has not changed. This may slow things down but it ensures
         # we are not waiting a day before finding any new collections that have been published.
@@ -834,7 +837,7 @@ class GalaxyAPI:
             if cached_modified_date != modified_date:
                 modified_cache['%s.%s' % (namespace, name)] = modified_date
                 if versions_url_info.path in server_cache:
-                    del server_cache[versions_url_info.path]
+                    del server_cache[cache_key]
 
                 self._set_cache()
 
@@ -842,7 +845,7 @@ class GalaxyAPI:
                             % (namespace, name, self.name, self.api_server)
 
         try:
-            data = self._call_galaxy(versions_url, error_context_msg=error_context_msg, cache=True)
+            data = self._call_galaxy(versions_url, error_context_msg=error_context_msg, cache=True, cache_key=cache_key)
         except GalaxyError as err:
             if err.http_code != 404:
                 raise
@@ -876,7 +879,7 @@ class GalaxyAPI:
                 next_link = versions_url.replace(versions_url_info.path, next_link)
 
             data = self._call_galaxy(to_native(next_link, errors='surrogate_or_strict'),
-                                     error_context_msg=error_context_msg, cache=True)
+                                     error_context_msg=error_context_msg, cache=True, cache_key=cache_key)
         self._set_cache()
 
         return versions

test/units/galaxy/test_api.py

@@ -75,6 +75,57 @@ def get_test_galaxy_api(url, version, token_ins=None, token_value=None, no_cache
     return api
 
 
+def get_v3_collection_versions(namespace='namespace', name='collection'):
+    pagination_path = f"/api/galaxy/content/community/v3/plugin/{namespace}/content/community/collections/index/{namespace}/{name}/versions"
+    page_versions = (('1.0.0', '1.0.1',), ('1.0.2', '1.0.3',), ('1.0.4', '1.0.5'),)
+    responses = [
+        {},  # TODO: initial response
+    ]
+
+    first = f"{pagination_path}/?limit=100"
+    last = f"{pagination_path}/?limit=100&offset=200"
+    page_versions = [
+        {
+            "versions": ('1.0.0', '1.0.1',),
+            "url": first,
+        },
+        {
+            "versions": ('1.0.2', '1.0.3',),
+            "url": f"{pagination_path}/?limit=100&offset=100",
+        },
+        {
+            "versions": ('1.0.4', '1.0.5'),
+            "url": last,
+        },
+    ]
+
+    previous = None
+    for page in range(0, len(page_versions)):
+        data = []
+
+        if page_versions[page]["url"] == last:
+            next_page = None
+        else:
+            next_page = page_versions[page + 1]["url"]
+        links = {"first": first, "last": last, "next": next_page, "previous": previous}
+
+        for version in page_versions[page]["versions"]:
+            data.append(
+                {
+                    "version": f"{version}",
+                    "href": f"{pagination_path}/{version}/",
+                    "created_at": "2022-05-13T15:55:58.913107Z",
+                    "updated_at": "2022-05-13T15:55:58.913121Z",
+                    "requires_ansible": ">=2.9.10"
+                }
+            )
+
+        responses.append({"meta": {"count": 6}, "links": links, "data": data})
+
+        previous = page_versions[page]["url"]
+    return responses
+
+
 def get_collection_versions(namespace='namespace', name='collection'):
     base_url = 'https://galaxy.server.com/api/v2/collections/{0}/{1}/'.format(namespace, name)
     versions_url = base_url + 'versions/'
@@ -1149,6 +1200,35 @@ def test_cache_complete_pagination(cache_dir, monkeypatch):
     assert cached_versions == actual_versions
 
 
+def test_cache_complete_pagination_v3(cache_dir, monkeypatch):
+
+    responses = get_v3_collection_versions()
+    cache_file = os.path.join(cache_dir, 'api.json')
+
+    api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v3', no_cache=False)
+
+    mock_open = MagicMock(
+        side_effect=[
+            StringIO(to_text(json.dumps(r)))
+            for r in responses
+        ]
+    )
+    monkeypatch.setattr(galaxy_api, 'open_url', mock_open)
+
+    actual_versions = api.get_collection_versions('namespace', 'collection')
+    assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5']
+
+    with open(cache_file) as fd:
+        final_cache = json.loads(fd.read())
+
+    cached_server = final_cache['galaxy.server.com:']
+    cached_collection = cached_server['/api/v3/collections/namespace/collection/versions/']
+    cached_versions = [r['version'] for r in cached_collection['results']]
+
+    assert final_cache == api._cache
+    assert cached_versions == actual_versions
+
+
 def test_cache_flaky_pagination(cache_dir, monkeypatch):
 
     responses = get_collection_versions()