From 4fcf731ced23f60c1ea7cebbbc2411ff451cd001 Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Wed, 5 Apr 2023 12:06:13 -0400 Subject: [PATCH] Fetch signatures from galaxy after the dependency resolver runs (#80334) (#80397) Reduce the number of Galaxy API calls made during dependency resolution by fetching remote signatures afterwards, since these are not used in backtracking. Reduce the verbosity to `-vvvv` (to match other Galaxy API calls) to see this activity. Co-authored-by: Sviatoslav Sydorenko (cherry picked from commit 460abe0ceffc5ca99b3cc2f2e3ef07aa2cb225dc) --- .../80334-reduce-ansible-galaxy-api-calls.yml | 2 ++ lib/ansible/galaxy/api.py | 3 +-- lib/ansible/galaxy/collection/__init__.py | 3 +++ .../galaxy/dependency_resolution/dataclasses.py | 12 +++++++++++- .../galaxy/dependency_resolution/providers.py | 1 - .../ansible-galaxy-collection/tasks/install.yml | 3 ++- 6 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml diff --git a/changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml b/changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml new file mode 100644 index 00000000000..c780a109618 --- /dev/null +++ b/changelogs/fragments/80334-reduce-ansible-galaxy-api-calls.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-galaxy - reduce API calls to servers by fetching signatures only for final candidates. diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 03d9104f5c9..0d519980440 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -926,8 +926,7 @@ class GalaxyAPI: try: signatures = data["signatures"] except KeyError: - # Noisy since this is used by the dep resolver, so require more verbosity than Galaxy calls - display.vvvvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}") + display.vvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}") return [] else: return [signature_info["signature"] for signature_info in signatures] diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index 5a3928f88df..07a81eee32f 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -769,6 +769,9 @@ def install_collections( "Skipping signature verification." ) + if concrete_coll_pin.type == 'galaxy': + concrete_coll_pin = concrete_coll_pin.with_signatures_repopulated() + try: install(concrete_coll_pin, output_path, artifacts_manager) except AnsibleError as err: diff --git a/lib/ansible/galaxy/dependency_resolution/dataclasses.py b/lib/ansible/galaxy/dependency_resolution/dataclasses.py index ac1edc77356..2299ea5edf0 100644 --- a/lib/ansible/galaxy/dependency_resolution/dataclasses.py +++ b/lib/ansible/galaxy/dependency_resolution/dataclasses.py @@ -27,7 +27,7 @@ if t.TYPE_CHECKING: ) -from ansible.errors import AnsibleError +from ansible.errors import AnsibleError, AnsibleAssertionError from ansible.galaxy.api import GalaxyAPI from ansible.galaxy.collection import HAS_PACKAGING, PkgReq from ansible.module_utils._text import to_bytes, to_native, to_text @@ -584,3 +584,13 @@ class Candidate( def __init__(self, *args, **kwargs): super(Candidate, self).__init__() + + def with_signatures_repopulated(self): # type: (Candidate) -> Candidate + """Populate a new Candidate instance with Galaxy signatures. + :raises AnsibleAssertionError: If the supplied candidate is not sourced from a Galaxy-like index. + """ + if self.type != 'galaxy': + raise AnsibleAssertionError(f"Invalid collection type for {self!r}: unable to get signatures from a galaxy server.") + + signatures = self.src.get_collection_signatures(self.namespace, self.name, self.ver) + return self.__class__(self.fqcn, self.ver, self.src, self.type, frozenset([*self.signatures, *signatures])) diff --git a/lib/ansible/galaxy/dependency_resolution/providers.py b/lib/ansible/galaxy/dependency_resolution/providers.py index 1dcaf1c8d2c..93f0e7a00bf 100644 --- a/lib/ansible/galaxy/dependency_resolution/providers.py +++ b/lib/ansible/galaxy/dependency_resolution/providers.py @@ -392,7 +392,6 @@ class CollectionDependencyProviderBase(AbstractProvider): if not unsatisfied: if self._include_signatures: - signatures = src_server.get_collection_signatures(first_req.namespace, first_req.name, version) for extra_source in extra_signature_sources: signatures.append(get_signature_from_source(extra_source)) latest_matches.append( diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml index a225d6a3ba1..f349d4f6edd 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml @@ -5,7 +5,7 @@ state: directory - name: install simple collection from first accessible server - command: ansible-galaxy collection install namespace1.name1 {{ galaxy_verbosity }} + command: ansible-galaxy collection install namespace1.name1 -vvvv environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: from_first_good_server @@ -30,6 +30,7 @@ - install_normal_files.files[1].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] - install_normal_files.files[2].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] - (install_normal_manifest.content | b64decode | from_json).collection_info.version == '1.0.9' + - 'from_first_good_server.stdout|regex_findall("has not signed namespace1\.name1")|length == 1' - name: Remove the collection file: