From 143e7fb45e7b916fa973613000e97ee889f5666c Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Tue, 7 Jun 2022 09:49:43 -0400 Subject: [PATCH] ansible-galaxy - support resolvelib versions >= 0.5.3, < 0.9.0 (#77649) * ansible-galaxy - support resolvelib versions >= 0.5.3, <= 0.8.1 Test incompatibilities are removed for resolvelib >= 0.6.0 Test against the latest 0.8.x version and fix requirements * Fix tests - use a venv for testing the range of resolvelib versions * Update temporary hardcoded fallback for ansible-test * Update hardcoded upperbound for sanity tests * Make error check more flexible --- ...649-support-recent-resolvelib-versions.yml | 3 + .../galaxy/dependency_resolution/providers.py | 118 +++++++++++++-- requirements.txt | 4 +- .../tasks/main.yml | 4 + .../test_supported_resolvelib_versions.yml | 25 ++++ .../vars/main.yml | 6 + .../tasks/download.yml | 5 + .../tasks/fail_fast_resolvelib.yml | 45 ++++++ .../tasks/install.yml | 140 +++++++++--------- .../ansible-galaxy-collection/tasks/main.yml | 11 ++ .../tasks/supported_resolvelib.yml | 44 ++++++ .../tasks/unsupported_resolvelib.yml | 6 +- .../ansible-galaxy-collection/vars/main.yml | 7 +- .../_data/requirements/ansible.txt | 4 +- .../code-smell/docs-build.requirements.in | 2 +- .../code-smell/package-data.requirements.in | 2 +- 16 files changed, 338 insertions(+), 88 deletions(-) create mode 100644 changelogs/fragments/77649-support-recent-resolvelib-versions.yml create mode 100644 test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml create mode 100644 test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml create mode 100644 test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml diff --git a/changelogs/fragments/77649-support-recent-resolvelib-versions.yml b/changelogs/fragments/77649-support-recent-resolvelib-versions.yml new file mode 100644 index 00000000000..9c72f99f1fa --- /dev/null +++ b/changelogs/fragments/77649-support-recent-resolvelib-versions.yml @@ -0,0 +1,3 @@ +minor_changes: + - ansible-galaxy - Support resolvelib versions 0.6.x, 0.7.x, and 0.8.x. + The full range of supported versions is now >= 0.5.3, < 0.9.0. diff --git a/lib/ansible/galaxy/dependency_resolution/providers.py b/lib/ansible/galaxy/dependency_resolution/providers.py index 5a6fcacfd47..ccb56a9d797 100644 --- a/lib/ansible/galaxy/dependency_resolution/providers.py +++ b/lib/ansible/galaxy/dependency_resolution/providers.py @@ -41,7 +41,7 @@ except ImportError: # TODO: add python requirements to ansible-test's ansible-core distribution info and remove the hardcoded lowerbound/upperbound fallback RESOLVELIB_LOWERBOUND = SemanticVersion("0.5.3") -RESOLVELIB_UPPERBOUND = SemanticVersion("0.6.0") +RESOLVELIB_UPPERBOUND = SemanticVersion("0.9.0") RESOLVELIB_VERSION = SemanticVersion.from_loose_version(LooseVersion(resolvelib_version)) @@ -73,11 +73,11 @@ class PinnedCandidateRequests(Set): return len(self._candidates) -class CollectionDependencyProvider(AbstractProvider): +class CollectionDependencyProviderBase(AbstractProvider): """Delegate providing a requirement interface for the resolver.""" def __init__( - self, # type: CollectionDependencyProvider + self, # type: CollectionDependencyProviderBase apis, # type: MultiGalaxyAPIProxy concrete_artifacts_manager=None, # type: ConcreteArtifactsManager user_requirements=None, # type: t.Iterable[Requirement] @@ -180,12 +180,8 @@ class CollectionDependencyProvider(AbstractProvider): """ return requirement_or_candidate.canonical_package_id - def get_preference( - self, # type: CollectionDependencyProvider - resolution, # type: t.Optional[Candidate] - candidates, # type: list[Candidate] - information, # type: list[t.NamedTuple] - ): # type: (...) -> t.Union[float, int] + def get_preference(self, *args, **kwargs): + # type: (t.Any, t.Any) -> t.Union[float, int] """Return sort key function return value for given requirement. This result should be based on preference that is defined as @@ -193,6 +189,8 @@ class CollectionDependencyProvider(AbstractProvider): The lower the return value is, the more preferred this group of arguments is. + resolvelib >=0.5.3, <0.7.0 + :param resolution: Currently pinned candidate, or ``None``. :param candidates: A list of possible candidates. @@ -208,6 +206,35 @@ class CollectionDependencyProvider(AbstractProvider): (dependend on) the requirement, or `None` to indicate a root requirement. + resolvelib >=0.7.0, < 0.8.0 + + :param identifier: The value returned by ``identify()``. + + :param resolutions: Mapping of identifier, candidate pairs. + + :param candidates: Possible candidates for the identifer. + Mapping of identifier, list of candidate pairs. + + :param information: Requirement information of each package. + Mapping of identifier, list of named tuple pairs. + The named tuples have the entries ``requirement`` and ``parent``. + + resolvelib >=0.8.0, <= 0.8.1 + + :param identifier: The value returned by ``identify()``. + + :param resolutions: Mapping of identifier, candidate pairs. + + :param candidates: Possible candidates for the identifer. + Mapping of identifier, list of candidate pairs. + + :param information: Requirement information of each package. + Mapping of identifier, list of named tuple pairs. + The named tuples have the entries ``requirement`` and ``parent``. + + :param backtrack_causes: Sequence of requirement information that were + the requirements that caused the resolver to most recently backtrack. + The preference could depend on a various of issues, including (not necessarily in this order): @@ -229,6 +256,10 @@ class CollectionDependencyProvider(AbstractProvider): the value is, the more preferred this requirement is (i.e. the sorting function is called with ``reverse=False``). """ + raise NotImplementedError + + def _get_preference(self, candidates): + # type: (list[Candidate]) -> t.Union[float, int] if any( candidate in self._preferred_candidates for candidate in candidates @@ -238,8 +269,8 @@ class CollectionDependencyProvider(AbstractProvider): return float('-inf') return len(candidates) - def find_matches(self, requirements): - # type: (list[Requirement]) -> list[Candidate] + def find_matches(self, *args, **kwargs): + # type: (t.Any, t.Any) -> list[Candidate] r"""Find all possible candidates satisfying given requirements. This tries to get candidates based on the requirements' types. @@ -251,15 +282,31 @@ class CollectionDependencyProvider(AbstractProvider): to find concrete candidates for this requirement. Of theres a pre-installed candidate, it's prepended in front of others. + resolvelib >=0.5.3, <0.6.0 + :param requirements: A collection of requirements which all of \ the returned candidates must match. \ All requirements are guaranteed to have \ the same identifier. \ The collection is never empty. + resolvelib >=0.6.0 + + :param identifier: The value returned by ``identify()``. + + :param requirements: The requirements all returned candidates must satisfy. + Mapping of identifier, iterator of requirement pairs. + + :param incompatibilities: Incompatible versions that must be excluded + from the returned list. + :returns: An iterable that orders candidates by preference, \ e.g. the most preferred candidate comes first. """ + raise NotImplementedError + + def _find_matches(self, requirements): + # type: (list[Requirement]) -> list[Candidate] # FIXME: The first requirement may be a Git repo followed by # FIXME: its cloned tmp dir. Using only the first one creates # FIXME: loops that prevent any further dependency exploration. @@ -438,3 +485,52 @@ class CollectionDependencyProvider(AbstractProvider): self._make_req_from_dict({'name': dep_name, 'version': dep_req}) for dep_name, dep_req in req_map.items() ] + + +# Classes to handle resolvelib API changes between minor versions for 0.X +class CollectionDependencyProvider050(CollectionDependencyProviderBase): + def find_matches(self, requirements): # type: ignore[override] + # type: (list[Requirement]) -> list[Candidate] + return self._find_matches(requirements) + + def get_preference(self, resolution, candidates, information): # type: ignore[override] + # type: (t.Optional[Candidate], list[Candidate], list[t.NamedTuple]) -> t.Union[float, int] + return self._get_preference(candidates) + + +class CollectionDependencyProvider060(CollectionDependencyProviderBase): + def find_matches(self, identifier, requirements, incompatibilities): # type: ignore[override] + # type: (str, t.Mapping[str, t.Iterator[Requirement]], t.Mapping[str, t.Iterator[Requirement]]) -> list[Candidate] + return [ + match for match in self._find_matches(list(requirements[identifier])) + if not any(match.ver == incompat.ver for incompat in incompatibilities[identifier]) + ] + + def get_preference(self, resolution, candidates, information): # type: ignore[override] + # type: (t.Optional[Candidate], list[Candidate], list[t.NamedTuple]) -> t.Union[float, int] + return self._get_preference(candidates) + + +class CollectionDependencyProvider070(CollectionDependencyProvider060): + def get_preference(self, identifier, resolutions, candidates, information): # type: ignore[override] + # type: (str, t.Mapping[str, Candidate], t.Mapping[str, t.Iterator[Candidate]], t.Iterator[t.NamedTuple]) -> t.Union[float, int] + return self._get_preference(list(candidates[identifier])) + + +class CollectionDependencyProvider080(CollectionDependencyProvider060): + def get_preference(self, identifier, resolutions, candidates, information, backtrack_causes): # type: ignore[override] + # type: (str, t.Mapping[str, Candidate], t.Mapping[str, t.Iterator[Candidate]], t.Iterator[t.NamedTuple], t.Sequence) -> t.Union[float, int] + return self._get_preference(list(candidates[identifier])) + + +def _get_provider(): # type () -> CollectionDependencyProviderBase + if RESOLVELIB_VERSION >= SemanticVersion("0.8.0"): + return CollectionDependencyProvider080 + if RESOLVELIB_VERSION >= SemanticVersion("0.7.0"): + return CollectionDependencyProvider070 + if RESOLVELIB_VERSION >= SemanticVersion("0.6.0"): + return CollectionDependencyProvider060 + return CollectionDependencyProvider050 + + +CollectionDependencyProvider = _get_provider() diff --git a/requirements.txt b/requirements.txt index a732a5951d2..9f25e7e4a22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ packaging # NOTE: resolvelib 0.x version bumps should be considered major/breaking # NOTE: and we should update the upper cap with care, at least until 1.0 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69 -resolvelib >= 0.5.3, < 0.6.0 # dependency resolver used by ansible-galaxy +# NOTE: When updating the upper bound, also update the latest version used +# NOTE: in the ansible-galaxy-collection test suite. +resolvelib >= 0.5.3, < 0.9.0 # dependency resolver used by ansible-galaxy diff --git a/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml index 986da2f98d6..546c408395c 100644 --- a/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml @@ -23,6 +23,10 @@ - include_tasks: ./multi_collection_repo_individual.yml - include_tasks: ./setup_recursive_scm_dependency.yml - include_tasks: ./scm_dependency_deduplication.yml + - include_tasks: ./test_supported_resolvelib_versions.yml + loop: "{{ supported_resolvelib_versions }}" + loop_control: + loop_var: resolvelib_version - include_tasks: ./download.yml - include_tasks: ./setup_collection_bad_version.yml - include_tasks: ./test_invalid_version.yml diff --git a/test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml b/test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml new file mode 100644 index 00000000000..029cbb3a82b --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml @@ -0,0 +1,25 @@ +- vars: + venv_cmd: "{{ ansible_python_interpreter ~ ' -m venv' }}" + venv_dest: "{{ galaxy_dir }}/test_venv_{{ resolvelib_version }}" + block: + - name: install another version of resolvelib that is supported by ansible-galaxy + pip: + name: resolvelib + version: "{{ resolvelib_version }}" + state: present + virtualenv_command: "{{ venv_cmd }}" + virtualenv: "{{ venv_dest }}" + virtualenv_site_packages: True + + - include_tasks: ./scm_dependency_deduplication.yml + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + + always: + - name: remove test venv + file: + path: "{{ venv_dest }}" + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml b/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml index a82f25dc0fe..cd198c64b9f 100644 --- a/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml +++ b/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml @@ -3,3 +3,9 @@ alt_install_path: "{{ galaxy_dir }}/other_collections/ansible_collections" scm_path: "{{ galaxy_dir }}/development" test_repo_path: "{{ galaxy_dir }}/development/ansible_test" test_error_repo_path: "{{ galaxy_dir }}/development/error_test" + +supported_resolvelib_versions: + - "0.5.3" # Oldest supported + - "0.6.0" + - "0.7.0" + - "0.8.0" diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml index e00d0b833af..b651a73e73e 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml @@ -169,3 +169,8 @@ that: - '"Downloading collection ''ansible_test.my_collection:1.0.0'' to" in download_collection.stdout' - download_collection_actual.stat.exists + +- name: remove test download dir + file: + path: '{{ galaxy_dir }}/download' + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml b/test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml new file mode 100644 index 00000000000..eb471f8e7dd --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml @@ -0,0 +1,45 @@ +# resolvelib>=0.6.0 added an 'incompatibilities' parameter to find_matches +# If incompatibilities aren't removed from the viable candidates, this example causes infinite resursion +- name: test resolvelib removes incompatibilites in find_matches and errors quickly (prevent infinite recursion) + block: + - name: create collection dir + file: + dest: "{{ galaxy_dir }}/resolvelib/ns/coll" + state: directory + + - name: create galaxy.yml with a dependecy on a galaxy-sourced collection + copy: + dest: "{{ galaxy_dir }}/resolvelib/ns/coll/galaxy.yml" + content: | + namespace: ns + name: coll + authors: + - ansible-core + readme: README.md + version: "1.0.0" + dependencies: + namespace1.name1: "0.0.5" + + - name: build the collection + command: ansible-galaxy collection build ns/coll + args: + chdir: "{{ galaxy_dir }}/resolvelib" + + - name: install a conflicting version of the dep with the tarfile (expected failure) + command: ansible-galaxy collection install namespace1.name1:1.0.9 ns-coll-1.0.0.tar.gz -vvvvv -s {{ test_name }} -p collections/ + args: + chdir: "{{ galaxy_dir }}/resolvelib" + timeout: 30 + ignore_errors: yes + register: incompatible + + - assert: + that: + - incompatible.failed + - not incompatible.msg.startswith("The command action failed to execute in the expected time frame") + + always: + - name: cleanup resolvelib test + file: + dest: "{{ galaxy_dir }}/resolvelib" + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml index d23c9cb3ff0..dabaa117339 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml @@ -1,5 +1,5 @@ --- -- name: create test collection install directory - {{ test_name }} +- name: create test collection install directory - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: directory @@ -36,24 +36,24 @@ path: '{{ galaxy_dir }}/ansible_collections/namespace1' state: absent -- name: install simple collection with implicit path - {{ test_name }} +- name: install simple collection with implicit path - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: install_normal -- name: get installed files of install simple collection with implicit path - {{ test_name }} +- name: get installed files of install simple collection with implicit path - {{ test_id }} find: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1' file_type: file register: install_normal_files -- name: get the manifest of install simple collection with implicit path - {{ test_name }} +- name: get the manifest of install simple collection with implicit path - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' register: install_normal_manifest -- name: assert install simple collection with implicit path - {{ test_name }} +- name: assert install simple collection with implicit path - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.0.9'' to" in install_normal.stdout' @@ -63,43 +63,43 @@ - install_normal_files.files[2].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] - (install_normal_manifest.content | b64decode | from_json).collection_info.version == '1.0.9' -- name: install existing without --force - {{ test_name }} +- name: install existing without --force - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: install_existing_no_force -- name: assert install existing without --force - {{ test_name }} +- name: assert install existing without --force - {{ test_id }} assert: that: - '"Nothing to do. All requested collections are already installed" in install_existing_no_force.stdout' -- name: install existing with --force - {{ test_name }} +- name: install existing with --force - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' --force {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: install_existing_force -- name: assert install existing with --force - {{ test_name }} +- name: assert install existing with --force - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.0.9'' to" in install_existing_force.stdout' -- name: remove test installed collection - {{ test_name }} +- name: remove test installed collection - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections/namespace1' state: absent -- name: install pre-release as explicit version to custom dir - {{ test_name }} +- name: install pre-release as explicit version to custom dir - {{ test_id }} command: ansible-galaxy collection install 'namespace1.name1:1.1.0-beta.1' -s '{{ test_name }}' -p '{{ galaxy_dir }}/ansible_collections' {{ galaxy_verbosity }} register: install_prerelease -- name: get result of install pre-release as explicit version to custom dir - {{ test_name }} +- name: get result of install pre-release as explicit version to custom dir - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' register: install_prerelease_actual -- name: assert install pre-release as explicit version to custom dir - {{ test_name }} +- name: assert install pre-release as explicit version to custom dir - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.1.0-beta.1'' to" in install_prerelease.stdout' @@ -110,22 +110,22 @@ path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1' state: absent -- name: install pre-release version with --pre to custom dir - {{ test_name }} +- name: install pre-release version with --pre to custom dir - {{ test_id }} command: ansible-galaxy collection install --pre 'namespace1.name1' -s '{{ test_name }}' -p '{{ galaxy_dir }}/ansible_collections' {{ galaxy_verbosity }} register: install_prerelease -- name: get result of install pre-release version with --pre to custom dir - {{ test_name }} +- name: get result of install pre-release version with --pre to custom dir - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' register: install_prerelease_actual -- name: assert install pre-release version with --pre to custom dir - {{ test_name }} +- name: assert install pre-release version with --pre to custom dir - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.1.0-beta.1'' to" in install_prerelease.stdout' - (install_prerelease_actual.content | b64decode | from_json).collection_info.version == '1.1.0-beta.1' -- name: install multiple collections with dependencies - {{ test_name }} +- name: install multiple collections with dependencies - {{ test_id }} command: ansible-galaxy collection install parent_dep.parent_collection:1.0.0 namespace2.name -s {{ test_name }} {{ galaxy_verbosity }} args: chdir: '{{ galaxy_dir }}/ansible_collections' @@ -134,7 +134,7 @@ ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' register: install_multiple_with_dep -- name: get result of install multiple collections with dependencies - {{ test_name }} +- name: get result of install multiple collections with dependencies - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection.namespace }}/{{ collection.name }}/MANIFEST.json' register: install_multiple_with_dep_actual @@ -150,7 +150,7 @@ - namespace: child_dep name: child_dep2 -- name: assert install multiple collections with dependencies - {{ test_name }} +- name: assert install multiple collections with dependencies - {{ test_id }} assert: that: - (install_multiple_with_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' @@ -158,7 +158,7 @@ - (install_multiple_with_dep_actual.results[2].content | b64decode | from_json).collection_info.version == '0.9.9' - (install_multiple_with_dep_actual.results[3].content | b64decode | from_json).collection_info.version == '1.2.2' -- name: expect failure with dep resolution failure +- name: expect failure with dep resolution failure - {{ test_id }} command: ansible-galaxy collection install fail_namespace.fail_collection -s {{ test_name }} {{ galaxy_verbosity }} register: fail_dep_mismatch failed_when: @@ -173,23 +173,23 @@ force_basic_auth: true register: artifact_url_response -- name: download a collection for an offline install - {{ test_name }} +- name: download a collection for an offline install - {{ test_id }} get_url: url: '{{ artifact_url_response.json.download_url }}' dest: '{{ galaxy_dir }}/namespace3.tar.gz' -- name: install a collection from a tarball - {{ test_name }} +- name: install a collection from a tarball - {{ test_id }} command: ansible-galaxy collection install '{{ galaxy_dir }}/namespace3.tar.gz' {{ galaxy_verbosity }} register: install_tarball environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collection from a tarball - {{ test_name }} +- name: get result of install collection from a tarball - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace3/name/MANIFEST.json' register: install_tarball_actual -- name: assert install a collection from a tarball - {{ test_name }} +- name: assert install a collection from a tarball - {{ test_id }} assert: that: - '"Installing ''namespace3.name:1.0.0'' to" in install_tarball.stdout' @@ -272,22 +272,22 @@ - "{{ galaxy_dir }}/scratch/tmp_parent/" - "{{ galaxy_dir }}/tmp_parent-name-1.0.0.tar.gz" -- name: setup bad tarball - {{ test_name }} +- name: setup bad tarball - {{ test_id }} script: build_bad_tar.py {{ galaxy_dir | quote }} -- name: fail to install a collection from a bad tarball - {{ test_name }} +- name: fail to install a collection from a bad tarball - {{ test_id }} command: ansible-galaxy collection install '{{ galaxy_dir }}/suspicious-test-1.0.0.tar.gz' {{ galaxy_verbosity }} register: fail_bad_tar failed_when: fail_bad_tar.rc != 1 and "Cannot extract tar entry '../../outside.sh' as it will be placed outside the collection directory" not in fail_bad_tar.stderr environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of failed collection install - {{ test_name }} +- name: get result of failed collection install - {{ test_id }} stat: path: '{{ galaxy_dir }}/ansible_collections\suspicious' register: fail_bad_tar_actual -- name: assert result of failed collection install - {{ test_name }} +- name: assert result of failed collection install - {{ test_id }} assert: that: - not fail_bad_tar_actual.stat.exists @@ -300,24 +300,24 @@ force_basic_auth: true register: artifact_url_response -- name: install a collection from a URI - {{ test_name }} +- name: install a collection from a URI - {{ test_id }} command: ansible-galaxy collection install {{ artifact_url_response.json.download_url}} {{ galaxy_verbosity }} register: install_uri environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collection from a URI - {{ test_name }} +- name: get result of install collection from a URI - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace4/name/MANIFEST.json' register: install_uri_actual -- name: assert install a collection from a URI - {{ test_name }} +- name: assert install a collection from a URI - {{ test_id }} assert: that: - '"Installing ''namespace4.name:1.0.0'' to" in install_uri.stdout' - (install_uri_actual.content | b64decode | from_json).collection_info.version == '1.0.0' -- name: fail to install a collection with an undefined URL - {{ test_name }} +- name: fail to install a collection with an undefined URL - {{ test_id }} command: ansible-galaxy collection install namespace5.name {{ galaxy_verbosity }} register: fail_undefined_server failed_when: '"No setting was provided for required configuration plugin_type: galaxy_server plugin: undefined" not in fail_undefined_server.stderr' @@ -326,25 +326,25 @@ - when: not requires_auth block: - - name: install a collection with an empty server list - {{ test_name }} + - name: install a collection with an empty server list - {{ test_id }} command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' {{ galaxy_verbosity }} register: install_empty_server_list environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' ANSIBLE_GALAXY_SERVER_LIST: '' - - name: get result of a collection with an empty server list - {{ test_name }} + - name: get result of a collection with an empty server list - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace5/name/MANIFEST.json' register: install_empty_server_list_actual - - name: assert install a collection with an empty server list - {{ test_name }} + - name: assert install a collection with an empty server list - {{ test_id }} assert: that: - '"Installing ''namespace5.name:1.0.0'' to" in install_empty_server_list.stdout' - (install_empty_server_list_actual.content | b64decode | from_json).collection_info.version == '1.0.0' -- name: create test requirements file with both roles and collections - {{ test_name }} +- name: create test requirements file with both roles and collections - {{ test_id }} copy: content: | collections: @@ -368,13 +368,13 @@ - "'unrecognized arguments: --keyring' in invalid_opt.stderr" # Need to run with -vvv to validate the roles will be skipped msg -- name: install collections only with requirements-with-role.yml - {{ test_name }} +- name: install collections only with requirements-with-role.yml - {{ test_id }} command: ansible-galaxy collection install -r '{{ galaxy_dir }}/ansible_collections/requirements-with-role.yml' -s '{{ test_name }}' -vvv register: install_req_collection environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collections only with requirements-with-roles.yml - {{ test_name }} +- name: get result of install collections only with requirements-with-roles.yml - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_collection_actual @@ -384,7 +384,7 @@ - namespace6 - namespace7 -- name: assert install collections only with requirements-with-role.yml - {{ test_name }} +- name: assert install collections only with requirements-with-role.yml - {{ test_id }} assert: that: - '"contains roles which will be ignored" in install_req_collection.stdout' @@ -393,7 +393,7 @@ - (install_req_collection_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' - (install_req_collection_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' -- name: create test requirements file with just collections - {{ test_name }} +- name: create test requirements file with just collections - {{ test_id }} copy: content: | collections: @@ -401,13 +401,13 @@ - name: namespace9.name dest: '{{ galaxy_dir }}/ansible_collections/requirements.yaml' -- name: install collections with ansible-galaxy install - {{ test_name }} +- name: install collections with ansible-galaxy install - {{ test_id }} command: ansible-galaxy install -r '{{ galaxy_dir }}/ansible_collections/requirements.yaml' -s '{{ test_name }}' register: install_req environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -417,7 +417,7 @@ - namespace8 - namespace9 -- name: assert install collections with ansible-galaxy install - {{ test_name }} +- name: assert install collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''namespace8.name:1.0.0'' to" in install_req.stdout' @@ -514,7 +514,7 @@ - required_together is failed - '"ERROR! Signatures were provided to verify namespace1.name1 but no keyring was configured." in required_together.stderr' -- name: install collections with ansible-galaxy install -r with invalid signatures - {{ test_name }} +- name: install collections with ansible-galaxy install -r with invalid signatures - {{ test_id }} # Note that --keyring is a valid option for 'ansible-galaxy install -r ...', not just 'ansible-galaxy collection ...' command: ansible-galaxy install -r {{ req_file }} -s {{ test_name }} --keyring {{ keyring }} {{ galaxy_verbosity }} register: install_req @@ -526,7 +526,7 @@ ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' ANSIBLE_GALAXY_REQUIRED_VALID_SIGNATURE_COUNT: all -- name: assert invalid signature is fatal with ansible-galaxy install - {{ test_name }} +- name: assert invalid signature is fatal with ansible-galaxy install - {{ test_id }} assert: that: - install_req is failed @@ -538,7 +538,7 @@ - '"Installing ''namespace9.name:1.0.0'' to" not in install_req.stdout' # This command is hardcoded with -vvvv purposefully to evaluate extra verbosity messages -- name: install collections with ansible-galaxy install and --ignore-errors - {{ test_name }} +- name: install collections with ansible-galaxy install and --ignore-errors - {{ test_id }} command: ansible-galaxy install -r {{ req_file }} {{ cli_opts }} -vvvv register: install_req vars: @@ -551,7 +551,7 @@ ANSIBLE_NOCOLOR: True ANSIBLE_FORCE_COLOR: False -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -562,7 +562,7 @@ - namespace9 # SIVEL -- name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_name }} +- name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_id }} assert: that: - install_req is success @@ -587,7 +587,7 @@ - namespace8 - namespace9 -- name: install collections with only one valid signature using ansible-galaxy install - {{ test_name }} +- name: install collections with only one valid signature using ansible-galaxy install - {{ test_id }} command: ansible-galaxy install -r {{ req_file }} {{ cli_opts }} {{ galaxy_verbosity }} register: install_req vars: @@ -599,7 +599,7 @@ ANSIBLE_NOCOLOR: True ANSIBLE_FORCE_COLOR: False -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -610,7 +610,7 @@ - namespace8 - namespace9 -- name: assert just one valid signature is not fatal with ansible-galaxy install - {{ test_name }} +- name: assert just one valid signature is not fatal with ansible-galaxy install - {{ test_id }} assert: that: - install_req is success @@ -650,7 +650,7 @@ ANSIBLE_NOCOLOR: True ANSIBLE_FORCE_COLOR: False -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -661,7 +661,7 @@ - namespace8 - namespace9 -- name: assert invalid signature is not fatal with ansible-galaxy install - {{ test_name }} +- name: assert invalid signature is not fatal with ansible-galaxy install - {{ test_id }} assert: that: - install_req is success @@ -704,24 +704,24 @@ # name: cache # version: 1.0.{{ cache_version_build }} # -#- name: make sure the cache version list is ignored on a collection version change - {{ test_name }} +#- name: make sure the cache version list is ignored on a collection version change - {{ test_id }} # command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv # register: install_cached_update # environment: # ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' # -#- name: get result of cache version list is ignored on a collection version change - {{ test_name }} +#- name: get result of cache version list is ignored on a collection version change - {{ test_id }} # slurp: # path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json' # register: install_cached_update_actual # -#- name: assert cache version list is ignored on a collection version change - {{ test_name }} +#- name: assert cache version list is ignored on a collection version change - {{ test_id }} # assert: # that: # - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout' # - (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build -- name: install collection with symlink - {{ test_name }} +- name: install collection with symlink - {{ test_id }} command: ansible-galaxy collection install symlink.symlink -s '{{ test_name }}' {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' @@ -732,7 +732,7 @@ recurse: yes file_type: any -- name: get result of install collection with symlink - {{ test_name }} +- name: get result of install collection with symlink - {{ test_id }} stat: path: '{{ galaxy_dir }}/ansible_collections/symlink/symlink/{{ path }}' register: install_symlink_actual @@ -746,7 +746,7 @@ - docs-link - docs-link/REÅDMÈ.md -- name: assert install collection with symlink - {{ test_name }} +- name: assert install collection with symlink - {{ test_id }} assert: that: - '"Installing ''symlink.symlink:1.0.0'' to" in install_symlink.stdout' @@ -762,18 +762,18 @@ - install_symlink_actual.results[5].stat.islnk - install_symlink_actual.results[5].stat.lnk_target == '../REÅDMÈ.md' -- name: remove install directory for the next test because parent_dep.parent_collection was installed - {{ test_name }} +- name: remove install directory for the next test because parent_dep.parent_collection was installed - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: absent -- name: install collection and dep compatible with multiple requirements - {{ test_name }} +- name: install collection and dep compatible with multiple requirements - {{ test_id }} command: ansible-galaxy collection install parent_dep.parent_collection parent_dep2.parent_collection environment: ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' register: install_req -- name: assert install collections with ansible-galaxy install - {{ test_name }} +- name: assert install collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''parent_dep.parent_collection:1.0.0'' to" in install_req.stdout' @@ -789,18 +789,18 @@ state: directory path: '{{ galaxy_dir }}/ansible_collections/unrelated_namespace/collection_without_metadata/plugins' - - name: install a collection to the same installation directory - {{ test_name }} + - name: install a collection to the same installation directory - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 environment: ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' register: install_req - - name: assert installed collections with ansible-galaxy install - {{ test_name }} + - name: assert installed collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.0.9'' to" in install_req.stdout' -- name: remove test collection install directory - {{ test_name }} +- name: remove test collection install directory - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: absent @@ -991,10 +991,10 @@ path: '{{ galaxy_dir }}/ansible_collections/namespace1' state: absent -- name: download collections with pre-release dep - {{ test_name }} +- name: download collections with pre-release dep - {{ test_id }} command: ansible-galaxy collection download dep_with_beta.parent namespace1.name1:1.1.0-beta.1 -p '{{ galaxy_dir }}/scratch' -- name: install collection with concrete pre-release dep - {{ test_name }} +- name: install collection with concrete pre-release dep - {{ test_id }} command: ansible-galaxy collection install -r '{{ galaxy_dir }}/scratch/requirements.yml' args: chdir: '{{ galaxy_dir }}/scratch' @@ -1002,7 +1002,7 @@ ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' register: install_concrete_pre -- name: get result of install collections with concrete pre-release dep - {{ test_name }} +- name: get result of install collections with concrete pre-release dep - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/MANIFEST.json' register: install_concrete_pre_actual @@ -1012,7 +1012,7 @@ - namespace1/name1 - dep_with_beta/parent -- name: assert install collections with ansible-galaxy install - {{ test_name }} +- name: assert install collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.1.0-beta.1'' to" in install_concrete_pre.stdout' @@ -1020,7 +1020,7 @@ - (install_concrete_pre_actual.results[0].content | b64decode | from_json).collection_info.version == '1.1.0-beta.1' - (install_concrete_pre_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' -- name: remove collection dir after round of testing - {{ test_name }} +- name: remove collection dir after round of testing - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml index c3b124e9951..063b7f0896f 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml @@ -98,6 +98,7 @@ - name: run ansible-galaxy collection install tests for {{ test_name }} include_tasks: install.yml vars: + test_id: '{{ item.name }}' test_name: '{{ item.name }}' test_server: '{{ item.server }}' vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}' @@ -117,6 +118,16 @@ server: '{{ pulp_server }}published/api/' v3: true +- name: test installing and downloading collections with the range of supported resolvelib versions + include_tasks: supported_resolvelib.yml + args: + apply: + environment: + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + loop: '{{ supported_resolvelib_versions }}' + loop_control: + loop_var: resolvelib_version + - name: publish collection with a dep on another server setup_collections: server: secondary diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml b/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml new file mode 100644 index 00000000000..763c5a19f7f --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml @@ -0,0 +1,44 @@ +- vars: + venv_cmd: "{{ ansible_python_interpreter ~ ' -m venv' }}" + venv_dest: "{{ galaxy_dir }}/test_venv_{{ resolvelib_version }}" + block: + - name: install another version of resolvelib that is supported by ansible-galaxy + pip: + name: resolvelib + version: "{{ resolvelib_version }}" + state: present + virtualenv_command: "{{ venv_cmd }}" + virtualenv: "{{ venv_dest }}" + virtualenv_site_packages: True + + - include_tasks: fail_fast_resolvelib.yml + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + + - include_tasks: install.yml + vars: + test_name: pulp_v3 + test_id: '{{ test_name }} (resolvelib {{ resolvelib_version }})' + test_server: '{{ pulp_server }}published/api/' + vX: "v3/" + requires_auth: false + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + + - include_tasks: download.yml + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + always: + - name: remove test venv + file: + path: "{{ venv_dest }}" + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/unsupported_resolvelib.yml b/test/integration/targets/ansible-galaxy-collection/tasks/unsupported_resolvelib.yml index d9667b93316..a208b2952e3 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/unsupported_resolvelib.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/unsupported_resolvelib.yml @@ -27,10 +27,12 @@ - assert: that: - resolvelib_version_error is failed - - compat_error in resolvelib_version_error.stderr or import_error in resolvelib_version_error.stderr + - resolvelib_version_error.stderr | regex_search(error) vars: + error: "({{ import_error }}|{{ compat_error }})" import_error: "Failed to import resolvelib" - compat_error: "ansible-galaxy requires resolvelib<0.6.0,>=0.5.3" + compat_error: "ansible-galaxy requires resolvelib<{{major_minor_patch}},>={{major_minor_patch}}" + major_minor_patch: "[0-9]\\d*\\.[0-9]\\d*\\.[0-9]\\d*" always: - name: cleanup venv and install directory diff --git a/test/integration/targets/ansible-galaxy-collection/vars/main.yml b/test/integration/targets/ansible-galaxy-collection/vars/main.yml index 260f90e70f2..12e968aa548 100644 --- a/test/integration/targets/ansible-galaxy-collection/vars/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/vars/main.yml @@ -2,10 +2,15 @@ galaxy_verbosity: "{{ '' if not ansible_verbosity else '-' ~ ('v' * ansible_verb gpg_homedir: "{{ galaxy_dir }}/gpg" +supported_resolvelib_versions: + - "0.5.3" # Oldest supported + - "0.6.0" + - "0.7.0" + - "0.8.0" + unsupported_resolvelib_versions: - "0.2.0" # Fails on import - "0.5.1" - - "0.6.0" # Fails on dependency resolution pulp_repositories: - published diff --git a/test/lib/ansible_test/_data/requirements/ansible.txt b/test/lib/ansible_test/_data/requirements/ansible.txt index a732a5951d2..9f25e7e4a22 100644 --- a/test/lib/ansible_test/_data/requirements/ansible.txt +++ b/test/lib/ansible_test/_data/requirements/ansible.txt @@ -10,4 +10,6 @@ packaging # NOTE: resolvelib 0.x version bumps should be considered major/breaking # NOTE: and we should update the upper cap with care, at least until 1.0 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69 -resolvelib >= 0.5.3, < 0.6.0 # dependency resolver used by ansible-galaxy +# NOTE: When updating the upper bound, also update the latest version used +# NOTE: in the ansible-galaxy-collection test suite. +resolvelib >= 0.5.3, < 0.9.0 # dependency resolver used by ansible-galaxy diff --git a/test/sanity/code-smell/docs-build.requirements.in b/test/sanity/code-smell/docs-build.requirements.in index f4f8c9b01fa..797ca326b1a 100644 --- a/test/sanity/code-smell/docs-build.requirements.in +++ b/test/sanity/code-smell/docs-build.requirements.in @@ -1,6 +1,6 @@ jinja2 pyyaml -resolvelib < 0.6.0 +resolvelib < 0.9.0 sphinx == 4.2.0 sphinx-notfound-page sphinx-ansible-theme diff --git a/test/sanity/code-smell/package-data.requirements.in b/test/sanity/code-smell/package-data.requirements.in index 68c2248e581..fd362d0eaf4 100644 --- a/test/sanity/code-smell/package-data.requirements.in +++ b/test/sanity/code-smell/package-data.requirements.in @@ -1,7 +1,7 @@ docutils < 0.18 # match version required by sphinx in the docs-build sanity test jinja2 pyyaml # ansible-core requirement -resolvelib < 0.6.0 +resolvelib < 0.9.0 rstcheck straight.plugin antsibull-changelog