From 67d9cc45bde602b43c93ede4fe0bf5bb2b51a184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= <4622652+pjrm@users.noreply.github.com> Date: Tue, 8 Oct 2019 15:24:50 +0100 Subject: [PATCH] maven_artifact.py - add support for version ranges by using spec (#54309) (#61813) --- .../packaging/language/maven_artifact.py | 123 +++++++++++++++--- .../_data/requirements/constraints.txt | 1 + .../ansible_test/_data/requirements/units.txt | 3 + test/sanity/ignore.txt | 1 - .../packaging/language/test_maven_artifact.py | 70 ++++++++++ 5 files changed, 181 insertions(+), 17 deletions(-) create mode 100644 test/units/modules/packaging/language/test_maven_artifact.py diff --git a/lib/ansible/modules/packaging/language/maven_artifact.py b/lib/ansible/modules/packaging/language/maven_artifact.py index 83255ab07e6..b1ed404b890 100644 --- a/lib/ansible/modules/packaging/language/maven_artifact.py +++ b/lib/ansible/modules/packaging/language/maven_artifact.py @@ -39,7 +39,14 @@ options: version: description: - The maven version coordinate - default: latest + - Mutually exclusive with I(version_by_spec). + version_by_spec: + description: + - The maven dependency version ranges. + - See supported version ranges on U(https://cwiki.apache.org/confluence/display/MAVENOLD/Dependency+Mediation+and+Conflict+Resolution) + - The range type "(,1.0],[1.2,)" and "(,1.1),(1.1,)" is not supported. + - Mutually exclusive with I(version). + version_added: "2.10" classifier: description: - The maven classifier coordinate @@ -90,7 +97,8 @@ options: keep_name: description: - If C(yes), the downloaded artifact's name is preserved, i.e the version number remains part of it. - - This option only has effect when C(dest) is a directory and C(version) is set to C(latest). + - This option only has effect when C(dest) is a directory and C(version) is set to C(latest) or C(version_by_spec) + is defined. type: bool default: 'no' version_added: "2.4" @@ -158,6 +166,13 @@ EXAMPLES = ''' artifact_id: junit dest: /tmp/junit-latest.jar repository_url: "file://{{ lookup('env','HOME') }}/.m2/repository" + +# Download the latest version between 3.8 and 4.0 (exclusive) of the JUnit framework artifact from Maven Central +- maven_artifact: + group_id: junit + artifact_id: junit + version_by_spec: "[3.8,4.0)" + dest: /tmp/ ''' import hashlib @@ -168,6 +183,9 @@ import io import tempfile import traceback +from ansible.module_utils.ansible_release import __version__ as ansible_version +from re import match + LXML_ETREE_IMP_ERR = None try: from lxml import etree @@ -184,6 +202,15 @@ except ImportError: BOTO_IMP_ERR = traceback.format_exc() HAS_BOTO = False +SEMANTIC_VERSION_IMP_ERR = None +try: + from semantic_version import Version, Spec + HAS_SEMANTIC_VERSION = True +except ImportError: + SEMANTIC_VERSION_IMP_ERR = traceback.format_exc() + HAS_SEMANTIC_VERSION = False + + from ansible.module_utils.basic import AnsibleModule, missing_required_lib from ansible.module_utils.six.moves.urllib.parse import urlparse from ansible.module_utils.urls import fetch_url @@ -224,7 +251,7 @@ def adjust_recursive_directory_permissions(pre_existing_dir, new_directory_list, class Artifact(object): - def __init__(self, group_id, artifact_id, version, classifier='', extension='jar'): + def __init__(self, group_id, artifact_id, version, version_by_spec, classifier='', extension='jar'): if not group_id: raise ValueError("group_id must be set") if not artifact_id: @@ -233,6 +260,7 @@ class Artifact(object): self.group_id = group_id self.artifact_id = artifact_id self.version = version + self.version_by_spec = version_by_spec self.classifier = classifier if not extension: @@ -290,17 +318,62 @@ class Artifact(object): class MavenDownloader: - def __init__(self, module, base="http://repo1.maven.org/maven2", local=False, headers=None): + def __init__(self, module, base, local=False, headers=None): self.module = module if base.endswith("/"): base = base.rstrip("/") self.base = base self.local = local self.headers = headers - self.user_agent = "Ansible {0} maven_artifact".format(self.module.ansible_version) + self.user_agent = "Ansible {0} maven_artifact".format(ansible_version) self.latest_version_found = None self.metadata_file_name = "maven-metadata-local.xml" if local else "maven-metadata.xml" + def find_version_by_spec(self, artifact): + path = "/%s/%s" % (artifact.path(False), self.metadata_file_name) + content = self._getContent(self.base + path, "Failed to retrieve the maven metadata file: " + path) + xml = etree.fromstring(content) + original_versions = xml.xpath("/metadata/versioning/versions/version/text()") + versions = [] + for version in original_versions: + try: + versions.append(Version.coerce(version)) + except ValueError: + # This means that version string is not a valid semantic versioning + pass + + parse_versions_syntax = { + # example -> (,1.0] + r"^\(,(?P[0-9.]*)]$": "<={upper_bound}", + # example -> 1.0 + r"^(?P[0-9.]*)$": "~={version}", + # example -> [1.0] + r"^\[(?P[0-9.]*)\]$": "=={version}", + # example -> [1.2, 1.3] + r"^\[(?P[0-9.]*),\s*(?P[0-9.]*)\]$": ">={lower_bound},<={upper_bound}", + # example -> [1.2, 1.3) + r"^\[(?P[0-9.]*),\s*(?P[0-9.]+)\)$": ">={lower_bound},<{upper_bound}", + # example -> [1.5,) + r"^\[(?P[0-9.]*),\)$": ">={lower_bound}", + } + + for regex, spec_format in parse_versions_syntax.items(): + regex_result = match(regex, artifact.version_by_spec) + if regex_result: + spec = Spec(spec_format.format(**regex_result.groupdict())) + selected_version = spec.select(versions) + + if not selected_version: + raise ValueError("No version found with this spec version: {0}".format(artifact.version_by_spec)) + + # To deal when repos on maven don't have patch number on first build (e.g. 3.8 instead of 3.8.0) + if str(selected_version) not in original_versions: + selected_version.patch = None + + return str(selected_version) + + raise ValueError("The spec version {0} is not supported! ".format(artifact.version_by_spec)) + def find_latest_version_available(self, artifact): if self.latest_version_found: return self.latest_version_found @@ -313,6 +386,9 @@ class MavenDownloader: return v[0] def find_uri_for_artifact(self, artifact): + if artifact.version_by_spec: + artifact.version = self.find_version_by_spec(artifact) + if artifact.version == "latest": artifact.version = self.find_latest_version_available(artifact) @@ -390,8 +466,8 @@ class MavenDownloader: return None def download(self, tmpdir, artifact, verify_download, filename=None): - if not artifact.version or artifact.version == "latest": - artifact = Artifact(artifact.group_id, artifact.artifact_id, self.find_latest_version_available(artifact), + if (not artifact.version and not artifact.version_by_spec) or artifact.version == "latest": + artifact = Artifact(artifact.group_id, artifact.artifact_id, self.find_latest_version_available(artifact), None, artifact.classifier, artifact.extension) url = self.find_uri_for_artifact(artifact) tempfd, tempname = tempfile.mkstemp(dir=tmpdir) @@ -464,10 +540,11 @@ def main(): argument_spec=dict( group_id=dict(required=True), artifact_id=dict(required=True), - version=dict(default="latest"), + version=dict(default=None), + version_by_spec=dict(default=None), classifier=dict(default=''), extension=dict(default='jar'), - repository_url=dict(default=None), + repository_url=dict(default='http://repo1.maven.org/maven2'), username=dict(default=None, aliases=['aws_secret_key']), password=dict(default=None, no_log=True, aliases=['aws_secret_access_key']), headers=dict(type='dict'), @@ -478,12 +555,16 @@ def main(): keep_name=dict(required=False, default=False, type='bool'), verify_checksum=dict(required=False, default='download', choices=['never', 'download', 'change', 'always']) ), - add_file_common_args=True + add_file_common_args=True, + mutually_exclusive=([('version', 'version_by_spec')]) ) if not HAS_LXML_ETREE: module.fail_json(msg=missing_required_lib('lxml'), exception=LXML_ETREE_IMP_ERR) + if module.params['version_by_spec'] and not HAS_SEMANTIC_VERSION: + module.fail_json(msg=missing_required_lib('semantic_version'), exception=SEMANTIC_VERSION_IMP_ERR) + repository_url = module.params["repository_url"] if not repository_url: repository_url = "http://repo1.maven.org/maven2" @@ -501,6 +582,7 @@ def main(): group_id = module.params["group_id"] artifact_id = module.params["artifact_id"] version = module.params["version"] + version_by_spec = module.params["version_by_spec"] classifier = module.params["classifier"] extension = module.params["extension"] headers = module.params['headers'] @@ -514,8 +596,11 @@ def main(): downloader = MavenDownloader(module, repository_url, local, headers) + if not version_by_spec and not version: + version = "latest" + try: - artifact = Artifact(group_id, artifact_id, version, classifier, extension) + artifact = Artifact(group_id, artifact_id, version, version_by_spec, classifier, extension) except ValueError as e: module.fail_json(msg=e.args[0]) @@ -537,13 +622,19 @@ def main(): if os.path.isdir(b_dest): version_part = version - if keep_name and version == 'latest': + if version == 'latest': version_part = downloader.find_latest_version_available(artifact) + elif version_by_spec: + version_part = downloader.find_version_by_spec(artifact) + + filename = "{artifact_id}{version_part}{classifier}.{extension}".format( + artifact_id=artifact_id, + version_part="-{0}".format(version_part) if keep_name else "", + classifier="-{0}".format(classifier) if classifier else "", + extension=extension + ) + dest = posixpath.join(dest, filename) - if classifier: - dest = posixpath.join(dest, "%s-%s-%s.%s" % (artifact_id, version_part, classifier, extension)) - else: - dest = posixpath.join(dest, "%s-%s.%s" % (artifact_id, version_part, extension)) b_dest = to_bytes(dest, errors='surrogate_or_strict') if os.path.lexists(b_dest) and ((not verify_change) or not downloader.is_invalid_md5(dest, downloader.find_uri_for_artifact(artifact))): diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index d619be6367d..da9159854da 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -45,3 +45,4 @@ mccabe == 0.6.1 pylint == 2.3.1 typed-ast == 1.4.0 # 1.4.0 is required to compile on Python 3.8 wrapt == 1.11.1 +semantic_version == 2.6.0 # newer versions are not supported on Python 2.6 diff --git a/test/lib/ansible_test/_data/requirements/units.txt b/test/lib/ansible_test/_data/requirements/units.txt index 307d7c353f7..2d2508945aa 100644 --- a/test/lib/ansible_test/_data/requirements/units.txt +++ b/test/lib/ansible_test/_data/requirements/units.txt @@ -5,3 +5,6 @@ pytest pytest-mock pytest-xdist pyyaml + +# requirement for maven_artifact +semantic_version diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index a88dfd5410d..26c2b7bd59f 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -4889,7 +4889,6 @@ lib/ansible/modules/packaging/language/easy_install.py validate-modules:doc-defa lib/ansible/modules/packaging/language/easy_install.py validate-modules:parameter-type-not-in-doc lib/ansible/modules/packaging/language/easy_install.py validate-modules:doc-missing-type lib/ansible/modules/packaging/language/gem.py validate-modules:parameter-type-not-in-doc -lib/ansible/modules/packaging/language/maven_artifact.py validate-modules:doc-default-does-not-match-spec lib/ansible/modules/packaging/language/maven_artifact.py validate-modules:parameter-type-not-in-doc lib/ansible/modules/packaging/language/maven_artifact.py validate-modules:doc-missing-type lib/ansible/modules/packaging/language/pear.py validate-modules:undocumented-parameter diff --git a/test/units/modules/packaging/language/test_maven_artifact.py b/test/units/modules/packaging/language/test_maven_artifact.py new file mode 100644 index 00000000000..8961de04d18 --- /dev/null +++ b/test/units/modules/packaging/language/test_maven_artifact.py @@ -0,0 +1,70 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import pytest + +from ansible.modules.packaging.language import maven_artifact +from ansible.module_utils import basic + + +pytestmark = pytest.mark.usefixtures('patch_ansible_module') + +maven_metadata_example = b""" + + junit + junit + + 4.13-beta-2 + 4.13-beta-2 + + 3.7 + 3.8 + 3.8.1 + 3.8.2 + 4.0 + 4.1 + 4.2 + 4.3 + 4.3.1 + 4.4 + 4.5 + 4.6 + 4.7 + 4.8 + 4.8.1 + 4.8.2 + 4.9 + 4.10 + 4.11-beta-1 + 4.11 + 4.12-beta-1 + 4.12-beta-2 + 4.12-beta-3 + 4.12 + 4.13-beta-1 + 4.13-beta-2 + + 20190202141051 + + +""" + + +@pytest.mark.parametrize('patch_ansible_module, version_by_spec, version_choosed', [ + (None, "(,3.9]", "3.8.2"), + (None, "3.0", "3.8.2"), + (None, "[3.7]", "3.7"), + (None, "[4.10, 4.12]", "4.12"), + (None, "[4.10, 4.12)", "4.11"), + (None, "[2.0,)", "4.13-beta-2"), +]) +def test_find_version_by_spec(mocker, version_by_spec, version_choosed): + _getContent = mocker.patch('ansible.modules.packaging.language.maven_artifact.MavenDownloader._getContent') + _getContent.return_value = maven_metadata_example + + artifact = maven_artifact.Artifact("junit", "junit", None, version_by_spec, "jar") + mvn_downloader = maven_artifact.MavenDownloader(basic.AnsibleModule, "http://repo1.maven.org/maven2") + + assert mvn_downloader.find_version_by_spec(artifact) == version_choosed