From f73329401b2e68eba22944074933a8d8f88bbeb9 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Sat, 3 Oct 2015 10:29:28 -0400 Subject: [PATCH 1/2] galaxy updates better error reporting on fetching errors use scm if it exists over src unified functions in requirements simplified logic added verbose to tests cleanup code refs, unused options and dead code moved get_opt to base class fixes #11920 fixes #12612 fixes #10454 --- lib/ansible/cli/__init__.py | 13 ++ lib/ansible/cli/galaxy.py | 103 +++------ lib/ansible/galaxy/role.py | 253 +++++++++++------------ lib/ansible/playbook/role/__init__.py | 5 - lib/ansible/playbook/role/requirement.py | 240 ++++++++++++--------- lib/ansible/utils/module_docs.py | 2 + test/integration/Makefile | 4 +- 7 files changed, 296 insertions(+), 324 deletions(-) diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index a3d128f9117..bf5e33e6be8 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -512,3 +512,16 @@ class CLI(object): return vault_pass + def get_opt(self, k, defval=""): + """ + Returns an option from an Optparse values instance. + """ + try: + data = getattr(self.options, k) + except: + return defval + if k == "roles_path": + if os.pathsep in data: + data = data.split(os.pathsep)[0] + return data + diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index 021698d8469..c564b46b798 100644 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -29,8 +29,6 @@ from distutils.version import LooseVersion from jinja2 import Environment import ansible.constants as C -import ansible.utils -import ansible.galaxy from ansible.cli import CLI from ansible.errors import AnsibleError, AnsibleOptionsError from ansible.galaxy import Galaxy @@ -126,19 +124,6 @@ class GalaxyCLI(CLI): self.execute() - def get_opt(self, k, defval=""): - """ - Returns an option from an Optparse values instance. - """ - try: - data = getattr(self.options, k) - except: - return defval - if k == "roles_path": - if os.pathsep in data: - data = data.split(os.pathsep)[0] - return data - def exit_without_ignore(self, rc=1): """ Exits with the specified return code unless the @@ -147,40 +132,6 @@ class GalaxyCLI(CLI): if not self.get_opt("ignore_errors", False): raise AnsibleError('- you can use --ignore-errors to skip failed roles and finish processing the list.') - def parse_requirements_files(self, role): - if 'role' in role: - # Old style: {role: "galaxy.role,version,name", other_vars: "here" } - role_info = role_spec_parse(role['role']) - if isinstance(role_info, dict): - # Warning: Slight change in behaviour here. name may be being - # overloaded. Previously, name was only a parameter to the role. - # Now it is both a parameter to the role and the name that - # ansible-galaxy will install under on the local system. - if 'name' in role and 'name' in role_info: - del role_info['name'] - role.update(role_info) - else: - # New style: { src: 'galaxy.role,version,name', other_vars: "here" } - if 'github.com' in role["src"] and 'http' in role["src"] and '+' not in role["src"] and not role["src"].endswith('.tar.gz'): - role["src"] = "git+" + role["src"] - - if '+' in role["src"]: - (scm, src) = role["src"].split('+') - role["scm"] = scm - role["src"] = src - - if 'name' not in role: - role["name"] = GalaxyRole.url_to_spec(role["src"]) - - if 'version' not in role: - role['version'] = '' - - if 'scm' not in role: - role['scm'] = None - - return role - - def _display_role_info(self, role_info): text = "\nRole: %s \n" % role_info['name'] @@ -298,9 +249,8 @@ class GalaxyCLI(CLI): data = '' for role in self.args: - role_info = {} + role_info = {'role_path': roles_path} gr = GalaxyRole(self.galaxy, role) - #self.galaxy.add_role(gr) install_info = gr.install_info if install_info: @@ -351,54 +301,45 @@ class GalaxyCLI(CLI): no_deps = self.get_opt("no_deps", False) force = self.get_opt('force', False) - roles_path = self.get_opt("roles_path") - roles_done = [] roles_left = [] if role_file: - self.display.debug('Getting roles from %s' % role_file) try: - self.display.debug('Processing role file: %s' % role_file) f = open(role_file, 'r') if role_file.endswith('.yaml') or role_file.endswith('.yml'): - try: - rolesparsed = map(self.parse_requirements_files, yaml.safe_load(f)) - except Exception as e: - raise AnsibleError("%s does not seem like a valid yaml file: %s" % (role_file, str(e))) - roles_left = [GalaxyRole(self.galaxy, **r) for r in rolesparsed] + for role in yaml.safe_load(f.read()): + self.display.debug('found role %s in yaml file' % str(role)) + if 'name' not in role: + if 'src' in role: + role['name'] = RoleRequirement.repo_url_to_role_name(role['src']) + else: + raise AnsibleError("Must specify name or src for role") + roles_left.append(GalaxyRole(self.galaxy, **role)) else: + self.display.deprecated("going forward only the yaml format will be supported") # roles listed in a file, one per line - self.display.deprecated("Non yaml files for role requirements") - for rname in f.readlines(): - if rname.startswith("#") or rname.strip() == '': - continue - roles_left.append(GalaxyRole(self.galaxy, rname.strip())) + for rline in f.readlines(): + self.display.debug('found role %s in text file' % str(rline)) + roles_left.append(GalaxyRole(self.galaxy, **RoleRequirement.role_spec_parse(rline))) f.close() - except (IOError,OSError) as e: - raise AnsibleError("Unable to read requirements file (%s): %s" % (role_file, str(e))) + except (IOError, OSError) as e: + self.display.error('Unable to open %s: %s' % (role_file, str(e))) else: # roles were specified directly, so we'll just go out grab them # (and their dependencies, unless the user doesn't want us to). for rname in self.args: roles_left.append(GalaxyRole(self.galaxy, rname.strip())) - while len(roles_left) > 0: + for role in roles_left: + self.display.debug('Installing role %s ' % role.name) # query the galaxy API for the role data role_data = None role = roles_left.pop(0) - role_path = role.path if role.install_info is not None and not force: self.display.display('- %s is already installed, skipping.' % role.name) continue - if role_path: - self.options.roles_path = role_path - else: - self.options.roles_path = roles_path - - self.display.debug('Installing role %s from %s' % (role.name, self.options.roles_path)) - tmp_file = None installed = False if role.src and os.path.isfile(role.src): @@ -407,7 +348,7 @@ class GalaxyCLI(CLI): else: if role.scm: # create tar file from scm url - tmp_file = GalaxyRole.scm_archive_role(role.scm, role.src, role.version, role.name) + tmp_file = RoleRequirement.scm_archive_role(role.scm, role.src, role.version, role.name) if role.src: if '://' not in role.src: role_data = self.api.lookup_role_by_name(role.src) @@ -438,11 +379,14 @@ class GalaxyCLI(CLI): # download the role. if --no-deps was specified, we stop here, # otherwise we recursively grab roles and all of their deps. tmp_file = role.fetch(role_data) + if tmp_file: + self.display.debug('using %s' % tmp_file) installed = role.install(tmp_file) - # we're done with the temp file, clean it up + # we're done with the temp file, clean it up if we created it if tmp_file != role.src: os.unlink(tmp_file) + # install dependencies, if we want them if not no_deps and installed: role_dependencies = role.metadata.get('dependencies', []) @@ -460,9 +404,10 @@ class GalaxyCLI(CLI): else: self.display.display('- dependency %s is already installed, skipping.' % dep_name) - if not tmp_file or not installed: + if not installed: self.display.warning("- %s was NOT installed successfully." % role.name) self.exit_without_ignore() + return 0 def execute_remove(self): diff --git a/lib/ansible/galaxy/role.py b/lib/ansible/galaxy/role.py index ea6debb8133..f4f3776c8a9 100644 --- a/lib/ansible/galaxy/role.py +++ b/lib/ansible/galaxy/role.py @@ -21,15 +21,16 @@ import datetime import os -import subprocess import tarfile import tempfile import yaml +from distutils.version import LooseVersion from shutil import rmtree -from ansible import constants as C from ansible.errors import AnsibleError from ansible.module_utils.urls import open_url +from ansible.playbook.role.requirement import RoleRequirement +from ansible.galaxy.api import GalaxyAPI try: from __main__ import display @@ -51,6 +52,7 @@ class GalaxyRole(object): self._install_info = None self.options = galaxy.options + self.galaxy = galaxy self.name = name self.version = version @@ -135,9 +137,9 @@ class GalaxyRole(object): def remove(self): """ - Removes the specified role from the roles path. There is a - sanity check to make sure there's a meta/main.yml file at this - path so the user doesn't blow away random directories + Removes the specified role from the roles path. + There is a sanity check to make sure there's a meta/main.yml file at this + path so the user doesn't blow away random directories. """ if self.metadata: try: @@ -159,6 +161,7 @@ class GalaxyRole(object): archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % (role_data["github_user"], role_data["github_repo"], self.version) else: archive_url = self.src + display.display("- downloading role from %s" % archive_url) try: @@ -170,87 +173,125 @@ class GalaxyRole(object): data = url_file.read() temp_file.close() return temp_file.name - except: - # TODO: better urllib2 error handling for error - # messages that are more exact - display.error("failed to download the file.") + except Exception as e: + display.error("failed to download the file: %s" % str(e)) return False - def install(self, role_filename): + def install(self): # the file is a tar, so open it that way and extract it # to the specified (or default) roles directory - if not tarfile.is_tarfile(role_filename): - display.error("the file downloaded was not a tar.gz") - return False - else: - if role_filename.endswith('.gz'): - role_tar_file = tarfile.open(role_filename, "r:gz") + if self.scm: + # create tar file from scm url + tmp_file = RoleRequirement.scm_archive_role(**self.spec) + elif self.src: + if os.path.isfile(self.src): + # installing a local tar.gz + tmp_file = self.src + elif '://' in self.src: + role_data = self.src + tmp_file = self.fetch(role_data) else: - role_tar_file = tarfile.open(role_filename, "r") - # verify the role's meta file - meta_file = None - members = role_tar_file.getmembers() - # next find the metadata file - for member in members: - if self.META_MAIN in member.name: - meta_file = member - break - if not meta_file: - display.error("this role does not appear to have a meta/main.yml file.") - return False + api = GalaxyAPI(self.galaxy, self.options.api_server) + role_data = api.lookup_role_by_name(self.src) + if not role_data: + raise AnsibleError("- sorry, %s was not found on %s." % (self.src, self.options.api_server)) + + role_versions = api.fetch_role_related('versions', role_data['id']) + if not self.version: + # convert the version names to LooseVersion objects + # and sort them to get the latest version. If there + # are no versions in the list, we'll grab the head + # of the master branch + if len(role_versions) > 0: + loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions] + loose_versions.sort() + self.version = str(loose_versions[-1]) + else: + self.version = 'master' + elif self.version != 'master': + if role_versions and self.version not in [a.get('name', None) for a in role_versions]: + raise AnsibleError("- the specified version (%s) of %s was not found in the list of available versions (%s)." % (self.version, self.name, role_versions)) + + tmp_file = self.fetch(role_data) + + else: + raise AnsibleError("No valid role data found") + + + if tmp_file: + + display.display("installing from %s" % tmp_file) + + if not tarfile.is_tarfile(tmp_file): + raise AnsibleError("the file downloaded was not a tar.gz") else: + if tmp_file.endswith('.gz'): + role_tar_file = tarfile.open(tmp_file, "r:gz") + else: + role_tar_file = tarfile.open(tmp_file, "r") + # verify the role's meta file + meta_file = None + members = role_tar_file.getmembers() + # next find the metadata file + for member in members: + if self.META_MAIN in member.name: + meta_file = member + break + if not meta_file: + raise AnsibleError("this role does not appear to have a meta/main.yml file.") + else: + try: + self._metadata = yaml.safe_load(role_tar_file.extractfile(meta_file)) + except: + raise AnsibleError("this role does not appear to have a valid meta/main.yml file.") + + # we strip off the top-level directory for all of the files contained within + # the tar file here, since the default is 'github_repo-target', and change it + # to the specified role's name + display.display("- extracting %s to %s" % (self.name, self.path)) try: - self._metadata = yaml.safe_load(role_tar_file.extractfile(meta_file)) - except: - display.error("this role does not appear to have a valid meta/main.yml file.") - return False - - # we strip off the top-level directory for all of the files contained within - # the tar file here, since the default is 'github_repo-target', and change it - # to the specified role's name - display.display("- extracting %s to %s" % (self.name, self.path)) - try: - if os.path.exists(self.path): - if not os.path.isdir(self.path): - display.error("the specified roles path exists and is not a directory.") - return False - elif not getattr(self.options, "force", False): - display.error("the specified role %s appears to already exist. Use --force to replace it." % self.name) - return False + if os.path.exists(self.path): + if not os.path.isdir(self.path): + raise AnsibleError("the specified roles path exists and is not a directory.") + elif not getattr(self.options, "force", False): + raise AnsibleError("the specified role %s appears to already exist. Use --force to replace it." % self.name) + else: + # using --force, remove the old path + if not self.remove(): + raise AnsibleError("%s doesn't appear to contain a role.\n please remove this directory manually if you really want to put the role here." % self.path) else: - # using --force, remove the old path - if not self.remove(): - display.error("%s doesn't appear to contain a role." % self.path) - display.error(" please remove this directory manually if you really want to put the role here.") - return False - else: - os.makedirs(self.path) + os.makedirs(self.path) + + # now we do the actual extraction to the path + for member in members: + # we only extract files, and remove any relative path + # bits that might be in the file for security purposes + # and drop the leading directory, as mentioned above + if member.isreg() or member.issym(): + parts = member.name.split(os.sep)[1:] + final_parts = [] + for part in parts: + if part != '..' and '~' not in part and '$' not in part: + final_parts.append(part) + member.name = os.path.join(*final_parts) + role_tar_file.extract(member, self.path) + + # write out the install info file for later use + self._write_galaxy_install_info() + except OSError as e: + raise AnsibleError("Could not update files in %s: %s" % (self.path, str(e))) + + # return the parsed yaml metadata + display.display("- %s was installed successfully" % self.name) + try: + os.unlink(tmp_file) + except (OSError,IOError) as e: + display.warning("Unable to remove tmp file (%s): %s" % (tmp_file, str(e))) + return True - # now we do the actual extraction to the path - for member in members: - # we only extract files, and remove any relative path - # bits that might be in the file for security purposes - # and drop the leading directory, as mentioned above - if member.isreg() or member.issym(): - parts = member.name.split(os.sep)[1:] - final_parts = [] - for part in parts: - if part != '..' and '~' not in part and '$' not in part: - final_parts.append(part) - member.name = os.path.join(*final_parts) - role_tar_file.extract(member, self.path) - - # write out the install info file for later use - self._write_galaxy_install_info() - except OSError as e: - display.error("Could not update files in %s: %s" % (self.path, str(e))) - return False - - # return the parsed yaml metadata - display.display("- %s was installed successfully" % self.name) - return True + return False @property def spec(self): @@ -266,65 +307,3 @@ class GalaxyRole(object): return dict(scm=self.scm, src=self.src, version=self.version, name=self.name) - @staticmethod - def url_to_spec(roleurl): - # gets the role name out of a repo like - # http://git.example.com/repos/repo.git" => "repo" - - if '://' not in roleurl and '@' not in roleurl: - return roleurl - trailing_path = roleurl.split('/')[-1] - if trailing_path.endswith('.git'): - trailing_path = trailing_path[:-4] - if trailing_path.endswith('.tar.gz'): - trailing_path = trailing_path[:-7] - if ',' in trailing_path: - trailing_path = trailing_path.split(',')[0] - return trailing_path - - @staticmethod - def scm_archive_role(scm, role_url, role_version, role_name): - if scm not in ['hg', 'git']: - display.display("- scm %s is not currently supported" % scm) - return False - tempdir = tempfile.mkdtemp() - clone_cmd = [scm, 'clone', role_url, role_name] - with open('/dev/null', 'w') as devnull: - try: - display.display("- executing: %s" % " ".join(clone_cmd)) - popen = subprocess.Popen(clone_cmd, cwd=tempdir, stdout=devnull, stderr=devnull) - except: - raise AnsibleError("error executing: %s" % " ".join(clone_cmd)) - rc = popen.wait() - if rc != 0: - display.display("- command %s failed" % ' '.join(clone_cmd)) - display.display(" in directory %s" % tempdir) - return False - - temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.tar') - if scm == 'hg': - archive_cmd = ['hg', 'archive', '--prefix', "%s/" % role_name] - if role_version: - archive_cmd.extend(['-r', role_version]) - archive_cmd.append(temp_file.name) - if scm == 'git': - archive_cmd = ['git', 'archive', '--prefix=%s/' % role_name, '--output=%s' % temp_file.name] - if role_version: - archive_cmd.append(role_version) - else: - archive_cmd.append('HEAD') - - with open('/dev/null', 'w') as devnull: - display.display("- executing: %s" % " ".join(archive_cmd)) - popen = subprocess.Popen(archive_cmd, cwd=os.path.join(tempdir, role_name), - stderr=devnull, stdout=devnull) - rc = popen.wait() - if rc != 0: - display.display("- command %s failed" % ' '.join(archive_cmd)) - display.display(" in directory %s" % tempdir) - return False - - rmtree(tempdir, ignore_errors=True) - - return temp_file.name - diff --git a/lib/ansible/playbook/role/__init__.py b/lib/ansible/playbook/role/__init__.py index d708d103788..14f8e6e5b27 100644 --- a/lib/ansible/playbook/role/__init__.py +++ b/lib/ansible/playbook/role/__init__.py @@ -21,19 +21,14 @@ __metaclass__ = type from six import iteritems, string_types -import inspect import os -from hashlib import sha1 - from ansible.errors import AnsibleError, AnsibleParserError -from ansible.parsing import DataLoader from ansible.playbook.attribute import FieldAttribute from ansible.playbook.base import Base from ansible.playbook.become import Become from ansible.playbook.conditional import Conditional from ansible.playbook.helpers import load_list_of_blocks -from ansible.playbook.role.include import RoleInclude from ansible.playbook.role.metadata import RoleMetadata from ansible.playbook.taggable import Taggable from ansible.plugins import get_all_plugin_loaders diff --git a/lib/ansible/playbook/role/requirement.py b/lib/ansible/playbook/role/requirement.py index d7ae9a626ae..a0c3f6ddac0 100644 --- a/lib/ansible/playbook/role/requirement.py +++ b/lib/ansible/playbook/role/requirement.py @@ -19,11 +19,14 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from six import iteritems, string_types +from six import string_types import os +import shutil +import subprocess +import tempfile -from ansible.errors import AnsibleError, AnsibleParserError +from ansible.errors import AnsibleError from ansible.playbook.role.definition import RoleDefinition __all__ = ['RoleRequirement'] @@ -73,7 +76,7 @@ class RoleRequirement(RoleDefinition): def _preprocess_role_spec(self, ds): if 'role' in ds: # Old style: {role: "galaxy.role,version,name", other_vars: "here" } - role_info = role_spec_parse(ds['role']) + role_info = RoleRequirement.role_spec_parse(ds['role']) if isinstance(role_info, dict): # Warning: Slight change in behaviour here. name may be being # overloaded. Previously, name was only a parameter to the role. @@ -96,7 +99,7 @@ class RoleRequirement(RoleDefinition): ds["role"] = ds["name"] del ds["name"] else: - ds["role"] = repo_url_to_role_name(ds["src"]) + ds["role"] = RoleRequirement.repo_url_to_role_name(ds["src"]) # set some values to a default value, if none were specified ds.setdefault('version', '') @@ -104,102 +107,137 @@ class RoleRequirement(RoleDefinition): return ds -def repo_url_to_role_name(repo_url): - # gets the role name out of a repo like - # http://git.example.com/repos/repo.git" => "repo" - - if '://' not in repo_url and '@' not in repo_url: - return repo_url - trailing_path = repo_url.split('/')[-1] - if trailing_path.endswith('.git'): - trailing_path = trailing_path[:-4] - if trailing_path.endswith('.tar.gz'): - trailing_path = trailing_path[:-7] - if ',' in trailing_path: - trailing_path = trailing_path.split(',')[0] - return trailing_path - -def role_spec_parse(role_spec): - # takes a repo and a version like - # git+http://git.example.com/repos/repo.git,v1.0 - # and returns a list of properties such as: - # { - # 'scm': 'git', - # 'src': 'http://git.example.com/repos/repo.git', - # 'version': 'v1.0', - # 'name': 'repo' - # } - - default_role_versions = dict(git='master', hg='tip') - - role_spec = role_spec.strip() - role_version = '' - if role_spec == "" or role_spec.startswith("#"): - return (None, None, None, None) - - tokens = [s.strip() for s in role_spec.split(',')] - - # assume https://github.com URLs are git+https:// URLs and not - # tarballs unless they end in '.zip' - if 'github.com/' in tokens[0] and not tokens[0].startswith("git+") and not tokens[0].endswith('.tar.gz'): - tokens[0] = 'git+' + tokens[0] - - if '+' in tokens[0]: - (scm, role_url) = tokens[0].split('+') - else: - scm = None - role_url = tokens[0] - - if len(tokens) >= 2: - role_version = tokens[1] - - if len(tokens) == 3: - role_name = tokens[2] - else: - role_name = repo_url_to_role_name(tokens[0]) - - if scm and not role_version: - role_version = default_role_versions.get(scm, '') - - return dict(scm=scm, src=role_url, version=role_version, role_name=role_name) - -# FIXME: all of these methods need to be cleaned up/reorganized below this -def get_opt(options, k, defval=""): - """ - Returns an option from an Optparse values instance. - """ - try: - data = getattr(options, k) - except: - return defval - if k == "roles_path": - if os.pathsep in data: - data = data.split(os.pathsep)[0] - return data - -def get_role_path(role_name, options): - """ - Returns the role path based on the roles_path option - and the role name. - """ - roles_path = get_opt(options,'roles_path') - roles_path = os.path.join(roles_path, role_name) - roles_path = os.path.expanduser(roles_path) - return roles_path + @staticmethod + def repo_url_to_role_name(repo_url): + # gets the role name out of a repo like + # http://git.example.com/repos/repo.git" => "repo" + + if '://' not in repo_url and '@' not in repo_url: + return repo_url + trailing_path = repo_url.split('/')[-1] + if trailing_path.endswith('.git'): + trailing_path = trailing_path[:-4] + if trailing_path.endswith('.tar.gz'): + trailing_path = trailing_path[:-7] + if ',' in trailing_path: + trailing_path = trailing_path.split(',')[0] + return trailing_path + + @staticmethod + def role_spec_parse(role_spec): + # takes a repo and a version like + # git+http://git.example.com/repos/repo.git,v1.0 + # and returns a list of properties such as: + # { + # 'scm': 'git', + # 'src': 'http://git.example.com/repos/repo.git', + # 'version': 'v1.0', + # 'name': 'repo' + # } + + default_role_versions = dict(git='master', hg='tip') + + role_spec = role_spec.strip() + role_version = '' + if role_spec == "" or role_spec.startswith("#"): + return (None, None, None, None) + + tokens = [s.strip() for s in role_spec.split(',')] + + # assume https://github.com URLs are git+https:// URLs and not + # tarballs unless they end in '.zip' + if 'github.com/' in tokens[0] and not tokens[0].startswith("git+") and not tokens[0].endswith('.tar.gz'): + tokens[0] = 'git+' + tokens[0] + + if '+' in tokens[0]: + (scm, role_url) = tokens[0].split('+') + else: + scm = None + role_url = tokens[0] -def get_role_metadata(role_name, options): - """ - Returns the metadata as YAML, if the file 'meta/main.yml' - exists in the specified role_path - """ - role_path = os.path.join(get_role_path(role_name, options), 'meta/main.yml') - try: - if os.path.isfile(role_path): - f = open(role_path, 'r') - meta_data = yaml.safe_load(f) - f.close() - return meta_data + if len(tokens) >= 2: + role_version = tokens[1] + + if len(tokens) == 3: + role_name = tokens[2] else: - return None - except: - return None + role_name = RoleRequirement.repo_url_to_role_name(tokens[0]) + + if scm and not role_version: + role_version = default_role_versions.get(scm, '') + + return dict(scm=scm, src=role_url, version=role_version, name=role_name) + + @staticmethod + def role_yaml_parse(role): + + if 'role' in role: + # Old style: {role: "galaxy.role,version,name", other_vars: "here" } + role_info = RoleRequirement.role_spec_parse(role['role']) + if isinstance(role_info, dict): + # Warning: Slight change in behaviour here. name may be being + # overloaded. Previously, name was only a parameter to the role. + # Now it is both a parameter to the role and the name that + # ansible-galaxy will install under on the local system. + if 'name' in role and 'name' in role_info: + del role_info['name'] + role.update(role_info) + else: + # New style: { src: 'galaxy.role,version,name', other_vars: "here" } + if 'github.com' in role["src"] and 'http' in role["src"] and '+' not in role["src"] and not role["src"].endswith('.tar.gz'): + role["src"] = "git+" + role["src"] + + if '+' in role["src"]: + (scm, src) = role["src"].split('+') + role["scm"] = scm + role["src"] = src + + if 'name' not in role: + role["name"] = RoleRequirement.repo_url_to_role_name(role["src"]) + + if 'version' not in role: + role['version'] = '' + + if 'scm' not in role: + role['scm'] = None + + return role + + @staticmethod + def scm_archive_role(src, scm='git', name=None, version='HEAD'): + if scm not in ['hg', 'git']: + raise AnsibleError("- scm %s is not currently supported" % scm) + tempdir = tempfile.mkdtemp() + clone_cmd = [scm, 'clone', src, name] + with open('/dev/null', 'w') as devnull: + try: + popen = subprocess.Popen(clone_cmd, cwd=tempdir, stdout=devnull, stderr=devnull) + except: + raise AnsibleError("error executing: %s" % " ".join(clone_cmd)) + rc = popen.wait() + if rc != 0: + raise AnsibleError ("- command %s failed in directory %s" % (' '.join(clone_cmd), tempdir)) + + temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.tar') + if scm == 'hg': + archive_cmd = ['hg', 'archive', '--prefix', "%s/" % name] + if version: + archive_cmd.extend(['-r', version]) + archive_cmd.append(temp_file.name) + if scm == 'git': + archive_cmd = ['git', 'archive', '--prefix=%s/' % name, '--output=%s' % temp_file.name] + if version: + archive_cmd.append(version) + else: + archive_cmd.append('HEAD') + + with open('/dev/null', 'w') as devnull: + popen = subprocess.Popen(archive_cmd, cwd=os.path.join(tempdir, name), + stderr=devnull, stdout=devnull) + rc = popen.wait() + if rc != 0: + raise AnsibleError("- command %s failed in directory %s" % (' '.join(archive_cmd), tempdir)) + + shutil.rmtree(tempdir, ignore_errors=True) + return temp_file.name + diff --git a/lib/ansible/utils/module_docs.py b/lib/ansible/utils/module_docs.py index ebb1f527791..0e25e565ec4 100755 --- a/lib/ansible/utils/module_docs.py +++ b/lib/ansible/utils/module_docs.py @@ -63,6 +63,7 @@ def get_docstring(filename, verbose=False): theid = t.id except AttributeError as e: # skip errors can happen when trying to use the normal code + display.warning("Failed to assign id for %t on %s, skipping" % (t, filename)) continue if 'DOCUMENTATION' in theid: @@ -119,6 +120,7 @@ def get_docstring(filename, verbose=False): except: display.error("unable to parse %s" % filename) if verbose == True: + display.display("unable to parse %s" % filename) raise return doc, plainexamples, returndocs diff --git a/test/integration/Makefile b/test/integration/Makefile index fe46dd9d2e6..b044121f5e0 100644 --- a/test/integration/Makefile +++ b/test/integration/Makefile @@ -172,7 +172,7 @@ test_galaxy: test_galaxy_spec test_galaxy_yaml test_galaxy_spec: mytmpdir=$(MYTMPDIR) ; \ - ansible-galaxy install -r galaxy_rolesfile -p $$mytmpdir/roles ; \ + ansible-galaxy install -r galaxy_rolesfile -p $$mytmpdir/roles -vvvv ; \ cp galaxy_playbook.yml $$mytmpdir ; \ ansible-playbook -i $(INVENTORY) $$mytmpdir/galaxy_playbook.yml -v $(TEST_FLAGS) ; \ RC=$$? ; \ @@ -181,7 +181,7 @@ test_galaxy_spec: test_galaxy_yaml: mytmpdir=$(MYTMPDIR) ; \ - ansible-galaxy install -r galaxy_roles.yml -p $$mytmpdir/roles ; \ + ansible-galaxy install -r galaxy_roles.yml -p $$mytmpdir/roles -vvvv; \ cp galaxy_playbook.yml $$mytmpdir ; \ ansible-playbook -i $(INVENTORY) $$mytmpdir/galaxy_playbook.yml -v $(TEST_FLAGS) ; \ RC=$$? ; \ From 6f88f79de9d24e8e7328ae485808936c79352cdb Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 5 Oct 2015 13:03:34 -0400 Subject: [PATCH 2/2] removed dupe install code, now trap errors forom role.install --- lib/ansible/cli/galaxy.py | 84 ++++++++++----------------------------- 1 file changed, 22 insertions(+), 62 deletions(-) diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index c564b46b798..3422815735c 100644 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -340,69 +340,29 @@ class GalaxyCLI(CLI): self.display.display('- %s is already installed, skipping.' % role.name) continue - tmp_file = None - installed = False - if role.src and os.path.isfile(role.src): - # installing a local tar.gz - tmp_file = role.src - else: - if role.scm: - # create tar file from scm url - tmp_file = RoleRequirement.scm_archive_role(role.scm, role.src, role.version, role.name) - if role.src: - if '://' not in role.src: - role_data = self.api.lookup_role_by_name(role.src) - if not role_data: - self.display.warning("- sorry, %s was not found on %s." % (role.src, self.options.api_server)) - self.exit_without_ignore() - continue - - role_versions = self.api.fetch_role_related('versions', role_data['id']) - if not role.version: - # convert the version names to LooseVersion objects - # and sort them to get the latest version. If there - # are no versions in the list, we'll grab the head - # of the master branch - if len(role_versions) > 0: - loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions] - loose_versions.sort() - role.version = str(loose_versions[-1]) - else: - role.version = 'master' - elif role.version != 'master': - if role_versions and role.version not in [a.get('name', None) for a in role_versions]: - self.display.warning('role is %s' % role) - self.display.warning("- the specified version (%s) was not found in the list of available versions (%s)." % (role.version, role_versions)) - self.exit_without_ignore() - continue - - # download the role. if --no-deps was specified, we stop here, - # otherwise we recursively grab roles and all of their deps. - tmp_file = role.fetch(role_data) - - if tmp_file: - self.display.debug('using %s' % tmp_file) - installed = role.install(tmp_file) - # we're done with the temp file, clean it up if we created it - if tmp_file != role.src: - os.unlink(tmp_file) - - # install dependencies, if we want them - if not no_deps and installed: - role_dependencies = role.metadata.get('dependencies', []) - for dep in role_dependencies: - self.display.debug('Installing dep %s' % dep) - dep_req = RoleRequirement() - __, dep_name, __ = dep_req.parse(dep) - dep_role = GalaxyRole(self.galaxy, name=dep_name) - if dep_role.install_info is None or force: - if dep_role not in roles_left: - self.display.display('- adding dependency: %s' % dep_name) - roles_left.append(GalaxyRole(self.galaxy, name=dep_name)) - else: - self.display.display('- dependency %s already pending installation.' % dep_name) + try: + installed = role.install() + except AnsibleError as e: + self.display.warning("- %s was NOT installed successfully: %s " % (role.name, str(e))) + self.exit_without_ignore() + continue + + # install dependencies, if we want them + if not no_deps and installed: + role_dependencies = role.metadata.get('dependencies', []) + for dep in role_dependencies: + self.display.debug('Installing dep %s' % dep) + dep_req = RoleRequirement() + __, dep_name, __ = dep_req.parse(dep) + dep_role = GalaxyRole(self.galaxy, name=dep_name) + if dep_role.install_info is None or force: + if dep_role not in roles_left: + self.display.display('- adding dependency: %s' % dep_name) + roles_left.append(GalaxyRole(self.galaxy, name=dep_name)) else: - self.display.display('- dependency %s is already installed, skipping.' % dep_name) + self.display.display('- dependency %s already pending installation.' % dep_name) + else: + self.display.display('- dependency %s is already installed, skipping.' % dep_name) if not installed: self.display.warning("- %s was NOT installed successfully." % role.name)