most of galaxy is working, install still needs work

normalized return codes in bin
refactored galaxy classes a bit, ignoring 'compatct' role spec
pull/10885/head
Brian Coca 9 years ago
parent cec4d0889b
commit ccc9a33b56

@ -25,6 +25,10 @@ import os
from ansible.errors import AnsibleError
from ansible.utils.display import Display
# default_readme_template
# default_meta_template
class Galaxy(object):
''' Keeps global galaxy info '''
@ -36,13 +40,31 @@ class Galaxy(object):
self.display = display
self.options = options
self.roles_path = os.path.expanduser(self.options.roles_path)
self.roles_path = getattr(self.options, 'roles_path', None)
if self.roles_path:
self.roles_path = os.path.expanduser(self.roles_path)
self.roles = {}
# load data path for resource usage
this_dir, this_filename = os.path.split(__file__)
self.DATA_PATH = os.path.join(this_dir, "data")
#TODO: move to getter for lazy loading
self.default_readme = self._str_from_data_file('readme')
self.default_meta = self._str_from_data_file('metadata_template.j2')
def add_role(self, role):
self.roles[role.name] = role
def remove_role(self, role_name):
del self.roles[role_name]
def _str_from_data_file(self, filename):
myfile = os.path.join(self.DATA_PATH, filename)
try:
return open(myfile).read()
except Exception as e:
raise AnsibleError("Could not open %s: %s" % (filename, str(e)))

@ -38,10 +38,12 @@ class GalaxyAPI(object):
try:
urlparse(api_server, scheme='https')
except:
raise AnsibleError("Invalid server API url passed: %s" % self.galaxy.api_server)
raise AnsibleError("Invalid server API url passed: %s" % api_server)
server_version = self.get_server_api_version('%s/api/' % (api_server))
if not server_version:
raise AnsibleError("Could not retrieve server API version: %s" % api_server)
server_version = self.get_server_api_version(api_server)
self.galaxy.display.vvvvv("Server version: %s" % server_version)
if server_version in self.SUPPORTED_VERSIONS:
self.baseurl = '%s/api/%s' % (api_server, server_version)
self.version = server_version # for future use
@ -54,22 +56,21 @@ class GalaxyAPI(object):
Fetches the Galaxy API current version to ensure
the API server is up and reachable.
"""
#TODO: fix galaxy server which returns current_version path (/api/v1) vs actual version (v1)
# also should set baseurl using supported_versions which has path
return 'v1'
try:
self.galaxy.display.vvvvv("Querying server version: %s" % api_server)
data = json.load(urlopen(api_server))
if not data.get("current_version", None):
return None
else:
return data
except:
return data.get("current_version", 'v1')
except Exception as e:
# TODO: report error
return None
def lookup_role_by_name(self, role_name, notify=True):
"""
Find a role by name
"""
role_name = urlquote(role_name)
try:
@ -82,6 +83,7 @@ class GalaxyAPI(object):
raise AnsibleError("- invalid role name (%s). Specify role as format: username.rolename" % role_name)
url = '%s/roles/?owner__username=%s&name=%s' % (self.baseurl, user_name, role_name)
self.galaxy.display.vvvv("- %s" % (url))
try:
data = json.load(urlopen(url))
if len(data["results"]) != 0:

@ -39,30 +39,20 @@ class GalaxyRole(object):
ROLE_DIRS = ('defaults','files','handlers','meta','tasks','templates','vars')
def __init__(self, galaxy, role_name, role_version=None, role_url=None):
def __init__(self, galaxy, name, src=None, version=None, scm=None):
self._metadata = None
self._install_info = None
self.options = galaxy.options
self.display = galaxy.display
self.name = role_name
self.meta_data = None
self.install_info = None
self.path = (os.path.join(galaxy.roles_path, self.name))
# TODO: possibly parse version and url from role_name
self.version = role_version
self.url = role_url
if self.url is None:
self._spec_parse()
if C.GALAXY_SCMS:
self.scms = self.SUPPORTED_SCMS.intersection(set(C.GALAXY_SCMS))
else:
self.scms = self.SUPPORTED_SCMS
if not self.scms:
self.display.warning("No valid SCMs configured for Galaxy.")
self.name = name
self.version = version
self.src = src
self.scm = scm
self.path = (os.path.join(galaxy.roles_path, self.name))
def fetch_from_scm_archive(self):
@ -112,59 +102,44 @@ class GalaxyRole(object):
return temp_file.name
def get_metadata(self):
@property
def metadata(self):
"""
Returns role metadata
"""
if self.meta_data is None:
self._read_metadata
return self.meta_data
if self._metadata is None:
meta_path = os.path.join(self.path, self.META_MAIN)
if os.path.isfile(meta_path):
try:
f = open(meta_path, 'r')
self._metadata = yaml.safe_load(f)
except:
self.display.vvvvv("Unable to load metadata for %s" % self.name)
return False
finally:
f.close()
def _read_metadata(self):
"""
Reads the metadata as YAML, if the file 'meta/main.yml' exists
"""
meta_path = os.path.join(self.path, self.META_MAIN)
if os.path.isfile(meta_path):
try:
f = open(meta_path, 'r')
self.meta_data = yaml.safe_load(f)
except:
self.display.vvvvv("Unable to load metadata for %s" % self.name)
return False
finally:
f.close()
return self._metadata
def get_galaxy_install_info(self):
@property
def install_info(self):
"""
Returns role install info
"""
if self.install_info is None:
self._read_galaxy_isntall_info()
return self.install_info
def _read_galaxy_install_info(self):
"""
Returns the YAML data contained in 'meta/.galaxy_install_info',
if it exists.
"""
if self._install_info is None:
info_path = os.path.join(self.path, self.META_INSTALL)
if os.path.isfile(info_path):
try:
f = open(info_path, 'r')
self.install_info = yaml.safe_load(f)
except:
self.display.vvvvv("Unable to load Galaxy install info for %s" % self.name)
return False
finally:
f.close()
info_path = os.path.join(self.path, self.META_INSTALL)
if os.path.isfile(info_path):
try:
f = open(info_path, 'r')
self._install_info = yaml.safe_load(f)
except:
self.display.vvvvv("Unable to load Galaxy install info for %s" % self.name)
return False
finally:
f.close()
return self._install_info
def _write_galaxy_install_info(self):
"""
@ -180,7 +155,7 @@ class GalaxyRole(object):
info_path = os.path.join(self.path, self.META_INSTALL)
try:
f = open(info_path, 'w+')
self.install_info = yaml.safe_dump(info, f)
self._install_info = yaml.safe_dump(info, f)
except:
return False
finally:
@ -194,7 +169,7 @@ class GalaxyRole(object):
sanity check to make sure there's a meta/main.yml file at this
path so the user doesn't blow away random directories
"""
if self.read_metadata():
if self.metadata:
try:
rmtree(self.path)
return True
@ -210,8 +185,8 @@ class GalaxyRole(object):
"""
# first grab the file and save it to a temp location
if self.url:
archive_url = self.url
if self.src:
archive_url = self.src
else:
archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % (role_data["github_user"], role_data["github_repo"], target)
self.display.display("- downloading role from %s" % archive_url)
@ -256,7 +231,7 @@ class GalaxyRole(object):
return False
else:
try:
self.meta_data = yaml.safe_load(role_tar_file.extractfile(meta_file))
self._metadata = yaml.safe_load(role_tar_file.extractfile(meta_file))
except:
self.display.error("this role does not appear to have a valid meta/main.yml file.")
return False
@ -306,7 +281,8 @@ class GalaxyRole(object):
self.display.display("- %s was installed successfully" % self.name)
return True
def get_spec(self):
@property
def spec(self):
"""
Returns role spec info
{
@ -316,40 +292,4 @@ class GalaxyRole(object):
'name': 'repo'
}
"""
if self.scm is None and self.url is None:
self._read_galaxy_isntall_info()
return dict(scm=self.scm, src=self.url, version=self.version, role_name=self.name)
def _spec_parse(self):
''' creates separated parts of role spec '''
default_role_versions = dict(git='master', hg='tip')
if not self.url and '://' in self.name:
role_spec = self.name.strip()
if role_spec == "" or role_spec.startswith("#"):
return
tokens = [s.strip() for s in role_spec.split(',')]
# assume https://github.com URLs are git+https:// URLs and not tarballs unless they end in '.zip'
if 'github.com/' in tokens[0] and not tokens[0].startswith("git+") and not tokens[0].endswith('.tar.gz'):
tokens[0] = 'git+' + tokens[0]
if '+' in tokens[0]:
(self.scm, self.url) = tokens[0].split('+')
else:
self.scm = None
self.url = tokens[0]
if len(tokens) >= 2:
self.version = tokens[1]
if len(tokens) == 3:
self.name = tokens[2]
else:
self.name = self._repo_url_to_role_name(tokens[0])
if self.scm and not self.version:
self.version = default_role_versions.get(scm, '')
return dict(scm=self.scm, src=self.src, version=self.version, name=self.name)

@ -180,9 +180,13 @@ if __name__ == '__main__':
cli = AdHocCli(sys.argv, display=display)
cli.parse()
sys.exit(cli.run())
except AnsibleError as e:
except AnsibleOptionsError as e:
cli.parser.print_help()
display.display(str(e), stderr=True, color='red')
sys.exit(1)
except AnsibleError as e:
display.display(str(e), stderr=True, color='red')
sys.exit(2)
except KeyboardInterrupt:
display.error("interrupted")
sys.exit(1)
sys.exit(4)

@ -46,6 +46,7 @@ from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.galaxy import Galaxy
from ansible.galaxy.api import GalaxyAPI
from ansible.galaxy.role import GalaxyRole
from ansible.playbook.role.requirement import RoleRequirement
from ansible.utils.display import Display
from ansible.utils.cli import CLI
@ -71,6 +72,10 @@ class GalaxyCLI(CLI):
self.parser = parser
self.set_action()
# verbose
self.parser.add_option('-v','--verbose', dest='verbosity', default=0, action="count",
help="verbose mode (-vvv for more, -vvvv to enable connection debugging)")
# options specific to actions
if self.action == "info":
self.parser.set_usage("usage: %prog info [options] role_name[,version]")
@ -108,8 +113,7 @@ class GalaxyCLI(CLI):
'ansible.cfg file (/etc/ansible/roles if not configured)')
if self.action in ("info","init","install"):
self.parser.add_option(
'-s', '--server', dest='api_server', default="galaxy.ansible.com",
self.parser.add_option( '-s', '--server', dest='api_server', default="https://galaxy.ansible.com",
help='The API server destination')
if self.action in ("init","install"):
@ -119,20 +123,16 @@ class GalaxyCLI(CLI):
# get options, args and galaxy object
self.options, self.args =self.parser.parse_args()
self.display.verbosity = self.options.verbosity
self.galaxy = Galaxy(self.options, self.display)
if len(self.args) != 1:
raise AnsibleOptionsError("Missing arguments")
return True
def run(self):
#self.display.verbosity = self.options.verbosity
api_server = self.get_opt("api_server", "galaxy.ansible.com")
# if not offline, get connect to galaxy api
if self.action == 'init' and not self.options.offline:
if self.action in ("info","install") or (self.action == 'init' and not self.options.offline):
api_server = self.options.api_server
self.api = GalaxyAPI(self.galaxy, api_server)
if not self.api:
raise AnsibleError("The API server (%s) is not responding, please try again later." % api_server)
@ -157,13 +157,10 @@ class GalaxyCLI(CLI):
Exits with the specified return code unless the
option --ignore-errors was specified
"""
if not self.get_opt("ignore_errors", False):
self.display.error('- you can use --ignore-errors to skip failed tasks/roles.')
self.display.error('- you can use --ignore-errors to skip failed roles and finish processing the list.')
return rc
def execute_init(self):
"""
Executes the init action, which creates the skeleton framework
@ -192,10 +189,10 @@ class GalaxyCLI(CLI):
os.makedirs(role_path)
readme_path = os.path.join(role_path, "README.md")
f = open(readme_path, "wb")
f.write(default_readme_template)
f.write(self.galaxy.default_readme)
f.close
for dir in self.ROLE_DIRS:
for dir in GalaxyRole.ROLE_DIRS:
dir_path = os.path.join(init_path, role_name, dir)
main_yml_path = os.path.join(dir_path, 'main.yml')
# create the directory if it doesn't exist already
@ -232,7 +229,7 @@ class GalaxyCLI(CLI):
platforms = platform_groups,
categories = categories,
)
rendered_meta = Environment().from_string(default_meta_template).render(inject)
rendered_meta = Environment().from_string(self.galaxy.default_meta).render(inject)
f = open(main_yml_path, 'w')
f.write(rendered_meta)
f.close()
@ -242,7 +239,7 @@ class GalaxyCLI(CLI):
f = open(main_yml_path, 'w')
f.write('---\n# %s file for %s\n' % (dir,role_name))
f.close()
print "- %s was created successfully" % role_name
self.display.display("- %s was created successfully" % role_name)
def execute_info(self):
"""
@ -260,43 +257,48 @@ class GalaxyCLI(CLI):
for role in self.args:
role_info = {}
gr = GalaxyRole(self.galaxy, role)
#self.galaxy.add_role(gr)
install_info = get_galaxy_install_info(role, options)
install_info = gr.install_info
if install_info:
if 'version' in install_info:
install_info['intalled_version'] = install_info['version']
del install_info['version']
role_info.update(install_info)
remote_data = self.api.lookup_role_by_name(role, False)
remote_data = False
if self.api:
remote_data = self.api.lookup_role_by_name(role, False)
if remote_data:
role_info.update(remote_data)
metadata = get_metadata(role, options)
if metadata:
role_info.update(metadata)
if gr.metadata:
role_info.update(gr.metadata)
role_spec = ansible.utils.role_spec_parse(role)
req = RoleRequirement()
__, __, role_spec= req.parse({'role': role})
if role_spec:
role_info.update(role_spec)
if role_info:
print "- %s:" % (role)
self.display.display("- %s:" % (role))
for k in sorted(role_info.keys()):
if k in SKIP_INFO_KEYS:
if k in self.SKIP_INFO_KEYS:
continue
if isinstance(role_info[k], dict):
print "\t%s: " % (k)
self.display.display("\t%s: " % (k))
for key in sorted(role_info[k].keys()):
if key in SKIP_INFO_KEYS:
if key in self.SKIP_INFO_KEYS:
continue
print "\t\t%s: %s" % (key, role_info[k][key])
self.display.display("\t\t%s: %s" % (key, role_info[k][key]))
else:
print "\t%s: %s" % (k, role_info[k])
self.display.display("\t%s: %s" % (k, role_info[k]))
else:
print "- the role %s was not found" % role
self.display.display("- the role %s was not found" % role)
def execute_install(self):
"""
@ -321,100 +323,111 @@ class GalaxyCLI(CLI):
roles_path = self.get_opt("roles_path")
roles_done = []
roles_left = []
role_name = self.args.pop(0).strip()
gr = GalaxyRole(self.galaxy, role_name)
if role_file:
f = open(role_file, 'r')
if role_file.endswith('.yaml') or role_file.endswith('.yml'):
roles_left = map(ansible.utils.role_yaml_parse, yaml.safe_load(f))
else:
# roles listed in a file, one per line
roles_left = map(gr.get_spec, f.readlines())
for rname in f.readlines():
roles_left.append(GalaxyRole(self.galaxy, rname))
f.close()
else:
# roles were specified directly, so we'll just go out grab them
# (and their dependencies, unless the user doesn't want us to).
roles_left = map(gr.get_spec, self.args)
for rname in self.args:
roles_left.append(GalaxyRole(self.galaxy, rname))
while len(roles_left) > 0:
# query the galaxy API for the role data
role_data = None
role = roles_left.pop(0)
role_src = role.get("src")
role_scm = role.get("scm")
role_path = role.get("path")
role_src = role.src
role_scm = role.scm
role_path = role.path
if role_path:
options.roles_path = role_path
self.options.roles_path = role_path
else:
options.roles_path = roles_path
self.options.roles_path = roles_path
if os.path.isfile(role_src):
tmp_file = None
if role_src and os.path.isfile(role_src):
# installing a local tar.gz
tmp_file = role_src
else:
if role_scm:
# create tar file from scm url
tmp_file = scm_archive_role(role_scm, role_src, role.get("version"), role.get("name"))
elif '://' in role_src:
# just download a URL - version will probably be in the URL
tmp_file = fetch_role(role_src, None, None, options)
else:
role_data = self.api.lookup_role_by_name(role_src)
if not role_data:
print "- sorry, %s was not found on %s." % (role_src, self.options.api_server)
exit_without_ignore(options)
continue
role_versions = self.api.fetch_role_related('versions', role_data['id'])
if "version" not in role or role['version'] == '':
# convert the version names to LooseVersion objects
# and sort them to get the latest version. If there
# are no versions in the list, we'll grab the head
# of the master branch
if len(role_versions) > 0:
loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions]
loose_versions.sort()
role["version"] = str(loose_versions[-1])
else:
role["version"] = 'master'
elif role['version'] != 'master':
if role_versions and role["version"] not in [a.get('name', None) for a in role_versions]:
print 'role is %s' % role
print "- the specified version (%s) was not found in the list of available versions (%s)." % (role['version'], role_versions)
exit_without_ignore(options)
tmp_file = scm_archive_role(role_scm, role_src, role.version, role.name)
if role_src:
if '://' in role_src:
# just download a URL - version will probably be in the URL
tmp_file = gr.fetch()
else:
role_data = self.api.lookup_role_by_name(role_src)
if not role_data:
self.display.warning("- sorry, %s was not found on %s." % (role_src, self.options.api_server))
self.exit_without_ignore()
continue
# download the role. if --no-deps was specified, we stop here,
# otherwise we recursively grab roles and all of their deps.
tmp_file = fetch_role(role_src, role["version"], role_data, options)
role_versions = self.api.fetch_role_related('versions', role_data['id'])
if not role.version:
# convert the version names to LooseVersion objects
# and sort them to get the latest version. If there
# are no versions in the list, we'll grab the head
# of the master branch
if len(role_versions) > 0:
loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions]
loose_versions.sort()
role["version"] = str(loose_versions[-1])
else:
role["version"] = 'master'
elif role['version'] != 'master':
if role_versions and role.version not in [a.get('name', None) for a in role_versions]:
self.display.warning('role is %s' % role)
self.display.warning("- the specified version (%s) was not found in the list of available versions (%s)." % (role.version, role_versions))
self.exit_without_ignore()
continue
# download the role. if --no-deps was specified, we stop here,
# otherwise we recursively grab roles and all of their deps.
tmp_file = gr.fetch(role_data)
installed = False
if tmp_file:
installed = install_role(role.get("name"), role.get("version"), tmp_file, options)
installed = install_role(role.name, role.version, tmp_file, options)
# we're done with the temp file, clean it up
if tmp_file != role_src:
os.unlink(tmp_file)
# install dependencies, if we want them
if not no_deps and installed:
if not role_data:
role_data = gr.get_metadata(role.get("name"), options)
role_dependencies = role_data['dependencies']
else:
role_dependencies = role_data['summary_fields']['dependencies'] # api_fetch_role_related(api_server, 'dependencies', role_data['id'])
for dep in role_dependencies:
if isinstance(dep, basestring):
dep = ansible.utils.role_spec_parse(dep)
else:
dep = ansible.utils.role_yaml_parse(dep)
if not get_role_metadata(dep["name"], options):
if dep not in roles_left:
print '- adding dependency: %s' % dep["name"]
roles_left.append(dep)
else:
print '- dependency %s already pending installation.' % dep["name"]
else:
print '- dependency %s is already installed, skipping.' % dep["name"]
# this should use new roledepenencies code
#if not no_deps and installed:
# if not role_data:
# role_data = gr.get_metadata(role.get("name"), options)
# role_dependencies = role_data['dependencies']
# else:
# role_dependencies = role_data['summary_fields']['dependencies'] # api_fetch_role_related(api_server, 'dependencies', role_data['id'])
# for dep in role_dependencies:
# if isinstance(dep, basestring):
# dep = ansible.utils.role_spec_parse(dep)
# else:
# dep = ansible.utils.role_yaml_parse(dep)
# if not get_role_metadata(dep["name"], options):
# if dep not in roles_left:
# print '- adding dependency: %s' % dep["name"]
# roles_left.append(dep)
# else:
# print '- dependency %s already pending installation.' % dep["name"]
# else:
# print '- dependency %s is already installed, skipping.' % dep["name"]
if not tmp_file or not installed:
print "- %s was NOT installed successfully." % role.get("name")
exit_without_ignore(options)
self.display.warning("- %s was NOT installed successfully." % role.name)
self.exit_without_ignore()
return 0
def execute_remove(self):
@ -426,14 +439,16 @@ class GalaxyCLI(CLI):
if len(self.args) == 0:
raise AnsibleOptionsError('- you must specify at least one role to remove.')
for role in self.args:
if get_role_metadata(role, options):
if remove_role(role, options):
self.display.display('- successfully removed %s' % role)
for role_name in self.args:
role = GalaxyRole(self.galaxy, role_name)
try:
if role.remove():
self.display.display('- successfully removed %s' % role_name)
else:
self.display.display("- failed to remove role: %s" % role)
else:
self.display.display('- %s is not installed, skipping.' % role)
self.display.display('- %s is not installed, skipping.' % role_name)
except Exception as e:
raise AnsibleError("Failed to remove role %s: %s" % (role_name, str(e)))
return 0
def execute_list(self):
@ -449,20 +464,18 @@ class GalaxyCLI(CLI):
if len(self.args) == 1:
# show only the request role, if it exists
role_name = self.args[0]
gr = GalaxyRole(self.galaxy, role_name)
metadata = gr.get_metadata()
if metadata:
install_info = gr.get_galaxy_install_info()
gr = GalaxyRole(self.galaxy, self.name)
if gr.metadata:
install_info = gr.install_info
version = None
if install_info:
version = install_info.get("version", None)
if not version:
version = "(unknown version)"
# show some more info about single roles here
self.display.display("- %s, %s" % (role_name, version))
self.display.display("- %s, %s" % (self.name, version))
else:
self.display.display("- the role %s was not found" % role_name)
self.display.display("- the role %s was not found" % self.name)
else:
# show all valid roles in the roles_path directory
roles_path = self.get_opt('roles_path')
@ -473,8 +486,8 @@ class GalaxyCLI(CLI):
raise AnsibleOptionsError("- %s exists, but it is not a directory. Please specify a valid path with --roles-path" % roles_path)
path_files = os.listdir(roles_path)
for path_file in path_files:
if get_role_metadata(path_file, options):
install_info = get_galaxy_install_info(path_file, options)
if gr.metadata:
install_info = gr.metadata
version = None
if install_info:
version = install_info.get("version", None)
@ -502,4 +515,4 @@ if __name__ == '__main__':
sys.exit(2)
except KeyboardInterrupt:
display.error("interrupted")
sys.exit(3)
sys.exit(4)

@ -201,9 +201,14 @@ if __name__ == "__main__":
cli = PlaybookCLI(sys.argv, display=display)
cli.parse()
sys.exit(cli.run())
except AnsibleError as e:
except AnsibleOptionsError as e:
cli.parser.print_help()
display.display(str(e), stderr=True, color='red')
sys.exit(1)
except AnsibleError as e:
display.display(str(e), stderr=True, color='red')
sys.exit(2)
except KeyboardInterrupt:
display.error("interrupted")
sys.exit(1)
sys.exit(4)

@ -33,7 +33,7 @@ import os
import sys
import traceback
from ansible.errors import AnsibleError
from ansible.errors import AnsibleError, AnsibleOptionsError
from ansible.parsing.vault import VaultEditor
from ansible.utils.cli import CLI
from ansible.utils.display import Display
@ -142,9 +142,13 @@ if __name__ == "__main__":
cli = VaultCli(sys.argv, display=display)
cli.parse()
sys.exit(cli.run())
except AnsibleError as e:
except AnsibleOptionsError as e:
cli.parser.print_help()
display.display(str(e), stderr=True, color='red')
sys.exit(1)
except AnsibleError as e:
display.display(str(e), stderr=True, color='red')
sys.exit(2)
except KeyboardInterrupt:
display.error("interrupted")
sys.exit(1)
sys.exit(4)

Loading…
Cancel
Save