From 9858b1f2f3e9b73d2f09f64c71ebac2058da4629 Mon Sep 17 00:00:00 2001 From: Michael DeHaan Date: Sat, 26 Oct 2013 11:09:30 -0400 Subject: [PATCH] Enable imports to work on a snippet based system, allowing for instance a library of common EC2 functions to be reused between modules. See library/system/service and library/system/ping for initial examples. Can work the old way to just import 'basic', or can import the new way to import multiple pieces of code from module_utils/. --- hacking/test-module | 65 +++++++++---------- .../basic.py} | 17 +---- lib/ansible/runner/__init__.py | 53 ++------------- lib/ansible/utils/template.py | 6 +- library/system/ping | 6 +- library/system/service | 8 ++- 6 files changed, 54 insertions(+), 101 deletions(-) rename lib/ansible/{module_common.py => module_utils/basic.py} (98%) diff --git a/hacking/test-module b/hacking/test-module index ef75eb3d911..e94501fe5b7 100755 --- a/hacking/test-module +++ b/hacking/test-module @@ -77,39 +77,32 @@ def write_argsfile(argstring, json=False): def boilerplate_module(modfile, args): """ simulate what ansible does with new style modules """ - module_fh = open(modfile) - module_data = module_fh.read() - included_boilerplate = module_data.find(module_common.REPLACER) != -1 - module_fh.close() - - if included_boilerplate: - - module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON) - encoded_args = repr(str(args)) - module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args) - encoded_lang = repr(C.DEFAULT_MODULE_LANG) - empty_complex = repr("{}") - module_data = module_data.replace(module_common.REPLACER_LANG, encoded_lang) - module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % C.DEFAULT_SYSLOG_FACILITY) - module_data = module_data.replace(module_common.REPLACER_COMPLEX, empty_complex) - - modfile2_path = os.path.expanduser("~/.ansible_module_generated") - print "* including generated source, if any, saving to: %s" % modfile2_path - print "* this will offset any line numbers in tracebacks/debuggers!" - modfile2 = open(modfile2_path, 'w') - modfile2.write(module_data) - modfile2.close() - modfile = modfile2_path - - return (modfile2_path, included_boilerplate, False) - else: + #module_fh = open(modfile) + #module_data = module_fh.read() + #module_fh.close() + + replacer = module_common.ModuleReplacer() + + #included_boilerplate = module_data.find(module_common.REPLACER) != -1 or module_data.find("import ansible.module_utils") != -1 - old_style_but_json = False - if 'WANT_JSON' in module_data: - old_style_but_json = True + complex_args = {} + inject = {} + (module_data, module_style, shebang) = replacer.modify_module( + modfile, + complex_args, + args, + inject + ) - print "* module boilerplate substitution not requested in module, line numbers will be unaltered" - return (modfile, included_boilerplate, old_style_but_json) + modfile2_path = os.path.expanduser("~/.ansible_module_generated") + print "* including generated source, if any, saving to: %s" % modfile2_path + print "* this may offset any line numbers in tracebacks/debuggers!" + modfile2 = open(modfile2_path, 'w') + modfile2.write(module_data) + modfile2.close() + modfile = modfile2_path + + return (modfile2_path, module_style) def runtest( modfile, argspath): """Test run a module, piping it's output for reporting.""" @@ -151,14 +144,16 @@ def rundebug(debugger, modfile, argspath): def main(): options, args = parse() - (modfile, is_new_style, old_style_but_json) = boilerplate_module(options.module_path, options.module_args) + (modfile, module_style) = boilerplate_module(options.module_path, options.module_args) argspath=None - if not is_new_style: - if old_style_but_json: + if module_style != 'new': + if module_style == 'non_native_want_json': argspath = write_argsfile(options.module_args, json=True) - else: + elif module_style == 'old': argspath = write_argsfile(options.module_args, json=False) + else: + raise Exception("internal error, unexpected module style: %s" % module_style) if options.debugger: rundebug(options.debugger, modfile, argspath) else: diff --git a/lib/ansible/module_common.py b/lib/ansible/module_utils/basic.py similarity index 98% rename from lib/ansible/module_common.py rename to lib/ansible/module_utils/basic.py index 648af8ff282..6f8494b8dde 100644 --- a/lib/ansible/module_common.py +++ b/lib/ansible/module_utils/basic.py @@ -27,18 +27,11 @@ # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -REPLACER = "#<>" -REPLACER_ARGS = "<>" -REPLACER_LANG = "<>" -REPLACER_COMPLEX = "<>" - -MODULE_COMMON = """ - # == BEGIN DYNAMICALLY INSERTED CODE == -MODULE_ARGS = <> -MODULE_LANG = <> -MODULE_COMPLEX_ARGS = <> +MODULE_ARGS = "<>" +MODULE_LANG = "<>" +MODULE_COMPLEX_ARGS = "<>" BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1] BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0] @@ -963,7 +956,3 @@ class AnsibleModule(object): if size >= limit: break return '%.2f %s' % (float(size)/ limit, suffix) - -# == END DYNAMICALLY INSERTED CODE === - -""" diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index 95b751f39e4..17558792f5f 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -40,6 +40,7 @@ from ansible.utils import template from ansible.utils import check_conditional from ansible import errors from ansible import module_common +from ansible.module_common import ModuleReplacer import poller import connection from return_data import ReturnData @@ -51,6 +52,7 @@ try: except ImportError: HAS_ATFORK=False +module_replacer = ModuleReplacer(strip_comments=False) multiprocessing_runner = None OUTPUT_LOCKFILE = tempfile.TemporaryFile() @@ -825,60 +827,19 @@ class Runner(object): def _copy_module(self, conn, tmp, module_name, module_args, inject, complex_args=None): ''' transfer a module over SFTP, does not run it ''' - # FIXME if complex args is none, set to {} - - if module_name.startswith("/"): - raise errors.AnsibleFileNotFound("%s is not a module" % module_name) - # Search module path(s) for named module. in_path = utils.plugins.module_finder.find_plugin(module_name) if in_path is None: raise errors.AnsibleFileNotFound("module %s not found in %s" % (module_name, utils.plugins.module_finder.print_paths())) - out_path = os.path.join(tmp, module_name) - module_data = "" - module_style = 'old' - - with open(in_path) as f: - module_data = f.read() - if module_common.REPLACER in module_data: - module_style = 'new' - if 'WANT_JSON' in module_data: - module_style = 'non_native_want_json' - - complex_args_json = utils.jsonify(complex_args) - # We force conversion of module_args to str because module_common calls shlex.split, - # a standard library function that incorrectly handles Unicode input before Python 2.7.3. - encoded_args = repr(module_args.encode('utf-8')) - encoded_lang = repr(C.DEFAULT_MODULE_LANG) - encoded_complex = repr(complex_args_json) - - module_data = module_data.replace(module_common.REPLACER, module_common.MODULE_COMMON) - module_data = module_data.replace(module_common.REPLACER_ARGS, encoded_args) - module_data = module_data.replace(module_common.REPLACER_LANG, encoded_lang) - module_data = module_data.replace(module_common.REPLACER_COMPLEX, encoded_complex) - - if module_style == 'new': - facility = C.DEFAULT_SYSLOG_FACILITY - if 'ansible_syslog_facility' in inject: - facility = inject['ansible_syslog_facility'] - module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility) - - lines = module_data.split("\n") - shebang = None - if lines[0].startswith("#!"): - shebang = lines[0].strip() - args = shlex.split(str(shebang[2:])) - interpreter = args[0] - interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter) - - if interpreter_config in inject: - lines[0] = shebang = "#!%s %s" % (inject[interpreter_config], " ".join(args[1:])) - module_data = "\n".join(lines) + # insert shared code and arguments into the module + (module_data, module_style, shebang) = module_replacer.modify_module( + in_path, complex_args, module_args, inject + ) + # ship the module self._transfer_str(conn, tmp, module_name, module_data) - return (out_path, module_style, shebang) # ***************************************************** diff --git a/lib/ansible/utils/template.py b/lib/ansible/utils/template.py index 003e6bc18e4..be1a4fd9f17 100644 --- a/lib/ansible/utils/template.py +++ b/lib/ansible/utils/template.py @@ -462,7 +462,7 @@ def template_from_file(basedir, path, vars): res = res + '\n' return template(basedir, res, vars) -def template_from_string(basedir, data, vars, fail_on_undefined=False, lookups=True): +def template_from_string(basedir, data, vars, fail_on_undefined=False, lookups=True, filters=True): ''' run a string through the (Jinja2) templating engine ''' def my_lookup(*args, **kwargs): @@ -472,7 +472,9 @@ def template_from_string(basedir, data, vars, fail_on_undefined=False, lookups=T if type(data) == str: data = unicode(data, 'utf-8') environment = jinja2.Environment(trim_blocks=True, undefined=StrictUndefined, extensions=_get_extensions()) - environment.filters.update(_get_filters()) + + if filters: + environment.filters.update(_get_filters()) if '_original_file' in vars: basedir = os.path.dirname(vars['_original_file']) diff --git a/library/system/ping b/library/system/ping index 69507111aec..e0ff5f2bbad 100644 --- a/library/system/ping +++ b/library/system/ping @@ -48,7 +48,9 @@ def main(): result['ping'] = module.params['data'] module.exit_json(**result) -# this is magic, see lib/ansible/module_common.py -#<> +### boilerplate: import common module snippets here +from ansible.module_utils.basic import * + +### invoke the module main() diff --git a/library/system/service b/library/system/service index fb9364c24f5..63f681f6ace 100644 --- a/library/system/service +++ b/library/system/service @@ -1205,7 +1205,11 @@ def main(): module.exit_json(**result) -# this is magic, see lib/ansible/module_common.py -#<> +### boilerplate: import common module snippets here +from ansible.module_utils.basic import * +### invoke the module main() + + +