Move module arg passing from the environment to stdin (from the wrapper to the module)

lib/ansible/executor/module_common.py

@@ -97,9 +97,35 @@ if sys.version_info < (3,):
 else:
     unicode = str
 
+try:
+    # Python-2.6+
+    from io import BytesIO as IOStream
+except ImportError:
+    # Python < 2.6
+    from StringIO import StringIO as IOStream
+
 ZIPDATA = """%(zipdata)s"""
 
-def debug(command, zipped_mod):
+def invoke_module(module_path, json_params):
+    pythonpath = os.environ.get('PYTHONPATH')
+    if pythonpath:
+        os.environ['PYTHONPATH'] = ':'.join((module_path, pythonpath))
+    else:
+        os.environ['PYTHONPATH'] = module_path
+
+    p = subprocess.Popen(['%(interpreter)s', '-m', 'ansible.module_exec.%(ansible_module)s.__main__'], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate(json_params)
+
+    if not isinstance(stderr, (bytes, unicode)):
+        stderr = stderr.read()
+    if not isinstance(stdout, (bytes, unicode)):
+        stdout = stdout.read()
+    sys.stderr.write(stderr)
+    sys.stdout.write(stdout)
+
+    return p.returncode
+
+def debug(command, zipped_mod, json_params):
     # The code here normally doesn't run.  It's only used for debugging on the
     # remote machine. Run with ANSIBLE_KEEP_REMOTE_FILES=1 envvar and -vvv
     # to save the module file remotely.  Login to the remote machine and use
@@ -107,7 +133,7 @@ def debug(command, zipped_mod):
     # files.  Edit the source files to instrument the code or experiment with
     # different values.  Then use /path/to/module execute to run the extracted
     # files you've edited instead of the actual zipped module.
-    #
+
     # Okay to use __file__ here because we're running from a kept file
     basedir = os.path.dirname(__file__)
     if command == 'explode':
@@ -120,6 +146,7 @@ def debug(command, zipped_mod):
         for filename in z.namelist():
             if filename.startswith('/'):
                 raise Exception('Something wrong with this module zip file: should not contain absolute paths')
+
             dest_filename = os.path.join(basedir, filename)
             if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename):
                 os.makedirs(dest_filename)
@@ -130,26 +157,17 @@ def debug(command, zipped_mod):
                 f = open(dest_filename, 'w')
                 f.write(z.read(filename))
                 f.close()
+
         print('Module expanded into:')
         print('%%s' %% os.path.join(basedir, 'ansible'))
+        exitcode = 0
+
     elif command == 'execute':
         # Execute the exploded code instead of executing the module from the
         # embedded ZIPDATA.  This allows people to easily run their modified
         # code on the remote machine to see how changes will affect it.
-        pythonpath = os.environ.get('PYTHONPATH')
+        exitcode = invoke_module(basedir, json_params)
-        if pythonpath:
+
-            os.environ['PYTHONPATH'] = ':'.join((basedir, pythonpath))
-        else:
-            os.environ['PYTHONPATH'] = basedir
-        p = subprocess.Popen(['%(interpreter)s', '-m', 'ansible.module_exec.%(ansible_module)s.__main__'], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (stdout, stderr) = p.communicate()
-        if not isinstance(stderr, (bytes, unicode)):
-            stderr = stderr.read()
-        if not isinstance(stdout, (bytes, unicode)):
-            stdout = stdout.read()
-        sys.stderr.write(stderr)
-        sys.stdout.write(stdout)
-        sys.exit(p.returncode)
     elif command == 'excommunicate':
         # This attempts to run the module in-process (by importing a main
         # function and then calling it).  It is not the way ansible generally
@@ -159,41 +177,41 @@ def debug(command, zipped_mod):
         # when using this that are only artifacts of how we're invoking here,
         # not actual bugs (as they don't affect the real way that we invoke
         # ansible modules)
+        sys.stdin = IOStream(json_params)
         sys.path.insert(0, basedir)
         from ansible.module_exec.%(ansible_module)s.__main__ import main
         main()
+        print('WARNING: Module returned to wrapper instead of exiting')
+        sys.exit(1)
+    else:
+        print('WARNING: Unknown debug command.  Doing nothing.')
+        exitcode = 0
 
-os.environ['ANSIBLE_MODULE_ARGS'] = %(args)s
+    return exitcode
-os.environ['ANSIBLE_MODULE_CONSTANTS'] = %(constants)s
+
+if __name__ == '__main__':
+    ZIPLOADER_PARAMS = %(params)s
 
-try:
-    temp_fd, temp_path = tempfile.mkstemp(prefix='ansible_')
-    os.write(temp_fd, base64.b64decode(ZIPDATA))
-    if len(sys.argv) == 2:
-        debug(sys.argv[1], temp_path)
-    else:
-        pythonpath = os.environ.get('PYTHONPATH')
-        if pythonpath:
-            os.environ['PYTHONPATH'] = ':'.join((temp_path, pythonpath))
-        else:
-            os.environ['PYTHONPATH'] = temp_path
-        p = subprocess.Popen(['%(interpreter)s', '-m', 'ansible.module_exec.%(ansible_module)s.__main__'], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (stdout, stderr) = p.communicate()
-        if not isinstance(stderr, (bytes, unicode)):
-            stderr = stderr.read()
-        if not isinstance(stdout, (bytes, unicode)):
-            stdout = stdout.read()
-        sys.stderr.write(stderr)
-        sys.stdout.write(stdout)
-        sys.exit(p.returncode)
-
-finally:
     try:
+        temp_fd, temp_path = tempfile.mkstemp(prefix='ansible_')
+        os.write(temp_fd, base64.b64decode(ZIPDATA))
         os.close(temp_fd)
-        os.remove(temp_path)
+        if len(sys.argv) == 2:
-    except NameError:
+            exitcode = debug(sys.argv[1], temp_path, ZIPLOADER_PARAMS)
-        # mkstemp failed
+        else:
-        pass
+            exitcode = invoke_module(temp_path, ZIPLOADER_PARAMS)
+    finally:
+        try:
+            try:
+                os.close(temp_fd)
+            except OSError:
+                # Already closed
+                pass
+            os.remove(temp_path)
+        except NameError:
+            # mkstemp failed
+            pass
+    sys.exit(exitcode)
 '''
 
 class ModuleDepFinder(ast.NodeVisitor):
@@ -336,19 +354,21 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
     if module_style in ('old', 'non_native_want_json'):
         return module_data, module_style, shebang
 
-    module_args_json = to_bytes(json.dumps(module_args))
-
     output = BytesIO()
     snippet_names = set()
 
     if module_substyle == 'python':
         # ziploader for new-style python classes
-        python_repred_args = to_bytes(repr(module_args_json))
         constants = dict(
                 SELINUX_SPECIAL_FS=C.DEFAULT_SELINUX_SPECIAL_FS,
                 SYSLOG_FACILITY=_get_facility(task_vars),
                 )
-        python_repred_constants = to_bytes(repr(json.dumps(constants)), errors='strict')
+        params = dict(ANSIBLE_MODULE_ARGS=module_args,
+                ANSIBLE_MODULE_CONSTANTS=constants,
+                )
+        #python_repred_args = to_bytes(repr(module_args_json))
+        #python_repred_constants = to_bytes(repr(json.dumps(constants)), errors='strict')
+        python_repred_params = to_bytes(repr(json.dumps(params)), errors='strict')
 
         try:
             compression_method = getattr(zipfile, module_compression)
@@ -411,8 +431,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
         output.write(to_bytes(STRIPPED_ZIPLOADER_TEMPLATE % dict(
             zipdata=zipdata,
             ansible_module=module_name,
-            args=python_repred_args,
+            #args=python_repred_args,
-            constants=python_repred_constants,
+            #constants=python_repred_constants,
+            params=python_repred_params,
             shebang=shebang,
             interpreter=interpreter,
             coding=ENCODING_STRING,
@@ -437,6 +458,8 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
                 continue
             output.write(line + b'\n')
         module_data = output.getvalue()
+
+        module_args_json = to_bytes(json.dumps(module_args))
         module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
 
         # Sanity check from 1.x days.  This is currently useless as we only
@@ -447,11 +470,14 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta
             raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path)
 
     elif module_substyle == 'jsonargs':
+        module_args_json = to_bytes(json.dumps(module_args))
+
         # these strings could be included in a third-party module but
         # officially they were included in the 'basic' snippet for new-style
         # python modules (which has been replaced with something else in
         # ziploader) If we remove them from jsonargs-style module replacer
         # then we can remove them everywhere.
+        python_repred_args = to_bytes(repr(module_args_json))
         module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
         module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
         module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))

lib/ansible/module_utils/basic.py

@@ -223,23 +223,6 @@ from ansible import __version__
 # Backwards compat. New code should just import and use __version__
 ANSIBLE_VERSION = __version__
 
-try:
-    # MODULE_COMPLEX_ARGS is an old name kept for backwards compat
-    MODULE_COMPLEX_ARGS = os.environ.pop('ANSIBLE_MODULE_ARGS')
-except KeyError:
-    # This file might be used for its utility functions.  So don't fail if
-    # running outside of a module environment (will fail in _load_params()
-    # instead)
-    MODULE_COMPLEX_ARGS = None
-
-try:
-    # ARGS are for parameters given in the playbook.  Constants are for things
-    # that ansible needs to configure controller side but are passed to all
-    # modules.
-    MODULE_CONSTANTS = os.environ.pop('ANSIBLE_MODULE_CONSTANTS')
-except KeyError:
-    MODULE_CONSTANTS = None
-
 FILE_COMMON_ARGUMENTS=dict(
     src = dict(),
     mode = dict(type='raw'),
@@ -560,7 +543,6 @@ class AnsibleModule(object):
                 if k not in self.argument_spec:
                     self.argument_spec[k] = v
 
-        self._load_constants()
         self._load_params()
         self._set_fallbacks()
 
@@ -1452,32 +1434,47 @@ class AnsibleModule(object):
                     continue
 
     def _load_params(self):
-        ''' read the input and set the params attribute'''
+        ''' read the input and set the params attribute.  Sets the constants as well.'''
-        if MODULE_COMPLEX_ARGS is None:
+        buffer = sys.stdin.read()
+        try:
+            params = json.loads(buffer)
+        except ValueError:
             # This helper used too early for fail_json to work.
-            print('{"msg": "Error: ANSIBLE_MODULE_ARGS not found in environment.  Unable to figure out what parameters were passed", "failed": true}')
+            print('{"msg": "Error: Module unable to decode valid JSON on stdin.  Unable to figure out what parameters were passed", "failed": true}')
             sys.exit(1)
 
-        params = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS))
+        try:
-        if params is None:
+            self.params = params['ANSIBLE_MODULE_ARGS']
-            params = dict()
+            self.constants = params['ANSIBLE_MODULE_CONSTANTS']
-        self.params = params
+        except KeyError:
-
-    def _load_constants(self):
-        ''' read the input and set the constants attribute'''
-        if MODULE_CONSTANTS is None:
             # This helper used too early for fail_json to work.
-            print('{"msg": "Error: ANSIBLE_MODULE_CONSTANTS not found in environment.  Unable to figure out what constants were passed", "failed": true}')
+            print('{"msg": "Error: Module unable to locate ANSIBLE_MODULE_ARGS and ANSIBLE_MODULE_CONSTANTS in json data from stdin.  Unable to figure out what parameters were passed", "failed": true}')
             sys.exit(1)
 
-        # Make constants into "native string"
+#        import select
-        if sys.version_info >= (3,):
+#        buffer = ''
-            constants = json_dict_bytes_to_unicode(json.loads(MODULE_CONSTANTS))
+#        while True:
-        else:
+#            input_list = select.select([sys.stdin], [],  [], 5.0)[0]
-            constants = json_dict_unicode_to_bytes(json.loads(MODULE_CONSTANTS))
+#            if sys.stdin not in input_list:
-        if constants is None:
+#                # This helper used too early for fail_json to work.
-            constants = dict()
+#                print('{"msg": "Error: Module unable to read arguments from stdin.  Unable to figure out what parameters were passed", "failed": true}')
-        self.constants = constants
+#                sys.exit(1)
+#            buffer += sys.stdin.read()
+#            if json.loads(buffer):
+#
+#        for line in sys.stdin:
+#            if line is None:
+#                print('s')
+#        data = sys.stdin.read()
+#        if MODULE_COMPLEX_ARGS is None:
+#            # This helper used too early for fail_json to work.
+#            print('{"msg": "Error: ANSIBLE_MODULE_ARGS not found in environment.  Unable to figure out what parameters were passed", "failed": true}')
+#            sys.exit(1)
+#
+#        params = json_dict_unicode_to_bytes(json.loads(data))
+#        if params is None:
+#            params = dict()
+#        self.params = params
 
     def _log_to_syslog(self, msg):
         if HAS_SYSLOG: