add JSON junk filter to async_wrapper (#5107)

lib/ansible/modules/utilities/logic/async_wrapper.py

@@ -70,6 +70,50 @@ def daemonize_self():
     os.dup2(dev_null.fileno(), sys.stdout.fileno())
     os.dup2(dev_null.fileno(), sys.stderr.fileno())
 
+# NB: this function copied from module_utils/json_utils.py. Ensure any changes are propagated there.
+# FUTURE: AnsibleModule-ify this module so it's Ansiballz-compatible and can use the module_utils copy of this function.
+def _filter_non_json_lines(data):
+    '''
+    Used to filter unrelated output around module JSON output, like messages from
+    tcagetattr, or where dropbear spews MOTD on every single command (which is nuts).
+
+    Filters leading lines before first line-starting occurrence of '{' or '[', and filter all
+    trailing lines after matching close character (working from the bottom of output).
+    '''
+    warnings = []
+
+    # Filter initial junk
+    lines = data.splitlines()
+
+    for start, line in enumerate(lines):
+        line = line.strip()
+        if line.startswith(u'{'):
+            endchar = u'}'
+            break
+        elif line.startswith(u'['):
+            endchar = u']'
+            break
+    else:
+        raise ValueError('No start of json char found')
+
+    # Filter trailing junk
+    lines = lines[start:]
+
+    for reverse_end_offset, line in enumerate(reversed(lines)):
+        if line.strip().endswith(endchar):
+            break
+    else:
+        raise ValueError('No end of json char found')
+
+    if reverse_end_offset > 0:
+        # Trailing junk is uncommon and can point to things the user might
+        # want to change.  So print a warning if we find any
+        trailing_junk = lines[len(lines) - reverse_end_offset:]
+        warnings.append('Module invocation had junk after the JSON data: %s' % '\n'.join(trailing_junk))
+
+    lines = lines[:(len(lines) - reverse_end_offset)]
+
+    return ('\n'.join(lines), warnings)
 
 def _run_module(wrapped_cmd, jid, job_path):
 
@@ -82,6 +126,8 @@ def _run_module(wrapped_cmd, jid, job_path):
     result = {}
 
     outdata = ''
+    filtered_outdata = ''
+    stderr = ''
     try:
         cmd = shlex.split(wrapped_cmd)
         script = subprocess.Popen(cmd, shell=False, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -89,7 +135,19 @@ def _run_module(wrapped_cmd, jid, job_path):
         if PY3:
             outdata = outdata.decode('utf-8', 'surrogateescape')
             stderr = stderr.decode('utf-8', 'surrogateescape')
-        result = json.loads(outdata)
+
+        (filtered_outdata, json_warnings) = _filter_non_json_lines(outdata)
+
+        result = json.loads(filtered_outdata)
+
+        if json_warnings:
+            # merge JSON junk warnings with any existing module warnings
+            module_warnings = result.get('warnings', [])
+            if type(module_warnings) is not list:
+                module_warnings = [module_warnings]
+            module_warnings.extend(json_warnings)
+            result['warnings'] = module_warnings
+
         if stderr:
             result['stderr'] = stderr
         jobfile.write(json.dumps(result))
@@ -100,11 +158,13 @@ def _run_module(wrapped_cmd, jid, job_path):
             "failed": 1,
             "cmd" : wrapped_cmd,
             "msg": str(e),
+            "outdata": outdata, # temporary notice only
+            "stderr": stderr
         }
         result['ansible_job_id'] = jid
         jobfile.write(json.dumps(result))
 
-    except:
+    except (ValueError, Exception):
         result = {
             "failed" : 1,
             "cmd" : wrapped_cmd,