Merge pull request #1417 from fdupoux/mysql-db-use-python-pipelines

Decompress mysql dumps on the fly using python subprocess …

database/mysql/mysql_db.py

@@ -111,6 +111,7 @@ import ConfigParser
 import os
 import pipes
 import stat
+import subprocess
 try:
     import MySQLdb
 except ImportError:
@@ -171,57 +172,26 @@ def db_import(module, host, user, password, db_name, target, all_databases, port
         cmd += " --host=%s --port=%i" % (pipes.quote(host), port)
     if not all_databases:
     	cmd += " -D %s" % pipes.quote(db_name)
+
+    comp_prog_path = None
     if os.path.splitext(target)[-1] == '.gz':
-        gzip_path = module.get_bin_path('gzip')
+        comp_prog_path = module.get_bin_path('gzip', required=True)
-        if not gzip_path:
-            module.fail_json(msg="gzip command not found")
-        #gzip -d file (uncompress)
-        rc, stdout, stderr = module.run_command('%s -d %s' % (gzip_path, target))
-        if rc != 0:
-            return rc, stdout, stderr
-        #Import sql
-        cmd += " < %s" % pipes.quote(os.path.splitext(target)[0])
-        try:
-            rc, stdout, stderr = module.run_command(cmd, use_unsafe_shell=True)
-            if rc != 0:
-                return rc, stdout, stderr
-        finally:
-            #gzip file back up
-            module.run_command('%s %s' % (gzip_path, os.path.splitext(target)[0]))
     elif os.path.splitext(target)[-1] == '.bz2':
-        bzip2_path = module.get_bin_path('bzip2')
+        comp_prog_path = module.get_bin_path('bzip2', required=True)
-        if not bzip2_path:
-            module.fail_json(msg="bzip2 command not found")
-        #bzip2 -d file (uncompress)
-        rc, stdout, stderr = module.run_command('%s -d %s' % (bzip2_path, target))
-        if rc != 0:
-            return rc, stdout, stderr
-        #Import sql
-        cmd += " < %s" % pipes.quote(os.path.splitext(target)[0])
-        try:
-            rc, stdout, stderr = module.run_command(cmd, use_unsafe_shell=True)
-            if rc != 0:
-                return rc, stdout, stderr
-        finally:
-            #bzip2 file back up
-            rc, stdout, stderr = module.run_command('%s %s' % (bzip2_path, os.path.splitext(target)[0]))
     elif os.path.splitext(target)[-1] == '.xz':
-        xz_path = module.get_bin_path('xz')
+        comp_prog_path = module.get_bin_path('xz', required=True)
-        if not xz_path:
+
-            module.fail_json(msg="xz command not found")
+    if comp_prog_path:
-        #xz -d file (uncompress)
+        p1 = subprocess.Popen([comp_prog_path, '-dc', target], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        rc, stdout, stderr = module.run_command('%s -d %s' % (xz_path, target))
+        p2 = subprocess.Popen(cmd.split(' '), stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        if rc != 0:
+        (stdout2, stderr2) = p2.communicate()
-            return rc, stdout, stderr
+        p1.stdout.close()
-        #Import sql
+        p1.wait()
-        cmd += " < %s" % pipes.quote(os.path.splitext(target)[0])
+        if p1.returncode != 0:
-        try:
+            stderr1 = p1.stderr.read()
-            rc, stdout, stderr = module.run_command(cmd, use_unsafe_shell=True)
+            return p1.returncode, '', stderr1
-            if rc != 0:
+        else:
-                return rc, stdout, stderr
+            return p2.returncode, stdout2, stderr2
-        finally:
-            #xz file back up
-            rc, stdout, stderr = module.run_command('%s %s' % (xz_path, os.path.splitext(target)[0]))
     else:
         cmd += " < %s" % pipes.quote(target)
         rc, stdout, stderr = module.run_command(cmd, use_unsafe_shell=True)