Merge pull request #559 from jkleint/iterative-md5

Use iterative MD5 hashing.

lib/ansible/utils.py

@@ -33,11 +33,9 @@ except ImportError:
     import simplejson as json
 
 try:
-    import hashlib 
-    HAVE_HASHLIB=True
+    from hashlib import md5 as _md5
 except ImportError: 
-    import md5
-    HAVE_HASHLIB=False
+    from md5 import md5 as _md5
 
 from ansible import errors
 import ansible.constants as C
@@ -321,13 +319,19 @@ def parse_kv(args):
     return options
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
     if not os.path.exists(filename):
         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
+
 
 
 ####################################################################

library/assemble

@@ -30,14 +30,10 @@ import syslog
 import tempfile
 
 try:
-    import hashlib
-    HAVE_HASHLIB=True
+    from hashlib import md5 as _md5
 except ImportError: 
-    import md5
-    HAVE_HASHLIB=False
+    from md5 import md5 as _md5
 
-# Since hashlib is only available in 2.5 and onwards, this module
-# uses md5 which is available in 2.4.
 
 # ===========================================
 # Support methods
@@ -66,13 +62,18 @@ def write_temp_file(data):
     return path
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
     if not os.path.exists(filename):
         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
 
 # ===========================================
 

library/copy

@@ -25,11 +25,9 @@ import shutil
 import syslog
 
 try:
-    import hashlib
-    HAVE_HASHLIB=True
+    from hashlib import md5 as _md5
 except ImportError: 
-    import md5
-    HAVE_HASHLIB=False
+    from md5 import md5 as _md5
 
 # ===========================================
 # convert arguments of form a=b c=d
@@ -46,13 +44,18 @@ def exit_kv(rc=0, **kwargs):
     sys.exit(rc)
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
     if not os.path.exists(filename):
         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
 
 # ===========================================
 

library/setup

@@ -34,11 +34,9 @@ import traceback
 import syslog
 
 try:
-    import hashlib
-    HAVE_HASHLIB=True
+    from hashlib import md5 as _md5
 except ImportError: 
-    import md5
-    HAVE_HASHLIB=False
+    from md5 import md5 as _md5
 
 try:
     import selinux
@@ -319,13 +317,18 @@ def ansible_facts():
     return facts
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
     if not os.path.exists(filename):
         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
 
 # ===========================================