@ -172,7 +172,7 @@ import zipfile
from traceback import format_exc
from traceback import format_exc
from ansible . module_utils . basic import AnsibleModule , missing_required_lib
from ansible . module_utils . basic import AnsibleModule , missing_required_lib
from ansible . module_utils . _text import to_ native
from ansible . module_utils . _text import to_ bytes, to_ native
from ansible . module_utils . six import PY3
from ansible . module_utils . six import PY3
@ -211,12 +211,14 @@ def main():
check_mode = module . check_mode
check_mode = module . check_mode
paths = params [ ' path ' ]
paths = params [ ' path ' ]
dest = params [ ' dest ' ]
dest = params [ ' dest ' ]
b_dest = None if not dest else to_bytes ( dest , errors = ' surrogate_or_strict ' )
exclude_paths = params [ ' exclude_path ' ]
exclude_paths = params [ ' exclude_path ' ]
remove = params [ ' remove ' ]
remove = params [ ' remove ' ]
expanded_paths = [ ]
b_expanded_paths = [ ]
expanded_exclude_paths = [ ]
b_expanded_exclude_paths = [ ]
format = params [ ' format ' ]
fmt = params [ ' format ' ]
b_fmt = to_bytes ( fmt , errors = ' surrogate_or_strict ' )
force_archive = params [ ' force_archive ' ]
force_archive = params [ ' force_archive ' ]
globby = False
globby = False
changed = False
changed = False
@ -224,111 +226,128 @@ def main():
# Simple or archive file compression (inapplicable with 'zip' since it's always an archive)
# Simple or archive file compression (inapplicable with 'zip' since it's always an archive)
archive = False
archive = False
successes = [ ]
b_ successes = [ ]
# Fail early
# Fail early
if not HAS_LZMA and forma t == ' xz ' :
if not HAS_LZMA and fm t == ' xz ' :
module . fail_json ( msg = missing_required_lib ( " lzma or backports.lzma " , reason = " when using xz format " ) ,
module . fail_json ( msg = missing_required_lib ( " lzma or backports.lzma " , reason = " when using xz format " ) ,
exception = LZMA_IMP_ERR )
exception = LZMA_IMP_ERR )
module . fail_json ( msg = " lzma or backports.lzma is required when using xz format. " )
module . fail_json ( msg = " lzma or backports.lzma is required when using xz format. " )
for path in paths :
for path in paths :
path = os . path . expanduser ( os . path . expandvars ( path ) )
b_path = os . path . expanduser (
os . path . expandvars (
to_bytes ( path , errors = ' surrogate_or_strict ' )
)
)
# Expand any glob characters. If found, add the expanded glob to the
# Expand any glob characters. If found, add the expanded glob to the
# list of expanded_paths, which might be empty.
# list of expanded_paths, which might be empty.
if ( ' * ' in path or ' ? ' in path ) :
if ( b ' * ' in b_ path or b ' ? ' in b_ path) :
expanded_paths = expanded_paths + glob . glob ( path )
b_expanded_paths. extend ( glob . glob ( b_path ) )
globby = True
globby = True
# If there are no glob characters the path is added to the expanded paths
# If there are no glob characters the path is added to the expanded paths
# whether the path exists or not
# whether the path exists or not
else :
else :
expanded_paths. append ( path)
b_ expanded_paths. append ( b_ path)
# Only attempt to expand the exclude paths if it exists
# Only attempt to expand the exclude paths if it exists
if exclude_paths :
if exclude_paths :
for exclude_path in exclude_paths :
for exclude_path in exclude_paths :
exclude_path = os . path . expanduser ( os . path . expandvars ( exclude_path ) )
b_exclude_path = os . path . expanduser (
os . path . expandvars (
to_bytes ( exclude_path , errors = ' surrogate_or_strict ' )
)
)
# Expand any glob characters. If found, add the expanded glob to the
# Expand any glob characters. If found, add the expanded glob to the
# list of expanded_paths, which might be empty.
# list of expanded_paths, which might be empty.
if ( ' * ' in exclude_path or ' ? ' in exclude_path ) :
if ( b ' * ' in b_ exclude_path or b ' ? ' in b_ exclude_path) :
expanded_exclude_paths = expanded_exclude_paths + glob . glob ( exclude_path )
b_expanded_exclude_paths. extend ( glob . glob ( b_exclude_path ) )
# If there are no glob character the exclude path is added to the expanded
# If there are no glob character the exclude path is added to the expanded
# exclude paths whether the path exists or not.
# exclude paths whether the path exists or not.
else :
else :
expanded_exclude_paths. append ( exclude_path)
b_ expanded_exclude_paths. append ( b_ exclude_path)
if not expanded_paths :
if not b_expanded_paths :
return module . fail_json ( path = ' , ' . join ( paths ) , expanded_paths = ' , ' . join ( expanded_paths ) , msg = ' Error, no source paths were found ' )
return module . fail_json (
path = ' , ' . join ( paths ) ,
expanded_paths = to_native ( b ' , ' . join ( b_expanded_paths ) , errors = ' surrogate_or_strict ' ) ,
msg = ' Error, no source paths were found '
)
# Only try to determine if we are working with an archive or not if we haven't set archive to true
# Only try to determine if we are working with an archive or not if we haven't set archive to true
if not force_archive :
if not force_archive :
# If we actually matched multiple files or TRIED to, then
# If we actually matched multiple files or TRIED to, then
# treat this as a multi-file archive
# treat this as a multi-file archive
archive = globby or os . path . isdir ( expanded_paths[ 0 ] ) or len ( expanded_paths) > 1
archive = globby or os . path . isdir ( b_ expanded_paths[ 0 ] ) or len ( b_ expanded_paths) > 1
else :
else :
archive = True
archive = True
# Default created file name (for single-file archives) to
# Default created file name (for single-file archives) to
# <file>.<format>
# <file>.<format>
if not dest and not archive :
if not b_ dest and not archive :
dest = ' %s . %s ' % ( expanded_paths[ 0 ] , forma t)
b_ dest = b ' %s . %s ' % ( b_ expanded_paths[ 0 ] , b_fm t)
# Force archives to specify 'dest'
# Force archives to specify 'dest'
if archive and not dest:
if archive and not b_ dest:
module . fail_json ( dest = dest , path = ' , ' . join ( paths ) , msg = ' Error, must specify " dest " when archiving multiple files or trees ' )
module . fail_json ( dest = dest , path = ' , ' . join ( paths ) , msg = ' Error, must specify " dest " when archiving multiple files or trees ' )
archive_paths = [ ]
b_sep = to_bytes ( os . sep , errors = ' surrogate_or_strict ' )
missing = [ ]
arcroot = ' '
b_archive_paths = [ ]
b_missing = [ ]
b_arcroot = b ' '
for path in expanded_paths :
for b_ path in b_ expanded_paths:
# Use the longest common directory name among all the files
# Use the longest common directory name among all the files
# as the archive root path
# as the archive root path
if arcroot == ' ' :
if b_ arcroot == b ' ' :
arcroot = os . path . dirname ( path) + os . sep
b_ arcroot = os . path . dirname ( b_path) + b_ sep
else :
else :
for i in range ( len ( arcroot) ) :
for i in range ( len ( b_ arcroot) ) :
if path[ i ] != arcroot[ i ] :
if b_ path[ i ] != b_ arcroot[ i ] :
break
break
if i < len ( arcroot) :
if i < len ( b_ arcroot) :
arcroot = os . path . dirname ( arcroot[ 0 : i + 1 ] )
b_ arcroot = os . path . dirname ( b_ arcroot[ 0 : i + 1 ] )
arcroot + = os . sep
b_arcroot + = b_ sep
# Don't allow archives to be created anywhere within paths to be removed
# Don't allow archives to be created anywhere within paths to be removed
if remove and os . path . isdir ( path ) :
if remove and os . path . isdir ( b_path ) :
path_dir = path
b_path_dir = b_path
if path [ - 1 ] != ' / ' :
if not b_path . endswith ( b ' / ' ) :
path_dir + = ' / '
b_path_dir + = b ' / '
if dest . startswith ( path_dir ) :
if b_dest . startswith ( b_path_dir ) :
module . fail_json ( path = ' , ' . join ( paths ) , msg = ' Error, created archive can not be contained in source paths when remove=True ' )
module . fail_json (
path = ' , ' . join ( paths ) ,
msg = ' Error, created archive can not be contained in source paths when remove=True '
)
if os . path . lexists ( path ) and path not in expanded_exclude_paths :
if os . path . lexists ( b_ path) and b_ path not in b_ expanded_exclude_paths:
archive_paths . append ( path )
b_ archive_paths. append ( b_ path)
else :
else :
missing. append ( path)
b_ missing. append ( b_ path)
# No source files were found but the named archive exists: are we 'compress' or 'archive' now?
# No source files were found but the named archive exists: are we 'compress' or 'archive' now?
if len ( missing) == len ( expanded_paths) and dest and os . path . exists ( dest) :
if len ( b_ missing) == len ( b_ expanded_paths) and b_ dest and os . path . exists ( b_ dest) :
# Just check the filename to know if it's an archive or simple compressed file
# Just check the filename to know if it's an archive or simple compressed file
if re . search ( r' ( \ .tar| \ .tar \ .gz| \ .tgz| \ .tbz2| \ .tar \ .bz2| \ .tar \ .xz| \ .zip)$ ' , os . path . basename ( dest) , re . IGNORECASE ) :
if re . search ( b r' ( \ .tar| \ .tar \ .gz| \ .tgz| \ .tbz2| \ .tar \ .bz2| \ .tar \ .xz| \ .zip)$ ' , os . path . basename ( b_ dest) , re . IGNORECASE ) :
state = ' archive '
state = ' archive '
else :
else :
state = ' compress '
state = ' compress '
# Multiple files, or globbiness
# Multiple files, or globbiness
elif archive :
elif archive :
if not archive_paths:
if not b_ archive_paths:
# No source files were found, but the archive is there.
# No source files were found, but the archive is there.
if os . path . lexists ( dest) :
if os . path . lexists ( b_ dest) :
state = ' archive '
state = ' archive '
elif missing:
elif b_ missing:
# SOME source files were found, but not all of them
# SOME source files were found, but not all of them
state = ' incomplete '
state = ' incomplete '
@ -336,8 +355,8 @@ def main():
size = 0
size = 0
errors = [ ]
errors = [ ]
if os . path . lexists ( dest) :
if os . path . lexists ( b_ dest) :
size = os . path . getsize ( dest)
size = os . path . getsize ( b_ dest)
if state != ' archive ' :
if state != ' archive ' :
if check_mode :
if check_mode :
@ -346,76 +365,88 @@ def main():
else :
else :
try :
try :
# Slightly more difficult (and less efficient!) compression using zipfile module
# Slightly more difficult (and less efficient!) compression using zipfile module
if format == ' zip ' :
if fmt == ' zip ' :
arcfile = zipfile . ZipFile ( dest , ' w ' , zipfile . ZIP_DEFLATED , True )
arcfile = zipfile . ZipFile (
to_native ( b_dest , errors = ' surrogate_or_strict ' , encoding = ' ascii ' ) ,
' w ' ,
zipfile . ZIP_DEFLATED ,
True
)
# Easier compression using tarfile module
# Easier compression using tarfile module
elif format == ' gz ' or format == ' bz2 ' :
elif fmt == ' gz ' or fm t == ' bz2 ' :
arcfile = tarfile . open ( dest , ' w| ' + format )
arcfile = tarfile . open ( to_native( b_ dest, errors = ' surrogate_or_strict ' , encoding = ' ascii ' ) , ' w| ' + fm t)
# python3 tarfile module allows xz format but for python2 we have to create the tarfile
# python3 tarfile module allows xz format but for python2 we have to create the tarfile
# in memory and then compress it with lzma.
# in memory and then compress it with lzma.
elif forma t == ' xz ' :
elif fm t == ' xz ' :
arcfileIO = io . BytesIO ( )
arcfileIO = io . BytesIO ( )
arcfile = tarfile . open ( fileobj = arcfileIO , mode = ' w ' )
arcfile = tarfile . open ( fileobj = arcfileIO , mode = ' w ' )
# Or plain tar archiving
# Or plain tar archiving
elif forma t == ' tar ' :
elif fm t == ' tar ' :
arcfile = tarfile . open ( dest, ' w ' )
arcfile = tarfile . open ( to_native( b_ dest, errors = ' surrogate_or_strict ' , encoding = ' ascii ' ) , ' w ' )
match_root = re . compile ( ' ^ %s ' % re . escape ( arcroot) )
b_ match_root = re . compile ( br ' ^ %s ' % re . escape ( b_ arcroot) )
for path in archive_paths:
for b_ path in b_ archive_paths:
if os . path . isdir ( path) :
if os . path . isdir ( b_ path) :
# Recurse into directories
# Recurse into directories
for dirpath, dirnames, filenames in os . walk ( path, topdown = True ) :
for b_ dirpath, b_ dirnames, b_ filenames in os . walk ( b_ path, topdown = True ) :
if not dirpath. endswith ( os . sep) :
if not b_dirpath. endswith ( b_ sep) :
dirpath + = os . sep
b_dirpath + = b_ sep
for dirname in dirnames :
for b_dirname in b_dirnames :
fullpath = dirpath + dirname
b_fullpath = b_dirpath + b_dirname
arcname = match_root . sub ( ' ' , fullpath )
n_fullpath = to_native ( b_fullpath , errors = ' surrogate_or_strict ' , encoding = ' ascii ' )
n_arcname = to_native ( b_match_root . sub ( b ' ' , b_fullpath ) , errors = ' surrogate_or_strict ' )
try :
try :
if forma t == ' zip ' :
if fm t == ' zip ' :
arcfile . write ( fullpath, arcname)
arcfile . write ( n_ fullpath, n_ arcname)
else :
else :
arcfile . add ( fullpath, arcname, recursive = False )
arcfile . add ( n_ fullpath, n_ arcname, recursive = False )
except Exception as e :
except Exception as e :
errors . append ( ' %s : %s ' % ( fullpath, to_native ( e ) ) )
errors . append ( ' %s : %s ' % ( n_ fullpath, to_native ( e ) ) )
for filename in filenames :
for b_filename in b_filenames :
fullpath = dirpath + filename
b_fullpath = b_dirpath + b_filename
arcname = match_root . sub ( ' ' , fullpath )
n_fullpath = to_native ( b_fullpath , errors = ' surrogate_or_strict ' , encoding = ' ascii ' )
n_arcname = to_native ( b_match_root . sub ( b ' ' , b_fullpath ) , errors = ' surrogate_or_strict ' )
if not filecmp . cmp ( fullpath, dest) :
if not filecmp . cmp ( b_ fullpath, b_ dest) :
try :
try :
if forma t == ' zip ' :
if fm t == ' zip ' :
arcfile . write ( fullpath, arcname)
arcfile . write ( n_ fullpath, n_ arcname)
else :
else :
arcfile . add ( fullpath, arcname, recursive = False )
arcfile . add ( n_ fullpath, n_ arcname, recursive = False )
successes. append ( fullpath)
b_ successes. append ( b_ fullpath)
except Exception as e :
except Exception as e :
errors . append ( ' Adding %s : %s ' % ( path, to_native ( e ) ) )
errors . append ( ' Adding %s : %s ' % ( to_native( b_ path) , to_native ( e ) ) )
else :
else :
if format == ' zip ' :
path = to_native ( b_path , errors = ' surrogate_or_strict ' , encoding = ' ascii ' )
arcfile . write ( path , match_root . sub ( ' ' , path ) )
arcname = to_native ( b_match_root . sub ( b ' ' , b_path ) , errors = ' surrogate_or_strict ' )
if fmt == ' zip ' :
arcfile . write ( path , arcname )
else :
else :
arcfile . add ( path , match_root . sub ( ' ' , path ) , recursive = False )
arcfile . add ( path , arcname , recursive = False )
successes. append ( path)
b_ successes. append ( b_ path)
except Exception as e :
except Exception as e :
module . fail_json ( msg = ' Error when writing %s archive at %s : %s ' % ( format == ' zip ' and ' zip ' or ( ' tar. ' + format ) , dest , to_native ( e ) ) ,
expanded_fmt = ' zip ' if fmt == ' zip ' else ( ' tar. ' + fmt )
exception = format_exc ( ) )
module . fail_json (
msg = ' Error when writing %s archive at %s : %s ' % ( expanded_fmt , dest , to_native ( e ) ) ,
exception = format_exc ( )
)
if arcfile :
if arcfile :
arcfile . close ( )
arcfile . close ( )
state = ' archive '
state = ' archive '
if forma t == ' xz ' :
if fm t == ' xz ' :
with lzma . open ( dest, ' wb ' ) as f :
with lzma . open ( b_ dest, ' wb ' ) as f :
f . write ( arcfileIO . getvalue ( ) )
f . write ( arcfileIO . getvalue ( ) )
arcfileIO . close ( )
arcfileIO . close ( )
@ -423,76 +454,89 @@ def main():
module . fail_json ( msg = ' Errors when writing archive at %s : %s ' % ( dest , ' ; ' . join ( errors ) ) )
module . fail_json ( msg = ' Errors when writing archive at %s : %s ' % ( dest , ' ; ' . join ( errors ) ) )
if state in [ ' archive ' , ' incomplete ' ] and remove :
if state in [ ' archive ' , ' incomplete ' ] and remove :
for path in successes:
for b_ path in b_ successes:
try :
try :
if os . path . isdir ( path) :
if os . path . isdir ( b_ path) :
shutil . rmtree ( path)
shutil . rmtree ( b_ path)
elif not check_mode :
elif not check_mode :
os . remove ( path)
os . remove ( b_ path)
except OSError as e :
except OSError as e :
errors . append ( path)
errors . append ( to_native( b_ path) )
if errors :
if errors :
module . fail_json ( dest = dest , msg = ' Error deleting some source files: ' , files = errors )
module . fail_json ( dest = dest , msg = ' Error deleting some source files: ' , files = errors )
# Rudimentary check: If size changed then file changed. Not perfect, but easy.
# Rudimentary check: If size changed then file changed. Not perfect, but easy.
if not check_mode and os . path . getsize ( dest) != size :
if not check_mode and os . path . getsize ( b_ dest) != size :
changed = True
changed = True
if successes and state != ' incomplete ' :
if b_ successes and state != ' incomplete ' :
state = ' archive '
state = ' archive '
# Simple, single-file compression
# Simple, single-file compression
else :
else :
path = expanded_paths[ 0 ]
b_ path = b_ expanded_paths[ 0 ]
# No source or compressed file
# No source or compressed file
if not ( os . path . exists ( path) or os . path . lexists ( dest) ) :
if not ( os . path . exists ( b_ path) or os . path . lexists ( b_ dest) ) :
state = ' absent '
state = ' absent '
# if it already exists and the source file isn't there, consider this done
# if it already exists and the source file isn't there, consider this done
elif not os . path . lexists ( path) and os . path . lexists ( dest) :
elif not os . path . lexists ( b_ path) and os . path . lexists ( b_ dest) :
state = ' compress '
state = ' compress '
else :
else :
if module . check_mode :
if module . check_mode :
if not os . path . exists ( dest) :
if not os . path . exists ( b_ dest) :
changed = True
changed = True
else :
else :
size = 0
size = 0
f_in = f_out = arcfile = None
f_in = f_out = arcfile = None
if os . path . lexists ( dest) :
if os . path . lexists ( b_ dest) :
size = os . path . getsize ( dest)
size = os . path . getsize ( b_ dest)
try :
try :
if format == ' zip ' :
if fmt == ' zip ' :
arcfile = zipfile . ZipFile ( dest , ' w ' , zipfile . ZIP_DEFLATED , True )
arcfile = zipfile . ZipFile (
arcfile . write ( path , path [ len ( arcroot ) : ] )
to_native ( b_dest , errors = ' surrogate_or_strict ' , encoding = ' ascii ' ) ,
' w ' ,
zipfile . ZIP_DEFLATED ,
True
)
arcfile . write (
to_native ( b_path , errors = ' surrogate_or_strict ' , encoding = ' ascii ' ) ,
to_native ( b_path [ len ( b_arcroot ) : ] , errors = ' surrogate_or_strict ' )
)
arcfile . close ( )
arcfile . close ( )
state = ' archive ' # because all zip files are archives
state = ' archive ' # because all zip files are archives
elif format == ' tar ' :
elif fm t == ' tar ' :
arcfile = tarfile . open ( dest , ' w ' )
arcfile = tarfile . open ( to_native( b_ dest, errors = ' surrogate_or_strict ' , encoding = ' ascii ' ) , ' w ' )
arcfile . add ( path )
arcfile . add ( to_native( b_ path, errors = ' surrogate_or_strict ' , encoding = ' ascii ' ) )
arcfile . close ( )
arcfile . close ( )
else :
else :
f_in = open ( path , ' rb ' )
f_in = open ( b_path , ' rb ' )
if format == ' gz ' :
n_dest = to_native ( b_dest , errors = ' surrogate_or_strict ' , encoding = ' ascii ' )
f_out = gzip . open ( dest , ' wb ' )
if fmt == ' gz ' :
elif format == ' bz2 ' :
f_out = gzip . open ( n_dest , ' wb ' )
f_out = bz2 . BZ2File ( dest , ' wb ' )
elif fmt == ' bz2 ' :
elif format == ' xz ' :
f_out = bz2 . BZ2File ( n_dest , ' wb ' )
f_out = lzma . LZMAFile ( dest , ' wb ' )
elif fmt == ' xz ' :
f_out = lzma . LZMAFile ( n_dest , ' wb ' )
else :
else :
raise OSError ( " Invalid format " )
raise OSError ( " Invalid format " )
shutil . copyfileobj ( f_in , f_out )
shutil . copyfileobj ( f_in , f_out )
successes. append ( path)
b_ successes. append ( b_ path)
except OSError as e :
except OSError as e :
module . fail_json ( path = path , dest = dest , msg = ' Unable to write to compressed file: %s ' % to_native ( e ) , exception = format_exc ( ) )
module . fail_json (
path = to_native ( b_path ) ,
dest = dest ,
msg = ' Unable to write to compressed file: %s ' % to_native ( e ) , exception = format_exc ( )
)
if arcfile :
if arcfile :
arcfile . close ( )
arcfile . close ( )
@ -502,32 +546,37 @@ def main():
f_out . close ( )
f_out . close ( )
# Rudimentary check: If size changed then file changed. Not perfect, but easy.
# Rudimentary check: If size changed then file changed. Not perfect, but easy.
if os . path . getsize ( dest) != size :
if os . path . getsize ( b_ dest) != size :
changed = True
changed = True
state = ' compress '
state = ' compress '
if remove and not check_mode :
if remove and not check_mode :
try :
try :
os . remove ( path)
os . remove ( b_ path)
except OSError as e :
except OSError as e :
module . fail_json ( path = path , msg = ' Unable to remove source file: %s ' % to_native ( e ) , exception = format_exc ( ) )
module . fail_json (
path = to_native ( b_path ) ,
msg = ' Unable to remove source file: %s ' % to_native ( e ) , exception = format_exc ( )
)
params [ ' path ' ] = dest
params [ ' path ' ] = b_ dest
file_args = module . load_file_common_arguments ( params )
file_args = module . load_file_common_arguments ( params )
if not check_mode :
if not check_mode :
changed = module . set_fs_attributes_if_different ( file_args , changed )
changed = module . set_fs_attributes_if_different ( file_args , changed )
module . exit_json ( archived = successes ,
module . exit_json (
archived = [ to_native ( p , errors = ' surrogate_or_strict ' ) for p in b_successes ] ,
dest = dest ,
dest = dest ,
changed = changed ,
changed = changed ,
state = state ,
state = state ,
arcroot = arcroot ,
arcroot = to_native ( b_arcroot , errors = ' surrogate_or_strict ' ) ,
missing = missing ,
missing = [ to_native ( p , errors = ' surrogate_or_strict ' ) for p in b_missing ] ,
expanded_paths = expanded_paths ,
expanded_paths = [ to_native ( p , errors = ' surrogate_or_strict ' ) for p in b_expanded_paths ] ,
expanded_exclude_paths = expanded_exclude_paths )
expanded_exclude_paths = [ to_native ( p , errors = ' surrogate_or_strict ' ) for p in b_expanded_exclude_paths ] ,
)
if __name__ == ' __main__ ' :
if __name__ == ' __main__ ' :