archive: Add support for xz (#34110)

* Add support for xz format on archive module

Fixes #34037
Fixes #34119

Signed-off-by: Alberto Murillo <albertomurillosilva@gmail.com>

lib/ansible/modules/files/archive.py

@@ -30,7 +30,8 @@ options:
   format:
     description:
       - The type of compression to use.
-    choices: [ bz2, gz, tar, zip ]
+      - Support for xz was added in version 2.5.
+    choices: [ bz2, gz, tar, xz, zip ]
     default: gz
   dest:
     description:
@@ -49,8 +50,9 @@ options:
 author:
 - Ben Doherty (@bendoh)
 notes:
-    - requires tarfile, zipfile, gzip, and bzip2 packages on target host
+    - requires tarfile, zipfile, gzip and bzip2 packages on target host
-    - can produce I(gzip), I(bzip2) and I(zip) compressed files or archives
+    - requires lzma or backports.lzma if using xz format
+    - can produce I(gzip), I(bzip2), I(lzma) and I(zip) compressed files or archives
 '''
 
 EXAMPLES = '''
@@ -133,6 +135,7 @@ import bz2
 import filecmp
 import glob
 import gzip
+import io
 import os
 import re
 import shutil
@@ -142,13 +145,27 @@ from traceback import format_exc
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
+from ansible.module_utils.six import PY3
+
+if PY3:
+    try:
+        import lzma
+        HAS_LZMA = True
+    except ImportError:
+        HAS_LZMA = False
+else:
+    try:
+        from backports import lzma
+        HAS_LZMA = True
+    except ImportError:
+        HAS_LZMA = False
 
 
 def main():
     module = AnsibleModule(
         argument_spec=dict(
             path=dict(type='list', required=True),
-            format=dict(type='str', default='gz', choices=['bz2', 'gz', 'tar', 'zip']),
+            format=dict(type='str', default='gz', choices=['bz2', 'gz', 'tar', 'xz', 'zip']),
             dest=dict(type='path'),
             exclude_path=dict(type='list'),
             remove=dict(type='bool', default=False),
@@ -175,6 +192,10 @@ def main():
     archive = False
     successes = []
 
+    # Fail early
+    if not HAS_LZMA and format == 'xz':
+        module.fail_json(msg="lzma or backports.lzma is required when using xz format.")
+
     for path in paths:
         path = os.path.expanduser(os.path.expandvars(path))
 
@@ -251,7 +272,7 @@ def main():
     # No source files were found but the named archive exists: are we 'compress' or 'archive' now?
     if len(missing) == len(expanded_paths) and dest and os.path.exists(dest):
         # Just check the filename to know if it's an archive or simple compressed file
-        if re.search(r'(\.tar|\.tar\.gz|\.tgz|.tbz2|\.tar\.bz2|\.zip)$', os.path.basename(dest), re.IGNORECASE):
+        if re.search(r'(\.tar|\.tar\.gz|\.tgz|\.tbz2|\.tar\.bz2|\.tar\.xz|\.zip)$', os.path.basename(dest), re.IGNORECASE):
             state = 'archive'
         else:
             state = 'compress'
@@ -287,6 +308,12 @@ def main():
                     elif format == 'gz' or format == 'bz2':
                         arcfile = tarfile.open(dest, 'w|' + format)
 
+                    # python3 tarfile module allows xz format but for python2 we have to create the tarfile
+                    # in memory and then compress it with lzma.
+                    elif format == 'xz':
+                        arcfileIO = io.BytesIO()
+                        arcfile = tarfile.open(fileobj=arcfileIO, mode='w')
+
                     # Or plain tar archiving
                     elif format == 'tar':
                         arcfile = tarfile.open(dest, 'w')
@@ -342,6 +369,11 @@ def main():
                     arcfile.close()
                     state = 'archive'
 
+                if format == 'xz':
+                    with lzma.open(dest, 'wb') as f:
+                        f.write(arcfileIO.getvalue())
+                    arcfileIO.close()
+
                 if errors:
                     module.fail_json(msg='Errors when writing archive at %s: %s' % (dest, '; '.join(errors)))
 
@@ -402,6 +434,8 @@ def main():
                             f_out = gzip.open(dest, 'wb')
                         elif format == 'bz2':
                             f_out = bz2.BZ2File(dest, 'wb')
+                        elif format == 'xz':
+                            f_out = lzma.LZMAFile(dest, 'wb')
                         else:
                             raise OSError("Invalid format")
 
@@ -447,5 +481,6 @@ def main():
                      expanded_paths=expanded_paths,
                      expanded_exclude_paths=expanded_exclude_paths)
 
+
 if __name__ == '__main__':
     main()

test/integration/targets/archive/tasks/main.yml

@@ -25,6 +25,44 @@
   apt: name=zip state=latest
   when: ansible_pkg_mgr == 'apt'
 
+- name: Install prerequisites for backports.lzma when using python2 (non OSX)
+  block:
+    - name: Set liblzma package name depending on the OS
+      set_fact:
+        liblzma_dev_package:
+          Debian: liblzma-dev
+          RedHat: xz-devel
+          Suse: xz-devel
+    - name: Ensure liblzma-dev is present to install backports-lzma
+      package: name={{ liblzma_dev_package[ansible_os_family] }} state=latest
+      when: ansible_os_family in liblzma_dev_package.keys()
+  when:
+    - ansible_python_version.split('.')[0] == '2'
+    - ansible_os_family != 'Darwin'
+
+- name: Install prerequisites for backports.lzma when using python2 (OSX)
+  block:
+    - name: Find brew binary
+      command: which brew
+      register: brew_which
+    - name: Get owner of brew binary
+      stat: path="{{ brew_which.stdout }}"
+      register: brew_stat
+    - name: "Install package"
+      homebrew:
+        name: xz
+        state: present
+        update_homebrew: no
+      become: yes
+      become_user: "{{ brew_stat.stat.pw_name }}"
+  when:
+    - ansible_python_version.split('.')[0] == '2'
+    - ansible_os_family == 'Darwin'
+
+- name: Ensure backports.lzma is present to create test archive (pip)
+  pip: name=backports.lzma state=latest
+  when: ansible_python_version.split('.')[0] == '2'
+
 - name: prep our file
   copy: src={{ item }} dest={{output_dir}}/{{ item }}
   with_items:
@@ -81,13 +119,32 @@
 - name: verify that the files archived
   file: path={{output_dir}}/archive_01.bz2 state=file
 
-- name: check if zip file exists
+- name: check if bzip file exists
   assert:
     that:
       - "{{ archive_bz2_result_01.changed }}"
       - "{{ 'archived' in archive_bz2_result_01 }}"
       - "{{ archive_bz2_result_01['archived'] | length }} == 2"
 
+- name: archive using xz
+  archive:
+    path: "{{ output_dir }}/*.txt"
+    dest: "{{ output_dir }}/archive_01.xz"
+    format: xz
+  register: archive_xz_result_01
+
+- debug: msg="{{ archive_xz_result_01 }}"
+
+- name: verify that the files archived
+  file: path={{output_dir}}/archive_01.xz state=file
+
+- name: check if xz file exists
+  assert:
+    that:
+      - "{{ archive_xz_result_01.changed }}"
+      - "{{ 'archived' in archive_xz_result_01 }}"
+      - "{{ archive_xz_result_01['archived'] | length }} == 2"
+
 - name: archive and set mode to 0600
   archive:
     path: "{{ output_dir }}/*.txt"
@@ -164,6 +221,30 @@
 - name: remove our bz2
   file: path="{{ output_dir }}/archive_02.bz2" state=absent
 
+- name: archive and set mode to 0600
+  archive:
+    path: "{{ output_dir }}/*.txt"
+    dest: "{{ output_dir }}/archive_02.xz"
+    format: xz
+    mode: "u+rwX,g-rwx,o-rwx"
+  register: archive_xz_result_02
+
+- name: Test that the file modes were changed
+  stat:
+    path: "{{ output_dir }}/archive_02.xz"
+  register: archive_02_xz_stat
+
+- name: Test that the file modes were changed
+  assert:
+    that:
+      - "archive_02_xz_stat.changed == False"
+      - "archive_02_xz_stat.stat.mode == '0600'"
+      - "'archived' in archive_xz_result_02"
+      - "{{ archive_xz_result_02['archived']| length}} == 2"
+
+- name: remove our xz
+  file: path="{{ output_dir }}/archive_02.xz" state=absent
+
 - name: test that gz archive that contains non-ascii filenames
   archive:
     path: "{{ output_dir }}/*.txt"
@@ -206,6 +287,27 @@
 - name: remove nonascii test
   file: path="{{ output_dir }}/test-archive-nonascii-くらとみ.bz2" state=absent
 
+- name: test that xz archive that contains non-ascii filenames
+  archive:
+    path: "{{ output_dir }}/*.txt"
+    dest: "{{ output_dir }}/test-archive-nonascii-くらとみ.xz"
+    format: xz
+  register: nonascii_result_1
+
+- name: Check that file is really there
+  stat:
+    path: "{{ output_dir }}/test-archive-nonascii-くらとみ.xz"
+  register: nonascii_stat_1
+
+- name: Assert that nonascii tests succeeded
+  assert:
+    that:
+      - "nonascii_result_1.changed == true"
+      - "nonascii_stat_1.stat.exists == true"
+
+- name: remove nonascii test
+  file: path="{{ output_dir }}/test-archive-nonascii-くらとみ.xz" state=absent
+
 - name: test that zip archive that contains non-ascii filenames
   archive:
     path: "{{ output_dir }}/*.txt"