From 1cc5efa77b3dd8b0e24cd60175b034a6c4ca4c2c Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 31 Aug 2023 12:02:48 -0700 Subject: [PATCH] tarfile: Handle deprecation warning for extract and extractall (#81545) * Python 3.11.4 introduces a new parameter 'filter' in extract and extractall in tarfile. Handle deprecation warning message emitted in Python 3.12. * added probing mechanism in ansible-galaxy code to detect broken data filter implementation in tarfile. Fixes: #80832 Signed-off-by: Abhijeet Kasurde Co-authored-by: Matt Clay --- changelogs/fragments/tarfile_extract_warn.yml | 4 +++ lib/ansible/galaxy/role.py | 34 ++++++++++++++++++- packaging/release.py | 6 +++- .../library/setup_collections.py | 7 +++- .../commands/sanity/validate_modules.py | 6 +++- 5 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 changelogs/fragments/tarfile_extract_warn.yml diff --git a/changelogs/fragments/tarfile_extract_warn.yml b/changelogs/fragments/tarfile_extract_warn.yml new file mode 100644 index 00000000000..13938faf39d --- /dev/null +++ b/changelogs/fragments/tarfile_extract_warn.yml @@ -0,0 +1,4 @@ +--- +bugfixes: +- tarfile - handle data filter deprecation warning message for extract and extractall (https://github.com/ansible/ansible/issues/80832). +- ansible-galaxy - Enabled the ``data`` tarfile filter during role installation for Python versions that support it. A probing mechanism is used to avoid Python versions with a broken implementation. diff --git a/lib/ansible/galaxy/role.py b/lib/ansible/galaxy/role.py index b5ba1006785..a71749fc131 100644 --- a/lib/ansible/galaxy/role.py +++ b/lib/ansible/galaxy/role.py @@ -24,6 +24,7 @@ __metaclass__ = type import errno import datetime +import functools import os import tarfile import tempfile @@ -45,6 +46,32 @@ from ansible.utils.display import Display display = Display() +@functools.cache +def _check_working_data_filter() -> bool: + """ + Check if tarfile.data_filter implementation is working + for the current Python version or not + """ + + # Implemented the following code to circumvent broken implementation of data_filter + # in tarfile. See for more information - https://github.com/python/cpython/issues/107845 + # deprecated: description='probing broken data filter implementation' python_version='3.11' + ret = False + if hasattr(tarfile, 'data_filter'): + # We explicitly check if tarfile.data_filter is broken or not + ti = tarfile.TarInfo('docs/README.md') + ti.type = tarfile.SYMTYPE + ti.linkname = '../README.md' + + try: + tarfile.data_filter(ti, '/foo') + except tarfile.LinkOutsideDestinationError: + pass + else: + ret = True + return ret + + class GalaxyRole(object): SUPPORTED_SCMS = set(['git', 'hg']) @@ -391,7 +418,12 @@ class GalaxyRole(object): continue n_final_parts.append(n_part) member.name = os.path.join(*n_final_parts) - role_tar_file.extract(member, to_native(self.path)) + + if _check_working_data_filter(): + # deprecated: description='extract fallback without filter' python_version='3.11' + role_tar_file.extract(member, to_native(self.path), filter='data') # type: ignore[call-arg] + else: + role_tar_file.extract(member, to_native(self.path)) # write out the install info file for later use self._write_galaxy_install_info() diff --git a/packaging/release.py b/packaging/release.py index b231f003af2..97c58a74248 100755 --- a/packaging/release.py +++ b/packaging/release.py @@ -1342,7 +1342,11 @@ def test_sdist() -> None: except FileNotFoundError: raise ApplicationError(f"Missing sdist: {sdist_file.relative_to(CHECKOUT_DIR)}") from None - sdist.extractall(temp_dir) + # deprecated: description='extractall fallback without filter' python_version='3.11' + if hasattr(tarfile, 'data_filter'): + sdist.extractall(temp_dir, filter='data') # type: ignore[call-arg] + else: + sdist.extractall(temp_dir) pyc_glob = "*.pyc*" pyc_files = sorted(path.relative_to(temp_dir) for path in temp_dir.rglob(pyc_glob)) diff --git a/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py index 7a40c5f6054..423edd9ee89 100644 --- a/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py +++ b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py @@ -156,7 +156,12 @@ def publish_collection(module, collection): # Extract the tarfile to sign the MANIFEST.json with tarfile.open(collection_path, mode='r') as collection_tar: - collection_tar.extractall(path=os.path.join(collection_dir, '%s-%s-%s' % (namespace, name, version))) + # deprecated: description='extractall fallback without filter' python_version='3.11' + # Replace 'tar_filter' with 'data_filter' and 'filter=tar' with 'filter=data' once Python 3.12 is minimum requirement. + if hasattr(tarfile, 'tar_filter'): + collection_tar.extractall(path=os.path.join(collection_dir, '%s-%s-%s' % (namespace, name, version)), filter='tar') + else: + collection_tar.extractall(path=os.path.join(collection_dir, '%s-%s-%s' % (namespace, name, version))) manifest_path = os.path.join(collection_dir, '%s-%s-%s' % (namespace, name, version), 'MANIFEST.json') signature_path = os.path.join(module.params['signature_dir'], '%s-%s-%s-MANIFEST.json.asc' % (namespace, name, version)) diff --git a/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py b/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py index d2b1c546d58..e29b5dec992 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py +++ b/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py @@ -159,7 +159,11 @@ class ValidateModulesTest(SanitySingleVersion): temp_dir = process_scoped_temporary_directory(args) with tarfile.open(path) as file: - file.extractall(temp_dir) + # deprecated: description='extractall fallback without filter' python_version='3.11' + if hasattr(tarfile, 'data_filter'): + file.extractall(temp_dir, filter='data') # type: ignore[call-arg] + else: + file.extractall(temp_dir) cmd.extend([ '--original-plugins', temp_dir,