diff --git a/test/sanity/import/importer.py b/test/sanity/import/importer.py index 58aa2255687..719341cb674 100755 --- a/test/sanity/import/importer.py +++ b/test/sanity/import/importer.py @@ -3,231 +3,230 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import contextlib -import os -import re -import sys -import traceback -import warnings - -try: - import importlib.util - imp = None # pylint: disable=invalid-name -except ImportError: - importlib = None # pylint: disable=invalid-name - import imp - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -import ansible.module_utils.basic -import ansible.module_utils.common.removed - -try: - from ansible.utils.collection_loader import AnsibleCollectionLoader -except ImportError: - AnsibleCollectionLoader = None - - -class ImporterAnsibleModuleException(Exception): - """Exception thrown during initialization of ImporterAnsibleModule.""" - - -class ImporterAnsibleModule: - """Replacement for AnsibleModule to support import testing.""" - def __init__(self, *args, **kwargs): - raise ImporterAnsibleModuleException() - - -# stop Ansible module execution during AnsibleModule instantiation -ansible.module_utils.basic.AnsibleModule = ImporterAnsibleModule -# no-op for _load_params since it may be called before instantiating AnsibleModule -ansible.module_utils.basic._load_params = lambda *args, **kwargs: {} # pylint: disable=protected-access -# no-op for removed_module since it is called in place of AnsibleModule instantiation -ansible.module_utils.common.removed.removed_module = lambda *args, **kwargs: None - def main(): - """Main program function.""" - base_dir = os.getcwd() - messages = set() - - if AnsibleCollectionLoader: - # allow importing code from collections - sys.meta_path.insert(0, AnsibleCollectionLoader()) - - for path in sys.argv[1:] or sys.stdin.read().splitlines(): - test_python_module(path, base_dir, messages, False) - test_python_module(path, base_dir, messages, True) - - if messages: - exit(10) - - -def test_python_module(path, base_dir, messages, ansible_module): - if ansible_module: - # importing modules with __main__ under Python 2.6 exits with status code 1 - if sys.version_info < (2, 7): - return - - # only run __main__ protected code for Ansible modules - if not path.startswith('lib/ansible/modules/'): - return + """ + Main program function used to isolate globals from imported code. + Changes to globals in imported modules on Python 2.7 will overwrite our own globals. + """ + import contextlib + import os + import re + import sys + import traceback + import warnings - # async_wrapper is not an Ansible module - if path == 'lib/ansible/modules/utilities/logic/async_wrapper.py': - return + try: + import importlib.util + imp = None # pylint: disable=invalid-name + except ImportError: + importlib = None # pylint: disable=invalid-name + import imp - # run code protected by __name__ conditional - name = '__main__' - # show the Ansible module responsible for the exception, even if it was thrown in module_utils - filter_dir = os.path.join(base_dir, 'lib/ansible/modules') - else: - # do not run code protected by __name__ conditional - name = 'module_import_test' - # show the Ansible file responsible for the exception, even if it was thrown in 3rd party code - filter_dir = base_dir + try: + from StringIO import StringIO + except ImportError: + from io import StringIO - capture = Capture() + import ansible.module_utils.basic + import ansible.module_utils.common.removed try: - if imp: - with open(path, 'r') as module_fd: - with capture_output(capture): - imp.load_module(name, module_fd, os.path.abspath(path), ('.py', 'r', imp.PY_SOURCE)) - else: - spec = importlib.util.spec_from_file_location(name, os.path.abspath(path)) - module = importlib.util.module_from_spec(spec) - - with capture_output(capture): - spec.loader.exec_module(module) - - capture_report(path, capture, messages) - except ImporterAnsibleModuleException: - # module instantiated AnsibleModule without raising an exception - pass - # We truly want to catch anything the plugin might do here, including call sys.exit() so we - # catch BaseException - except BaseException as ex: # pylint: disable=locally-disabled, broad-except - capture_report(path, capture, messages) - - exc_type, _exc, exc_tb = sys.exc_info() - message = str(ex) - results = list(reversed(traceback.extract_tb(exc_tb))) - source = None - line = 0 - offset = 0 - - if isinstance(ex, SyntaxError) and ex.filename.endswith(path): # pylint: disable=locally-disabled, no-member - # A SyntaxError in the source we're importing will have the correct path, line and offset. - # However, the traceback will report the path to this importer.py script instead. - # We'll use the details from the SyntaxError in this case, as it's more accurate. - source = path - line = ex.lineno or 0 # pylint: disable=locally-disabled, no-member - offset = ex.offset or 0 # pylint: disable=locally-disabled, no-member - message = str(ex) - - # Hack to remove the filename and line number from the message, if present. - message = message.replace(' (%s, line %d)' % (os.path.basename(path), line), '') + from ansible.utils.collection_loader import AnsibleCollectionLoader + except ImportError: + AnsibleCollectionLoader = None + + class ImporterAnsibleModuleException(Exception): + """Exception thrown during initialization of ImporterAnsibleModule.""" + + class ImporterAnsibleModule: + """Replacement for AnsibleModule to support import testing.""" + def __init__(self, *args, **kwargs): + raise ImporterAnsibleModuleException() + + # stop Ansible module execution during AnsibleModule instantiation + ansible.module_utils.basic.AnsibleModule = ImporterAnsibleModule + # no-op for _load_params since it may be called before instantiating AnsibleModule + ansible.module_utils.basic._load_params = lambda *args, **kwargs: {} # pylint: disable=protected-access + # no-op for removed_module since it is called in place of AnsibleModule instantiation + ansible.module_utils.common.removed.removed_module = lambda *args, **kwargs: None + + def run(): + """Main program function.""" + base_dir = os.getcwd() + messages = set() + + if AnsibleCollectionLoader: + # allow importing code from collections + sys.meta_path.insert(0, AnsibleCollectionLoader()) + + for path in sys.argv[1:] or sys.stdin.read().splitlines(): + test_python_module(path, base_dir, messages, False) + test_python_module(path, base_dir, messages, True) + + if messages: + exit(10) + + def test_python_module(path, base_dir, messages, ansible_module): + if ansible_module: + # importing modules with __main__ under Python 2.6 exits with status code 1 + if sys.version_info < (2, 7): + return + + # only run __main__ protected code for Ansible modules + if not path.startswith('lib/ansible/modules/'): + return + + # async_wrapper is not an Ansible module + if path == 'lib/ansible/modules/utilities/logic/async_wrapper.py': + return + + # run code protected by __name__ conditional + name = '__main__' + # show the Ansible module responsible for the exception, even if it was thrown in module_utils + filter_dir = os.path.join(base_dir, 'lib/ansible/modules') else: - for result in results: - if result[0].startswith(filter_dir): - source = result[0][len(base_dir) + 1:].replace('test/sanity/import/', '') - line = result[1] or 0 - break - - if not source: - # If none of our source files are found in the traceback, report the file we were testing. - # I haven't been able to come up with a test case that encounters this issue yet. - source = path - message += ' (in %s:%d)' % (results[-1][0], results[-1][1] or 0) - - message = re.sub(r'\n *', ': ', message) - error = '%s:%d:%d: %s: %s' % (source, line, offset, exc_type.__name__, message) - - report_message(error, messages) - - -class Capture: - """Captured output and/or exception.""" - def __init__(self): - self.stdout = StringIO() - self.stderr = StringIO() - self.warnings = [] + # do not run code protected by __name__ conditional + name = 'module_import_test' + # show the Ansible file responsible for the exception, even if it was thrown in 3rd party code + filter_dir = base_dir + capture = Capture() -def capture_report(path, capture, messages): - """Report on captured output. - :type path: str - :type capture: Capture - :type messages: set[str] - """ - if capture.stdout.getvalue(): - first = capture.stdout.getvalue().strip().splitlines()[0].strip() - message = '%s:%d:%d: %s: %s' % (path, 0, 0, 'StandardOutputUsed', first) - report_message(message, messages) - - if capture.stderr.getvalue(): - first = capture.stderr.getvalue().strip().splitlines()[0].strip() - message = '%s:%d:%d: %s: %s' % (path, 0, 0, 'StandardErrorUsed', first) - report_message(message, messages) - - for warning in capture.warnings: - msg = re.sub(r'\s+', ' ', '%s' % warning.message).strip() - - filepath = os.path.relpath(warning.filename) - lineno = warning.lineno - - import_dir = 'test/runner/.tox/import/' - minimal_dir = 'test/runner/.tox/minimal-' - - if filepath.startswith('../') or filepath.startswith(minimal_dir): - # The warning occurred outside our source tree. - # The best we can do is to report the file which was tested that triggered the warning. - # If the responsible import is in shared code this warning will be repeated for each file tested which imports the shared code. - msg += ' (in %s:%d)' % (warning.filename, warning.lineno) - filepath = path - lineno = 0 - elif filepath.startswith(import_dir): - # Strip the import dir from warning paths in shared code. - # Needed when warnings occur in places like module_utils but are caught by the modules importing the module_utils. - filepath = os.path.relpath(filepath, import_dir) - - message = '%s:%d:%d: %s: %s' % (filepath, lineno, 0, warning.category.__name__, msg) - report_message(message, messages) - - -def report_message(message, messages): - """Report message if not already reported. - :type message: str - :type messages: set[str] - """ - if message not in messages: - messages.add(message) - print(message) - - -@contextlib.contextmanager -def capture_output(capture): - """Capture sys.stdout and sys.stderr. - :type capture: Capture - """ - old_stdout = sys.stdout - old_stderr = sys.stderr - - sys.stdout = capture.stdout - sys.stderr = capture.stderr - - with warnings.catch_warnings(record=True) as captured_warnings: try: - yield - finally: - capture.warnings = captured_warnings + if imp: + with open(path, 'r') as module_fd: + with capture_output(capture): + imp.load_module(name, module_fd, os.path.abspath(path), ('.py', 'r', imp.PY_SOURCE)) + else: + spec = importlib.util.spec_from_file_location(name, os.path.abspath(path)) + module = importlib.util.module_from_spec(spec) - sys.stdout = old_stdout - sys.stderr = old_stderr + with capture_output(capture): + spec.loader.exec_module(module) + + capture_report(path, capture, messages) + except ImporterAnsibleModuleException: + # module instantiated AnsibleModule without raising an exception + pass + # We truly want to catch anything the plugin might do here, including call sys.exit() so we + # catch BaseException + except BaseException as ex: # pylint: disable=locally-disabled, broad-except + capture_report(path, capture, messages) + + exc_type, _exc, exc_tb = sys.exc_info() + message = str(ex) + results = list(reversed(traceback.extract_tb(exc_tb))) + source = None + line = 0 + offset = 0 + + if isinstance(ex, SyntaxError) and ex.filename.endswith(path): # pylint: disable=locally-disabled, no-member + # A SyntaxError in the source we're importing will have the correct path, line and offset. + # However, the traceback will report the path to this importer.py script instead. + # We'll use the details from the SyntaxError in this case, as it's more accurate. + source = path + line = ex.lineno or 0 # pylint: disable=locally-disabled, no-member + offset = ex.offset or 0 # pylint: disable=locally-disabled, no-member + message = str(ex) + + # Hack to remove the filename and line number from the message, if present. + message = message.replace(' (%s, line %d)' % (os.path.basename(path), line), '') + else: + for result in results: + if result[0].startswith(filter_dir): + source = result[0][len(base_dir) + 1:].replace('test/sanity/import/', '') + line = result[1] or 0 + break + + if not source: + # If none of our source files are found in the traceback, report the file we were testing. + # I haven't been able to come up with a test case that encounters this issue yet. + source = path + message += ' (in %s:%d)' % (results[-1][0], results[-1][1] or 0) + + message = re.sub(r'\n *', ': ', message) + error = '%s:%d:%d: %s: %s' % (source, line, offset, exc_type.__name__, message) + + report_message(error, messages) + + class Capture: + """Captured output and/or exception.""" + def __init__(self): + self.stdout = StringIO() + self.stderr = StringIO() + self.warnings = [] + + def capture_report(path, capture, messages): + """Report on captured output. + :type path: str + :type capture: Capture + :type messages: set[str] + """ + if capture.stdout.getvalue(): + first = capture.stdout.getvalue().strip().splitlines()[0].strip() + message = '%s:%d:%d: %s: %s' % (path, 0, 0, 'StandardOutputUsed', first) + report_message(message, messages) + + if capture.stderr.getvalue(): + first = capture.stderr.getvalue().strip().splitlines()[0].strip() + message = '%s:%d:%d: %s: %s' % (path, 0, 0, 'StandardErrorUsed', first) + report_message(message, messages) + + for warning in capture.warnings: + msg = re.sub(r'\s+', ' ', '%s' % warning.message).strip() + + filepath = os.path.relpath(warning.filename) + lineno = warning.lineno + + import_dir = 'test/runner/.tox/import/' + minimal_dir = 'test/runner/.tox/minimal-' + + if filepath.startswith('../') or filepath.startswith(minimal_dir): + # The warning occurred outside our source tree. + # The best we can do is to report the file which was tested that triggered the warning. + # If the responsible import is in shared code this warning will be repeated for each file tested which imports the shared code. + msg += ' (in %s:%d)' % (warning.filename, warning.lineno) + filepath = path + lineno = 0 + elif filepath.startswith(import_dir): + # Strip the import dir from warning paths in shared code. + # Needed when warnings occur in places like module_utils but are caught by the modules importing the module_utils. + filepath = os.path.relpath(filepath, import_dir) + + message = '%s:%d:%d: %s: %s' % (filepath, lineno, 0, warning.category.__name__, msg) + report_message(message, messages) + + def report_message(message, messages): + """Report message if not already reported. + :type message: str + :type messages: set[str] + """ + if message not in messages: + messages.add(message) + print(message) + + @contextlib.contextmanager + def capture_output(capture): + """Capture sys.stdout and sys.stderr. + :type capture: Capture + """ + old_stdout = sys.stdout + old_stderr = sys.stderr + + sys.stdout = capture.stdout + sys.stderr = capture.stderr + + with warnings.catch_warnings(record=True) as captured_warnings: + try: + yield + finally: + capture.warnings = captured_warnings + + sys.stdout = old_stdout + sys.stderr = old_stderr + + run() if __name__ == '__main__':