archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update ansible-test Python version handling.

Browse Source 
Minimum version requirements for sanity tests have been standardized:

- All single version sanity tests now require Python 3.5 or later.
- All multiple version sanity tests continue to use all supported Python versions.
- All version neutral sanity tests continue to work on any supported Python version.

Previously some tests required 3.5 or later with most of the remaining tests requiring 2.7 or later.

When using the `--python` option to specify a Python version:

- Tests which do not support the specified Python version will be skipped with a warning.
- If the specified Python version is not available, any test attempting to use it will generate an error.

When not using the `--python` option to specify a Python version:

- Multiple version tests will attempt to run on all supported versions.
- Single version tests will use the current version if supported and available, or if no supported version is available.
- Single version tests will use the lowest available and supported version if the current version is not supported.
- Any versions which are not available or supported will be skipped with a warning.

Unit tests automatically skip unavailable Python versions unless `--python` was used to specify a version.
pull/59742/head
Matt Clay 5 years ago
parent ecddbdf0cb
commit 66654475e1
14 changed files with 142 additions and 91 deletions
Show Stats Download Patch File Download Diff File

7
test/runner/lib/executor.py
Unescape Escape View File

@ -59,6 +59,7 @@ from lib.util import (
COVERAGE_OUTPUT_PATH,
cmd_quote,
ANSIBLE_ROOT,
get_available_python_versions,
)
from lib.util_common import (
@ -1331,11 +1332,17 @@ def command_units(args):
version_commands = []
available_versions = get_available_python_versions(SUPPORTED_PYTHON_VERSIONS)
for version in SUPPORTED_PYTHON_VERSIONS:
# run all versions unless version given, in which case run only that version
if args.python and version != args.python_version:
continue
if not args.python and version not in available_versions:
display.warning("Skipping unit tests on Python %s due to missing interpreter." % version)
continue
if args.requirements_mode != 'skip':
install_command_requirements(args, version)

112
test/runner/lib/sanity/__init__.py
Unescape Escape View File

@ -22,6 +22,8 @@ from lib.util import (
ANSIBLE_ROOT,
is_binary_file,
read_lines_without_comments,
get_available_python_versions,
find_python,
)
from lib.util_common import (
@ -108,30 +110,51 @@ def command_sanity(args):
display.info(test.name)
continue
if isinstance(test, SanityMultipleVersion):
versions = SUPPORTED_PYTHON_VERSIONS
available_versions = get_available_python_versions(SUPPORTED_PYTHON_VERSIONS)
if args.python:
# specific version selected
versions = (args.python,)
elif isinstance(test, SanityMultipleVersion):
# try all supported versions for multi-version tests when a specific version has not been selected
versions = test.supported_python_versions
elif not test.supported_python_versions or args.python_version in test.supported_python_versions:
# the test works with any version or the version we're already running
versions = (args.python_version,)
else:
versions = (None,)
# available versions supported by the test
versions = tuple(sorted(set(available_versions) & set(test.supported_python_versions)))
# use the lowest available version supported by the test or the current version as a fallback (which will be skipped)
versions = versions[:1] or (args.python_version,)
for version in versions:
if args.python and version and version != args.python_version:
continue
check_pyyaml(args, version or args.python_version)
display.info('Sanity check using %s%s' % (test.name, ' with Python %s' % version if version else ''))
options = ''
if isinstance(test, SanityCodeSmellTest):
result = test.test(args, targets)
elif isinstance(test, SanityMultipleVersion):
result = test.test(args, targets, python_version=version)
options = ' --python %s' % version
elif isinstance(test, SanitySingleVersion):
result = test.test(args, targets)
if test.supported_python_versions and version not in test.supported_python_versions:
display.warning("Skipping sanity test '%s' on unsupported Python %s." % (test.name, version))
result = SanitySkipped(test.name)
elif not args.python and version not in available_versions:
display.warning("Skipping sanity test '%s' on Python %s due to missing interpreter." % (test.name, version))
result = SanitySkipped(test.name)
else:
raise Exception('Unsupported test type: %s' % type(test))
check_pyyaml(args, version)
if test.supported_python_versions:
display.info("Running sanity test '%s' with Python %s" % (test.name, version))
else:
display.info("Running sanity test '%s'" % test.name)
if isinstance(test, SanityCodeSmellTest):
result = test.test(args, targets, version)
elif isinstance(test, SanityMultipleVersion):
result = test.test(args, targets, version)
options = ' --python %s' % version
elif isinstance(test, SanitySingleVersion):
result = test.test(args, targets, version)
elif isinstance(test, SanityVersionNeutral):
result = test.test(args, targets)
else:
raise Exception('Unsupported test type: %s' % type(test))
result.write(args)
@ -162,7 +185,7 @@ def collect_code_smell_tests():
if not data_context().content.is_ansible:
skip_tests += read_lines_without_comments(ansible_only_file, remove_blank_lines=True)
paths = glob.glob(os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/*'))
paths = glob.glob(os.path.join(ANSIBLE_ROOT, 'test/sanity/code-smell/*.py'))
paths = sorted(p for p in paths if os.access(p, os.X_OK) and os.path.isfile(p) and os.path.basename(p) not in skip_tests)
tests = tuple(SanityCodeSmellTest(p) for p in paths)
@ -210,7 +233,7 @@ class SanityIgnoreParser:
for test in sanity_get_tests():
if isinstance(test, SanityMultipleVersion):
versioned_test_names.add(test.name)
tests_by_name.update(dict(('%s-%s' % (test.name, python_version), test) for python_version in SUPPORTED_PYTHON_VERSIONS))
tests_by_name.update(dict(('%s-%s' % (test.name, python_version), test) for python_version in test.supported_python_versions))
else:
unversioned_test_names.update(dict(('%s-%s' % (test.name, python_version), test.name) for python_version in SUPPORTED_PYTHON_VERSIONS))
tests_by_name[test.name] = test
@ -503,13 +526,14 @@ class SanityTest(ABC):
"""True if the test supports skip entries."""
return True
@property
def supported_python_versions(self): # type: () -> t.Optional[t.Tuple[str, ...]]
"""A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
return tuple(python_version for python_version in SUPPORTED_PYTHON_VERSIONS if python_version.startswith('3.'))
class SanityCodeSmellTest(SanityTest):
"""Sanity test script."""
UNSUPPORTED_PYTHON_VERSIONS = (
'2.6', # some tests use voluptuous, but the version we require does not support python 2.6
)
def __init__(self, path):
name = os.path.splitext(os.path.basename(path))[0]
config_path = os.path.splitext(path)[0] + '.json'
@ -527,20 +551,14 @@ class SanityCodeSmellTest(SanityTest):
if self.config:
self.enabled = not self.config.get('disabled')
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
if args.python_version in self.UNSUPPORTED_PYTHON_VERSIONS:
display.warning('Skipping %s on unsupported Python version %s.' % (self.name, args.python_version))
return SanitySkipped(self.name)
if self.path.endswith('.py'):
cmd = [args.python_executable, self.path]
else:
cmd = [self.path]
cmd = [find_python(python_version), self.path]
env = ansible_environment(args, color=False)
@ -653,13 +671,34 @@ class SanityFunc(SanityTest):
super(SanityFunc, self).__init__(name)
class SanityVersionNeutral(SanityFunc):
"""Base class for sanity test plugins which are idependent of the python version being used."""
@abc.abstractmethod
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
def load_processor(self, args): # type: (SanityConfig) -> SanityIgnoreProcessor
"""Load the ignore processor for this sanity test."""
return SanityIgnoreProcessor(args, self.name, self.error_code, None)
@property
def supported_python_versions(self): # type: () -> t.Optional[t.Tuple[str, ...]]
"""A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
return None
class SanitySingleVersion(SanityFunc):
"""Base class for sanity test plugins which should run on a single python version."""
@abc.abstractmethod
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
@ -683,6 +722,11 @@ class SanityMultipleVersion(SanityFunc):
"""Load the ignore processor for this sanity test."""
return SanityIgnoreProcessor(args, self.name, self.error_code, python_version)
@property
def supported_python_versions(self): # type: () -> t.Optional[t.Tuple[str, ...]]
"""A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
return SUPPORTED_PYTHON_VERSIONS
SANITY_TESTS = (
)

5
test/runner/lib/sanity/ansible_doc.py
Unescape Escape View File

@ -42,10 +42,11 @@ from lib.coverage_util import (
class AnsibleDocTest(SanitySingleVersion):
"""Sanity test for ansible-doc."""
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
settings = self.load_processor(args)
@ -99,7 +100,7 @@ class AnsibleDocTest(SanitySingleVersion):
try:
with coverage_context(args):
stdout, stderr = intercept_command(args, cmd, target_name='ansible-doc', env=env, capture=True)
stdout, stderr = intercept_command(args, cmd, target_name='ansible-doc', env=env, capture=True, python_version=python_version)
status = 0
except SubprocessError as ex:

4
test/runner/lib/sanity/ignores.py
Unescape Escape View File

@ -5,7 +5,7 @@ __metaclass__ = type
from lib.sanity import (
SanityFailure,
SanityIgnoreParser,
SanitySingleVersion,
SanityVersionNeutral,
SanitySuccess,
SanityMessage,
)
@ -20,7 +20,7 @@ from lib.config import (
)
class IgnoresTest(SanitySingleVersion):
class IgnoresTest(SanityVersionNeutral):
"""Sanity test for sanity test ignore entries."""
@property
def can_ignore(self): # type: () -> bool

4
test/runner/lib/sanity/integration_aliases.py
Unescape Escape View File

@ -10,7 +10,7 @@ import os
import lib.types as t
from lib.sanity import (
SanitySingleVersion,
SanityVersionNeutral,
SanityMessage,
SanityFailure,
SanitySuccess,
@ -38,7 +38,7 @@ from lib.util import (
)
class IntegrationAliasesTest(SanitySingleVersion):
class IntegrationAliasesTest(SanityVersionNeutral):
"""Sanity test to evaluate integration test aliases."""
SHIPPABLE_YML = 'shippable.yml'

6
test/runner/lib/sanity/pep8.py
Unescape Escape View File

@ -18,6 +18,7 @@ from lib.util import (
read_lines_without_comments,
parse_to_list_of_dict,
ANSIBLE_ROOT,
find_python,
)
from lib.util_common import (
@ -36,10 +37,11 @@ class Pep8Test(SanitySingleVersion):
"""Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes."""
return 'A100'
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
current_ignore_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/pep8/current-ignore.txt')
@ -51,7 +53,7 @@ class Pep8Test(SanitySingleVersion):
paths = settings.filter_skipped_paths(paths)
cmd = [
args.python_executable,
find_python(python_version),
'-m', 'pycodestyle',
'--max-line-length', '160',
'--config', '/dev/null',

4
test/runner/lib/sanity/pslint.py
Unescape Escape View File

@ -9,7 +9,7 @@ import re
import lib.types as t
from lib.sanity import (
SanitySingleVersion,
SanityVersionNeutral,
SanityMessage,
SanityFailure,
SanitySuccess,
@ -35,7 +35,7 @@ from lib.data import (
)
class PslintTest(SanitySingleVersion):
class PslintTest(SanityVersionNeutral):
"""Sanity test using PSScriptAnalyzer."""
@property
def error_code(self): # type: () -> t.Optional[str]

30
test/runner/lib/sanity/pylint.py
Unescape Escape View File

@ -14,7 +14,6 @@ from lib.sanity import (
SanityMessage,
SanityFailure,
SanitySuccess,
SanitySkipped,
)
from lib.util import (
@ -23,6 +22,7 @@ from lib.util import (
ConfigParser,
ANSIBLE_ROOT,
is_subdir,
find_python,
)
from lib.util_common import (
@ -42,12 +42,6 @@ from lib.data import (
)
UNSUPPORTED_PYTHON_VERSIONS = (
'2.6',
'2.7',
)
class PylintTest(SanitySingleVersion):
"""Sanity test using pylint."""
@property
@ -55,16 +49,13 @@ class PylintTest(SanitySingleVersion):
"""Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes."""
return 'ansible-test'
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
display.warning('Skipping pylint on unsupported Python version %s.' % args.python_version)
return SanitySkipped(self.name)
plugin_dir = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/plugins')
plugin_names = sorted(p[0] for p in [
os.path.splitext(p) for p in os.listdir(plugin_dir)] if p[1] == '.py' and p[0] != '__init__')
@ -137,6 +128,8 @@ class PylintTest(SanitySingleVersion):
messages = []
context_times = []
python = find_python(python_version)
test_start = datetime.datetime.utcnow()
for context, context_paths in sorted(contexts):
@ -144,7 +137,7 @@ class PylintTest(SanitySingleVersion):
continue
context_start = datetime.datetime.utcnow()
messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names)
messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names, python)
context_end = datetime.datetime.utcnow()
context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start))
@ -176,7 +169,14 @@ class PylintTest(SanitySingleVersion):
return SanitySuccess(self.name)
@staticmethod
def pylint(args, context, paths, plugin_dir, plugin_names): # type: (SanityConfig, str, t.List[str], str, t.List[str]) -> t.List[t.Dict[str, str]]
def pylint(
args, # type: SanityConfig
context, # type: str
paths, # type: t.List[str]
plugin_dir, # type: str
plugin_names, # type: t.List[str]
python, # type: str
): # type: (...) -> t.List[t.Dict[str, str]]
"""Run pylint using the config specified by the context on the specified paths."""
rcfile = os.path.join(ANSIBLE_ROOT, 'test/sanity/pylint/config/%s' % context.split('/')[0])
@ -198,7 +198,7 @@ class PylintTest(SanitySingleVersion):
load_plugins = set(plugin_names) - disable_plugins
cmd = [
args.python_executable,
python,
'-m', 'pylint',
'--jobs', '0',
'--reports', 'n',

15
test/runner/lib/sanity/rstcheck.py
Unescape Escape View File

@ -15,9 +15,9 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
parse_to_list_of_dict,
display,
read_lines_without_comments,
ANSIBLE_ROOT,
find_python,
)
from lib.util_common import (
@ -28,23 +28,16 @@ from lib.config import (
SanityConfig,
)
UNSUPPORTED_PYTHON_VERSIONS = (
'2.6',
)
class RstcheckTest(SanitySingleVersion):
"""Sanity test using rstcheck."""
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version)
return SanitySkipped(self.name)
ignore_file = os.path.join(ANSIBLE_ROOT, 'test/sanity/rstcheck/ignore-substitutions.txt')
ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True)))
@ -57,7 +50,7 @@ class RstcheckTest(SanitySingleVersion):
return SanitySkipped(self.name)
cmd = [
args.python_executable,
find_python(python_version),
'-m', 'rstcheck',
'--report', 'warning',
'--ignore-substitutions', ','.join(ignore_substitutions),

4
test/runner/lib/sanity/sanity_docs.py
Unescape Escape View File

@ -5,7 +5,7 @@ __metaclass__ = type
import os
from lib.sanity import (
SanitySingleVersion,
SanityVersionNeutral,
SanityMessage,
SanityFailure,
SanitySuccess,
@ -21,7 +21,7 @@ from lib.data import (
)
class SanityDocsTest(SanitySingleVersion):
class SanityDocsTest(SanityVersionNeutral):
"""Sanity test for documentation of sanity tests."""
ansible_only = True

4
test/runner/lib/sanity/shellcheck.py
Unescape Escape View File

@ -12,7 +12,7 @@ from xml.etree.ElementTree import (
import lib.types as t
from lib.sanity import (
SanitySingleVersion,
SanityVersionNeutral,
SanityMessage,
SanityFailure,
SanitySuccess,
@ -34,7 +34,7 @@ from lib.config import (
)
class ShellcheckTest(SanitySingleVersion):
class ShellcheckTest(SanityVersionNeutral):
"""Sanity test using shellcheck."""
@property
def error_code(self): # type: () -> t.Optional[str]

15
test/runner/lib/sanity/validate_modules.py
Unescape Escape View File

@ -19,6 +19,7 @@ from lib.util import (
SubprocessError,
display,
ANSIBLE_ROOT,
find_python,
)
from lib.util_common import (
@ -37,11 +38,6 @@ from lib.data import (
data_context,
)
UNSUPPORTED_PYTHON_VERSIONS = (
'2.6',
'2.7',
)
class ValidateModulesTest(SanitySingleVersion):
"""Sanity test using validate-modules."""
@ -50,16 +46,13 @@ class ValidateModulesTest(SanitySingleVersion):
"""Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes."""
return 'A100'
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
display.warning('Skipping validate-modules on unsupported Python version %s.' % args.python_version)
return SanitySkipped(self.name)
if data_context().content.is_ansible:
ignore_codes = ()
else:
@ -78,7 +71,7 @@ class ValidateModulesTest(SanitySingleVersion):
return SanitySkipped(self.name)
cmd = [
args.python_executable,
find_python(python_version),
os.path.join(ANSIBLE_ROOT, 'test/sanity/validate-modules/validate-modules'),
'--format', 'json',
'--arg-spec',

13
test/runner/lib/sanity/yamllint.py
Unescape Escape View File

@ -20,6 +20,7 @@ from lib.util import (
display,
ANSIBLE_ROOT,
is_subdir,
find_python,
)
from lib.util_common import (
@ -42,10 +43,11 @@ class YamllintTest(SanitySingleVersion):
"""Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes."""
return 'ansible-test'
def test(self, args, targets):
def test(self, args, targets, python_version):
"""
:type args: SanityConfig
:type targets: SanityTargets
:type python_version: str
:rtype: TestResult
"""
settings = self.load_processor(args)
@ -66,7 +68,9 @@ class YamllintTest(SanitySingleVersion):
if not paths:
return SanitySkipped(self.name)
results = self.test_paths(args, paths)
python = find_python(python_version)
results = self.test_paths(args, paths, python)
results = settings.process_errors(results, paths)
if results:
@ -75,14 +79,15 @@ class YamllintTest(SanitySingleVersion):
return SanitySuccess(self.name)
@staticmethod
def test_paths(args, paths):
def test_paths(args, paths, python):
"""
:type args: SanityConfig
:type paths: list[str]
:type python: str
:rtype: list[SanityMessage]
"""
cmd = [
args.python_executable,
python,
os.path.join(ANSIBLE_ROOT, 'test/sanity/yamllint/yamllinter.py'),
]

10
test/runner/lib/util.py
Unescape Escape View File

@ -246,10 +246,11 @@ def find_executable(executable, cwd=None, path=None, required=True):
return match
def find_python(version, path=None):
def find_python(version, path=None, required=True):
"""
:type version: str
:type path: str | None
:type required: bool
:rtype: str
"""
version_info = tuple(int(n) for n in version.split('.'))
@ -257,11 +258,16 @@ def find_python(version, path=None):
if not path and version_info == sys.version_info[:len(version_info)]:
python_bin = sys.executable
else:
python_bin = find_executable('python%s' % version, path=path)
python_bin = find_executable('python%s' % version, path=path, required=required)
return python_bin
def get_available_python_versions(versions): # type: (t.List[str]) -> t.Tuple[str, ...]
"""Return a tuple indicating which of the requested Python versions are available."""
return tuple(python_version for python_version in versions if find_python(python_version, required=False))
def generate_pip_command(python):
"""
:type python: str

Write Preview
Loading…
Cancel
Save