Upgrade pylint to version 2.1.1. (#47036)

* Fix issues reported by the latest pylint.
* Split pylint runs into more contexts.
* Upgrade pylint.

test/runner/lib/core_ci.py

@@ -117,7 +117,7 @@ class AnsibleCoreCI(object):
                 region = 'us-east-1'
 
             self.path = "%s-%s" % (self.path, region)
-            self.endpoints = AWS_ENDPOINTS[region],
+            self.endpoints = (AWS_ENDPOINTS[region],)
             self.ssh_key = SshKey(args)
 
             if self.platform == 'windows':
@@ -192,7 +192,7 @@ class AnsibleCoreCI(object):
         if self.started:
             display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id),
                          verbosity=1)
-            return
+            return None
 
         if is_shippable():
             return self.start_shippable()

test/runner/lib/delegation.py

@@ -117,7 +117,7 @@ def delegate_tox(args, exclude, require, integration_targets):
     :type integration_targets: tuple[IntegrationTarget]
     """
     if args.python:
-        versions = args.python_version,
+        versions = (args.python_version,)
 
         if args.python_version not in SUPPORTED_PYTHON_VERSIONS:
             raise ApplicationError('tox does not support Python version %s' % args.python_version)

test/runner/lib/sanity/pylint.py

@@ -44,6 +44,7 @@ PYLINT_IGNORE_PATH = 'test/sanity/pylint/ignore.txt'
 
 UNSUPPORTED_PYTHON_VERSIONS = (
     '2.6',
+    '2.7',
 )
 
 
@@ -99,7 +100,7 @@ class PylintTest(SanitySingleVersion):
                     invalid_ignores.append((line, 'Invalid version: %s' % version))
                     continue
 
-                if version != args.python_version and version != args.python_version.split('.')[0]:
+                if version not in (args.python_version, args.python_version.split('.')[0]):
                     continue  # ignore version specific entries for other versions
 
             ignore[path][code] = line
@@ -108,7 +109,10 @@ class PylintTest(SanitySingleVersion):
 
         paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set)
 
-        contexts = {}
+        module_paths = [p.split(os.path.sep) for p in paths if p.startswith('lib/ansible/modules/')]
+        module_dirs = sorted(set([p[3] for p in module_paths if len(p) > 4]))
+
+        contexts = []
         remaining_paths = set(paths)
 
         def add_context(available_paths, context_name, context_filter):
@@ -118,15 +122,33 @@ class PylintTest(SanitySingleVersion):
             :type context_filter: (str) -> bool
             """
             filtered_paths = set(p for p in available_paths if context_filter(p))
-            contexts[context_name] = sorted(filtered_paths)
+            contexts.append((context_name, sorted(filtered_paths)))
             available_paths -= filtered_paths
 
-        add_context(remaining_paths, 'ansible-test', lambda p: p.startswith('test/runner/'))
+        def filter_path(path_filter=None):
-        add_context(remaining_paths, 'units', lambda p: p.startswith('test/units/'))
+            """
-        add_context(remaining_paths, 'test', lambda p: p.startswith('test/'))
+            :type path_filter: str
-        add_context(remaining_paths, 'hacking', lambda p: p.startswith('hacking/'))
+            :rtype: (str) -> bool
-        add_context(remaining_paths, 'modules', lambda p: p.startswith('lib/ansible/modules/'))
+            """
-        add_context(remaining_paths, 'module_utils', lambda p: p.startswith('lib/ansible/module_utils/'))
+            def context_filter(path_to_filter):
+                """
+                :type path_to_filter: str
+                :rtype: bool
+                """
+                return path_to_filter.startswith(path_filter)
+
+            return context_filter
+
+        add_context(remaining_paths, 'ansible-test', filter_path('test/runner/'))
+        add_context(remaining_paths, 'units', filter_path('test/units/'))
+        add_context(remaining_paths, 'test', filter_path('test/'))
+        add_context(remaining_paths, 'hacking', filter_path('hacking/'))
+
+        for module_dir in module_dirs:
+            add_context(remaining_paths, 'modules/%s' % module_dir, filter_path('lib/ansible/modules/%s/' % module_dir))
+
+        add_context(remaining_paths, 'modules', filter_path('lib/ansible/modules/'))
+        add_context(remaining_paths, 'module_utils', filter_path('lib/ansible/module_utils/'))
         add_context(remaining_paths, 'ansible', lambda p: True)
 
         messages = []
@@ -134,9 +156,7 @@ class PylintTest(SanitySingleVersion):
 
         test_start = datetime.datetime.utcnow()
 
-        for context in sorted(contexts):
+        for context, context_paths in sorted(contexts):
-            context_paths = contexts[context]
-
             if not context_paths:
                 continue
 
@@ -235,7 +255,7 @@ class PylintTest(SanitySingleVersion):
         :type paths: list[str]
         :rtype: list[dict[str, str]]
         """
-        rcfile = 'test/sanity/pylint/config/%s' % context
+        rcfile = 'test/sanity/pylint/config/%s' % context.split('/')[0]
 
         if not os.path.exists(rcfile):
             rcfile = 'test/sanity/pylint/config/default'
@@ -266,6 +286,8 @@ class PylintTest(SanitySingleVersion):
         env['PYTHONPATH'] += '%s%s' % (os.path.pathsep, self.plugin_dir)
 
         if paths:
+            display.info('Checking %d file(s) in context "%s" with config: %s' % (len(paths), context, rcfile), verbosity=1)
+
             try:
                 stdout, stderr = run_command(args, cmd, env=env, capture=True)
                 status = 0

test/runner/lib/target.py

@@ -448,7 +448,7 @@ class TestTarget(CompletionTarget):
 
         if module_path and path.startswith(module_path) and name != '__init__' and ext in MODULE_EXTENSIONS:
             self.module = name[len(module_prefix or ''):].lstrip('_')
-            self.modules = self.module,
+            self.modules = (self.module,)
         else:
             self.module = None
             self.modules = tuple()

test/runner/requirements/constraints.txt

@@ -1,13 +1,9 @@
 coverage >= 4.2, != 4.3.2 # features in 4.2+ required, avoid known bug in 4.3.2 on python 2.6
 cryptography < 2.2 ; python_version < '2.7' # cryptography 2.2 drops support for python 2.6
 pywinrm >= 0.3.0 # message encryption support
-astroid == 1.5.3 ; python_version >= '3.5' # newer versions of astroid require newer versions of pylint to avoid bugs
-pylint == 1.7.4 ; python_version >= '3.5' # versions before 1.7.1 hang or fail to install on python 3.x
-pylint == 1.6.5 ; python_version <= '2.7' # versions after 1.6.5 hang or fail during test on python 2.x
 sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7 or later
 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
 yamllint != 1.8.0 ; python_version < '2.7' # yamllint 1.8.0 requires python 2.7+ while earlier/later versions do not
-isort < 4.2.8 # 4.2.8 changes import sort order requirements which breaks previously passing pylint tests
 pycrypto >= 2.6 # Need features found in 2.6 and greater
 ncclient >= 0.5.2 # Need features added in 0.5.2 and greater
 idna < 2.6 # requests requires idna < 2.6, but cryptography will cause the latest version to be installed instead
@@ -21,3 +17,13 @@ openshift >= 0.6.2 # merge_type support
 virtualenv < 16.0.0 ; python_version < '2.7' # virtualenv 16.0.0 and later require python 2.7 or later
 pyopenssl < 18.0.0 ; python_version < '2.7' # pyOpenSSL 18.0.0 and later require python 2.7 or later
 pyfmg == 0.6.1 # newer versions do not pass current unit tests
+
+# freeze pylint and its requirements for consistent test results
+astroid == 2.0.4
+isort == 4.3.4
+lazy-object-proxy == 1.3.1
+mccabe == 0.6.1
+pylint == 2.1.1
+six == 1.11.0
+typed-ast == 1.1.0
+wrapt == 1.10.11

test/runner/requirements/sanity.txt

@@ -3,7 +3,7 @@ jinja2
 mock
 paramiko
 pycodestyle
-pylint ; python_version >= '2.7' # pylint 1.5.3+ is required for non-buggy JSON output, but 1.4+ requires python 2.7+
+pylint ; python_version >= '3.5' # pylint 2.0.0 and later require python 3+
 pytest
 rstcheck ; python_version >= '2.7' # rstcheck requires python 2.7+
 sphinx

test/sanity/pylint/config/ansible-test

@@ -11,6 +11,9 @@ disable=
     too-many-return-statements,
     too-many-statements,
     unused-import,
+    useless-object-inheritance,
+    consider-using-dict-comprehension,
+    consider-using-set-comprehension,
 
 [BASIC]
 

test/sanity/pylint/config/default

@@ -1,11 +1,12 @@
 [MESSAGES CONTROL]
 
 disable=
-    ansible-deprecated-version,
     abstract-method,
     access-member-before-definition,
+    ansible-deprecated-version,
     arguments-differ,
     assignment-from-no-return,
+    assignment-from-none,
     attribute-defined-outside-init,
     bad-continuation,
     bad-indentation,
@@ -14,10 +15,18 @@ disable=
     bare-except,
     blacklisted-name,
     broad-except,
+    c-extension-no-member,
     cell-var-from-loop,
+    chained-comparison,
+    comparison-with-callable,
+    comparison-with-itself,
     consider-iterating-dictionary,
     consider-merging-isinstance,
+    consider-using-dict-comprehension,
     consider-using-enumerate,
+    consider-using-get,
+    consider-using-in,
+    consider-using-set-comprehension,
     consider-using-ternary,
     deprecated-lambda,
     deprecated-method,
@@ -33,13 +42,17 @@ disable=
     global-variable-undefined,
     import-error,
     import-self,
+    inconsistent-return-statements,
+    invalid-envvar-default,
     invalid-name,
     invalid-sequence-index,
     invalid-unary-operand-type,
+    keyword-arg-before-vararg,
     len-as-condition,
     line-too-long,
     literal-comparison,
     locally-disabled,
+    logging-not-lazy,
     method-hidden,
     misplaced-comparison-constant,
     missing-docstring,
@@ -56,7 +69,9 @@ disable=
     old-style-class,
     pointless-statement,
     pointless-string-statement,
+    possibly-unused-variable,
     protected-access,
+    raising-format-tuple,
     redefined-argument-from-local,
     redefined-builtin,
     redefined-outer-name,
@@ -65,6 +80,8 @@ disable=
     relative-import,
     signature-differs,
     simplifiable-if-statement,
+    stop-iteration-return,
+    subprocess-popen-preexec-fn,
     super-init-not-called,
     superfluous-parens,
     too-few-public-methods,
@@ -81,6 +98,8 @@ disable=
     too-many-return-statements,
     too-many-statements,
     trailing-comma-tuple,
+    trailing-comma-tuple,
+    try-except-raise,
     unbalanced-tuple-unpacking,
     undefined-loop-variable,
     unexpected-keyword-arg,
@@ -95,6 +114,9 @@ disable=
     unused-import,
     unused-variable,
     used-before-assignment,
+    useless-import-alias,
+    useless-object-inheritance,
+    useless-return,
     useless-super-delegation,
     wrong-import-order,
     wrong-import-position,