Pylint deprecated comment checker (#81071)

Co-authored-by: Matt Clay <matt@mystile.com>

changelogs/fragments/pylint-deprecated-comment-checker.yml

@@ -0,0 +1,3 @@
+minor_changes:
+- ansible-test - Add new pylint checker for new ``# deprecated:`` comments within code to trigger errors when time to remove code
+  that has no user facing deprecation message

test/lib/ansible_test/_internal/commands/sanity/pylint.py

@@ -18,6 +18,11 @@ from . import (
     SANITY_ROOT,
 )
 
+from ...constants import (
+    CONTROLLER_PYTHON_VERSIONS,
+    REMOTE_ONLY_PYTHON_VERSIONS,
+)
+
 from ...io import (
     make_dirs,
 )
@@ -38,6 +43,7 @@ from ...util import (
 
 from ...util_common import (
     run_command,
+    process_scoped_temporary_file,
 )
 
 from ...ansible_util import (
@@ -81,6 +87,8 @@ class PylintTest(SanitySingleVersion):
         return [target for target in targets if os.path.splitext(target.path)[1] == '.py' or is_subdir(target.path, 'bin')]
 
     def test(self, args: SanityConfig, targets: SanityTargets, python: PythonConfig) -> TestResult:
+        min_python_version_db_path = self.create_min_python_db(args, targets.targets)
+
         plugin_dir = os.path.join(SANITY_ROOT, 'pylint', 'plugins')
         plugin_names = sorted(p[0] for p in [
             os.path.splitext(p) for p in os.listdir(plugin_dir)] if p[1] == '.py' and p[0] != '__init__')
@@ -163,7 +171,7 @@ class PylintTest(SanitySingleVersion):
                 continue
 
             context_start = datetime.datetime.now(tz=datetime.timezone.utc)
-            messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names, python, collection_detail)
+            messages += self.pylint(args, context, context_paths, plugin_dir, plugin_names, python, collection_detail, min_python_version_db_path)
             context_end = datetime.datetime.now(tz=datetime.timezone.utc)
 
             context_times.append('%s: %d (%s)' % (context, len(context_paths), context_end - context_start))
@@ -194,6 +202,22 @@ class PylintTest(SanitySingleVersion):
 
         return SanitySuccess(self.name)
 
+    def create_min_python_db(self, args: SanityConfig, targets: t.Iterable[TestTarget]) -> str:
+        """Create a database of target file paths and their minimum required Python version, returning the path to the database."""
+        target_paths = set(target.path for target in self.filter_remote_targets(list(targets)))
+        controller_min_version = CONTROLLER_PYTHON_VERSIONS[0]
+        target_min_version = REMOTE_ONLY_PYTHON_VERSIONS[0]
+        min_python_versions = {
+            os.path.abspath(target.path): target_min_version if target.path in target_paths else controller_min_version for target in targets
+        }
+
+        min_python_version_db_path = process_scoped_temporary_file(args)
+
+        with open(min_python_version_db_path, 'w') as database_file:
+            json.dump(min_python_versions, database_file)
+
+        return min_python_version_db_path
+
     @staticmethod
     def pylint(
         args: SanityConfig,
@@ -203,6 +227,7 @@ class PylintTest(SanitySingleVersion):
         plugin_names: list[str],
         python: PythonConfig,
         collection_detail: CollectionDetail,
+        min_python_version_db_path: str,
     ) -> list[dict[str, str]]:
         """Run pylint using the config specified by the context on the specified paths."""
         rcfile = os.path.join(SANITY_ROOT, 'pylint', 'config', context.split('/')[0] + '.cfg')
@@ -234,6 +259,7 @@ class PylintTest(SanitySingleVersion):
             '--rcfile', rcfile,
             '--output-format', 'json',
             '--load-plugins', ','.join(sorted(load_plugins)),
+            '--min-python-version-db', min_python_version_db_path,
         ] + paths  # fmt: skip
 
         if data_context().content.collection:

test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py

@@ -5,13 +5,17 @@
 from __future__ import annotations
 
 import datetime
+import functools
+import json
 import re
+import shlex
 import typing as t
+from tokenize import COMMENT, TokenInfo
 
 import astroid
 
-from pylint.interfaces import IAstroidChecker
-from pylint.checkers import BaseChecker
+from pylint.interfaces import IAstroidChecker, ITokenChecker
+from pylint.checkers import BaseChecker, BaseTokenChecker
 from pylint.checkers.utils import check_messages
 
 from ansible.module_utils.compat.version import LooseVersion
@@ -277,6 +281,137 @@ class AnsibleDeprecatedChecker(BaseChecker):
             pass
 
 
+class AnsibleDeprecatedCommentChecker(BaseTokenChecker):
+    """Checks for ``# deprecated:`` comments to ensure that the ``version``
+    has not passed or met the time for removal
+    """
+
+    __implements__ = (ITokenChecker,)
+
+    name = 'deprecated-comment'
+    msgs = {
+        'E9601': ("Deprecated core version (%r) found: %s",
+                  "ansible-deprecated-version-comment",
+                  "Used when a '# deprecated:' comment specifies a version "
+                  "less than or equal to the current version of Ansible",
+                  {'minversion': (2, 6)}),
+        'E9602': ("Deprecated comment contains invalid keys %r",
+                  "ansible-deprecated-version-comment-invalid-key",
+                  "Used when a '#deprecated:' comment specifies invalid data",
+                  {'minversion': (2, 6)}),
+        'E9603': ("Deprecated comment missing version",
+                  "ansible-deprecated-version-comment-missing-version",
+                  "Used when a '#deprecated:' comment specifies invalid data",
+                  {'minversion': (2, 6)}),
+        'E9604': ("Deprecated python version (%r) found: %s",
+                  "ansible-deprecated-python-version-comment",
+                  "Used when a '#deprecated:' comment specifies a python version "
+                  "less than or equal to the minimum python version",
+                  {'minversion': (2, 6)}),
+        'E9605': ("Deprecated comment contains invalid version %r: %s",
+                  "ansible-deprecated-version-comment-invalid-version",
+                  "Used when a '#deprecated:' comment specifies an invalid version",
+                  {'minversion': (2, 6)}),
+    }
+
+    options = (
+        ('min-python-version-db', {
+            'default': None,
+            'type': 'string',
+            'metavar': '<path>',
+            'help': 'The path to the DB mapping paths to minimum Python versions.',
+        }),
+    )
+
+    def process_tokens(self, tokens: list[TokenInfo]) -> None:
+        for token in tokens:
+            if token.type == COMMENT:
+                self._process_comment(token)
+
+    def _deprecated_string_to_dict(self, token: TokenInfo, string: str) -> dict[str, str]:
+        valid_keys = {'description', 'core_version', 'python_version'}
+        data = dict.fromkeys(valid_keys)
+        for opt in shlex.split(string):
+            if '=' not in opt:
+                data[opt] = None
+                continue
+            key, _sep, value = opt.partition('=')
+            data[key] = value
+        if not any((data['core_version'], data['python_version'])):
+            self.add_message(
+                'ansible-deprecated-version-comment-missing-version',
+                line=token.start[0],
+                col_offset=token.start[1],
+            )
+        bad = set(data).difference(valid_keys)
+        if bad:
+            self.add_message(
+                'ansible-deprecated-version-comment-invalid-key',
+                line=token.start[0],
+                col_offset=token.start[1],
+                args=(','.join(bad),)
+            )
+        return data
+
+    @functools.cached_property
+    def _min_python_version_db(self) -> dict[str, str]:
+        """A dictionary of absolute file paths and their minimum required Python version."""
+        with open(self.linter.config.min_python_version_db) as db_file:
+            return json.load(db_file)
+
+    def _process_python_version(self, token: TokenInfo, data: dict[str, str]) -> None:
+        current_file = self.linter.current_file
+        check_version = self._min_python_version_db[current_file]
+
+        try:
+            if LooseVersion(data['python_version']) < LooseVersion(check_version):
+                self.add_message(
+                    'ansible-deprecated-python-version-comment',
+                    line=token.start[0],
+                    col_offset=token.start[1],
+                    args=(
+                        data['python_version'],
+                        data['description'] or 'description not provided',
+                    ),
+                )
+        except (ValueError, TypeError) as exc:
+            self.add_message(
+                'ansible-deprecated-version-comment-invalid-version',
+                line=token.start[0],
+                col_offset=token.start[1],
+                args=(data['python_version'], exc)
+            )
+
+    def _process_core_version(self, token: TokenInfo, data: dict[str, str]) -> None:
+        try:
+            if ANSIBLE_VERSION >= LooseVersion(data['core_version']):
+                self.add_message(
+                    'ansible-deprecated-version-comment',
+                    line=token.start[0],
+                    col_offset=token.start[1],
+                    args=(
+                        data['core_version'],
+                        data['description'] or 'description not provided',
+                    )
+                )
+        except (ValueError, TypeError) as exc:
+            self.add_message(
+                'ansible-deprecated-version-comment-invalid-version',
+                line=token.start[0],
+                col_offset=token.start[1],
+                args=(data['core_version'], exc)
+            )
+
+    def _process_comment(self, token: TokenInfo) -> None:
+        if token.string.startswith('# deprecated:'):
+            data = self._deprecated_string_to_dict(token, token.string[13:].strip())
+            if data['core_version']:
+                self._process_core_version(token, data)
+            if data['python_version']:
+                self._process_python_version(token, data)
+
+
 def register(linter):
     """required method to auto register this checker """
     linter.register_checker(AnsibleDeprecatedChecker(linter))
+    linter.register_checker(AnsibleDeprecatedCommentChecker(linter))