[2.9] Prevent Ansible 2.9 to choke on collections using deprecation by date or collection_name for deprecation calls (#69935)

* Prevent Ansible 2.9 to choke on collections using deprecation by date or collection_name for deprecation calls.

* Add changelog fragment.

* Fix YAML.

* Improve C# compatibility.

* Add tests for AnsibleModule.

* Fix var name.

* Fix type.

* Update C# code.

* Show deprecation warning if removed_at_date is used for Python modules.

* Update changelogs/fragments/69935-2.10-deprecation-support.yml

Co-authored-by: Alicia Cozine <879121+acozine@users.noreply.github.com>

* Prevent crash of validate-modules if 'removed_in' is not in 'deprecated'.

Co-authored-by: Alicia Cozine <879121+acozine@users.noreply.github.com>

changelogs/fragments/69935-2.10-deprecation-support.yml

@@ -0,0 +1,2 @@
+minor_changes:
+ - "``Display.deprecated()``, ``AnsibleModule.deprecate()`` and ``Ansible.Basic.Deprecate()`` now also accept the deprecation-by-date parameters and collection name parameters from Ansible 2.10, so plugins and modules in collections that conform to Ansible 2.10 will run with newer versions of Ansible 2.9."

lib/ansible/module_utils/basic.py

@@ -752,7 +752,10 @@ class AnsibleModule(object):
         else:
             raise TypeError("warn requires a string not a %s" % type(warning))
 
-    def deprecate(self, msg, version=None):
+    def deprecate(self, msg, version=None, date=None, collection_name=None):
+        # `date` and `collection_name` are Ansible 2.10 parameters. We accept and ignore them,
+        # to avoid modules/plugins from 2.10 conformant collections to break with new enough
+        # versions of Ansible 2.9.
         if isinstance(msg, string_types):
             self._deprecations.append({
                 'msg': msg,
@@ -1438,7 +1441,7 @@ class AnsibleModule(object):
             if deprecation['name'] in param.keys():
                 self._deprecations.append(
                     {'msg': "Alias '%s' is deprecated. See the module docs for more information" % deprecation['name'],
-                     'version': deprecation['version']})
+                     'version': deprecation.get('version')})
         return alias_results
 
     def _handle_no_log_values(self, spec=None, param=None):

lib/ansible/module_utils/common/parameters.py

@@ -134,7 +134,7 @@ def list_deprecations(argument_spec, params, prefix=''):
                 sub_prefix = '%s["%s"]' % (prefix, arg_name)
             else:
                 sub_prefix = arg_name
-            if arg_opts.get('removed_in_version') is not None:
+            if arg_opts.get('removed_in_version') is not None or arg_opts.get('removed_at_date') is not None:
                 deprecations.append({
                     'msg': "Param '%s' is deprecated. See the module docs for more information" % sub_prefix,
                     'version': arg_opts.get('removed_in_version')

lib/ansible/module_utils/csharp/Ansible.Basic.cs

@@ -83,6 +83,8 @@ namespace Ansible.Basic
             { "no_log", new List<object>() { false, typeof(bool) } },
             { "options", new List<object>() { typeof(Hashtable), typeof(Hashtable) } },
             { "removed_in_version", new List<object>() { null, typeof(string) } },
+            { "removed_at_date", new List<object>() { null, typeof(DateTime) } },  // Ansible 2.10 compatibility
+            { "removed_from_collection", new List<object>() { null, typeof(string) } },  // Ansible 2.10 compatibility
             { "required", new List<object>() { false, typeof(bool) } },
             { "required_by", new List<object>() { typeof(Hashtable), typeof(Hashtable) } },
             { "required_if", new List<object>() { typeof(List<List<object>>), null } },
@@ -244,10 +246,35 @@ namespace Ansible.Basic
 
         public void Deprecate(string message, string version)
         {
+            Deprecate(message, version, null);
+        }
+
+        public void Deprecate(string message, string version, string collectionName)
+        {
+            // `collectionName` is a Ansible 2.10 parameter. We accept and ignore it,
+            // to avoid modules/plugins from 2.10 conformant collections to break with
+            // new enough versions of Ansible 2.9.
             deprecations.Add(new Dictionary<string, string>() { { "msg", message }, { "version", version } });
             LogEvent(String.Format("[DEPRECATION WARNING] {0} {1}", message, version));
         }
 
+        public void Deprecate(string message, DateTime date)
+        {
+            // This function is only available for Ansible 2.10. We still accept and ignore it,
+            // to avoid modules/plugins from 2.10 conformant collections to break with new enough
+            // versions of Ansible 2.9.
+            Deprecate(message, date, null);
+        }
+
+        public void Deprecate(string message, DateTime date, string collectionName)
+        {
+            // This function is only available for Ansible 2.10. We still accept and ignore it,
+            // to avoid modules/plugins from 2.10 conformant collections to break with new enough
+            // versions of Ansible 2.9.
+            deprecations.Add(new Dictionary<string, string>() { { "msg", message }, { "version", null } });
+            LogEvent(String.Format("[DEPRECATION WARNING] {0} {1}", message, null));
+        }
+
         public void ExitJson()
         {
             WriteLine(GetFormattedResults(Result));
@@ -708,6 +735,10 @@ namespace Ansible.Basic
                 object removedInVersion = v["removed_in_version"];
                 if (removedInVersion != null && parameters.Contains(k))
                     Deprecate(String.Format("Param '{0}' is deprecated. See the module docs for more information", k), removedInVersion.ToString());
+
+                object removedAtDate = v["removed_at_date"];
+                if (removedAtDate != null && parameters.Contains(k))
+                    Deprecate(String.Format("Param '{0}' is deprecated. See the module docs for more information", k), (string)null);
             }
         }
 

lib/ansible/utils/display.py

@@ -249,9 +249,13 @@ class Display(with_metaclass(Singleton, object)):
             else:
                 self.display("<%s> %s" % (host, msg), color=C.COLOR_VERBOSE, stderr=to_stderr)
 
-    def deprecated(self, msg, version=None, removed=False):
+    def deprecated(self, msg, version=None, removed=False, date=None, collection_name=None):
         ''' used to print out a deprecation message.'''
 
+        # `date` and `collection_name` are Ansible 2.10 parameters. We accept and ignore them,
+        # to avoid modules/plugins from 2.10 conformant collections to break with new enough
+        # versions of Ansible 2.9.
+
         if not removed and not C.DEPRECATION_WARNINGS:
             return
 

test/integration/targets/win_csharp_utils/library/ansible_basic_tests.ps1

@@ -1469,7 +1469,8 @@ test_no_log - Invoked with:
 
         $expected_msg = "internal error: argument spec entry contains an invalid key 'invalid', valid keys: apply_defaults, "
         $expected_msg += "aliases, choices, default, elements, mutually_exclusive, no_log, options, removed_in_version, "
-        $expected_msg += "required, required_by, required_if, required_one_of, required_together, supports_check_mode, type"
+        $expected_msg += "removed_at_date, removed_from_collection, required, required_by, required_if, required_one_of, "
+        $expected_msg += "required_together, supports_check_mode, type"
 
         $actual.Keys.Count | Assert-Equals -Expected 3
         $actual.failed | Assert-Equals -Expected $true
@@ -1501,8 +1502,8 @@ test_no_log - Invoked with:
 
         $expected_msg = "internal error: argument spec entry contains an invalid key 'invalid', valid keys: apply_defaults, "
         $expected_msg += "aliases, choices, default, elements, mutually_exclusive, no_log, options, removed_in_version, "
-        $expected_msg += "required, required_by, required_if, required_one_of, required_together, supports_check_mode, type - "
+        $expected_msg += "removed_at_date, removed_from_collection, required, required_by, required_if, required_one_of, "
-        $expected_msg += "found in option_key -> sub_option_key"
+        $expected_msg += "required_together, supports_check_mode, type - found in option_key -> sub_option_key"
 
         $actual.Keys.Count | Assert-Equals -Expected 3
         $actual.failed | Assert-Equals -Expected $true
@@ -2407,6 +2408,60 @@ test_no_log - Invoked with:
         $actual.Length | Assert-Equals -Expected 1
         $actual[0] | Assert-DictionaryEquals -Expected @{"abc" = "def"}
     }
+
+    "Ansible 2.10 compatibility" = {
+        $spec = @{
+            options = @{
+                removed1 = @{removed_at_date = [DateTime]"2020-01-01"; removed_from_collection = "foo.bar"}
+            }
+        }
+        $complex_args = @{
+            removed1 = "value"
+        }
+
+        $m = [Ansible.Basic.AnsibleModule]::Create(@(), $spec)
+        $m.Deprecate("version w collection", "2.10", "foo.bar")
+        $m.Deprecate("date w/o collection", [DateTime]"2020-01-01")
+        $m.Deprecate("date w collection", [DateTime]"2020-01-01", "foo.bar")
+
+        $failed = $false
+        try {
+            $m.ExitJson()
+        } catch [System.Management.Automation.RuntimeException] {
+            $failed = $true
+            $_.Exception.Message | Assert-Equals -Expected "exit: 0"
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
+        }
+        $failed | Assert-Equals -Expected $true
+
+        $expected = @{
+            changed = $false
+            invocation = @{
+                module_args = @{
+                    removed1 = "value"
+                }
+            }
+            deprecations = @(
+                @{
+                    msg = "Param 'removed1' is deprecated. See the module docs for more information"
+                    version = $null
+                },
+                @{
+                    msg = "version w collection"
+                    version = "2.10"
+                },
+                @{
+                    msg = "date w/o collection"
+                    version = $null
+                },
+                @{
+                    msg = "date w collection"
+                    version = $null
+                }
+            )
+        }
+        $actual | Assert-DictionaryEquals -Expected $expected
+    }
 }
 
 try {

test/lib/ansible_test/_data/sanity/validate-modules/validate_modules/main.py

@@ -1623,9 +1623,9 @@ class ModuleValidator(Validator):
             # See if current version => deprecated.removed_in, ie, should be docs only
             if isinstance(doc_info['ANSIBLE_METADATA']['value'], ast.Dict) and 'removed' in ast.literal_eval(doc_info['ANSIBLE_METADATA']['value'])['status']:
                 end_of_deprecation_should_be_removed_only = True
-            elif docs and 'deprecated' in docs and docs['deprecated'] is not None:
+            elif docs and 'deprecated' in docs and docs['deprecated'] is not None and 'removed_in' in docs['deprecated']:
                 try:
-                    removed_in = StrictVersion(str(docs.get('deprecated')['removed_in']))
+                    removed_in = StrictVersion(str(docs['deprecated']['removed_in']))
                 except ValueError:
                     end_of_deprecation_should_be_removed_only = False
                 else:

test/units/module_utils/basic/test_deprecate_warn.py

@@ -23,9 +23,12 @@ def test_warn(am, capfd):
 def test_deprecate(am, capfd):
     am.deprecate('deprecation1')
     am.deprecate('deprecation2', '2.3')
+    am.deprecate('deprecation3', '2.5', collection_name='foo.bar')  # Ansible 2.10 compatibility
+    am.deprecate('deprecation4', date='2020-01-01')  # Ansible 2.10 compatibility
+    am.deprecate('deprecation5', date='2020-01-01', collection_name='foo.bar')  # Ansible 2.10 compatibility
 
     with pytest.raises(SystemExit):
-        am.exit_json(deprecations=['deprecation3', ('deprecation4', '2.4')])
+        am.exit_json(deprecations=['deprecation6', ('deprecation7', '2.4')])
 
     out, err = capfd.readouterr()
     output = json.loads(out)
@@ -33,8 +36,11 @@ def test_deprecate(am, capfd):
     assert output['deprecations'] == [
         {u'msg': u'deprecation1', u'version': None},
         {u'msg': u'deprecation2', u'version': '2.3'},
-        {u'msg': u'deprecation3', u'version': None},
+        {u'msg': u'deprecation3', u'version': '2.5'},
-        {u'msg': u'deprecation4', u'version': '2.4'},
+        {u'msg': u'deprecation4', u'version': None},
+        {u'msg': u'deprecation5', u'version': None},
+        {u'msg': u'deprecation6', u'version': None},
+        {u'msg': u'deprecation7', u'version': '2.4'},
     ]
 
 

test/units/module_utils/common/parameters/test_list_deprecations.py

@@ -25,11 +25,14 @@ def test_list_deprecations():
         'old': {'type': 'str', 'removed_in_version': '2.5'},
         'foo': {'type': 'dict', 'options': {'old': {'type': 'str', 'removed_in_version': 1.0}}},
         'bar': {'type': 'list', 'elements': 'dict', 'options': {'old': {'type': 'str', 'removed_in_version': '2.10'}}},
+        # Ansible 2.10 compatibility:
+        'compat': {'type': 'str', 'removed_at_date': '2020-01-01', 'removed_from_collection': 'foo.bar'},
     }
 
     params = {
         'name': 'rod',
         'old': 'option',
+        'old2': 'option2',
         'foo': {'old': 'value'},
         'bar': [{'old': 'value'}, {}],
     }