aws ssm parameter lookup - change to return Nones for missing variables (#36456)

* aws ssm parameter lookup - change to return Nones for missing variables

* aws ssm parameter lookup - fix error case message to dump response

* aws ssm parameter lookup - fix integration test cases

lib/ansible/plugins/lookup/aws_ssm.py

@@ -193,10 +193,16 @@ class LookupModule(LookupBase):
                 response = client.get_parameters(**ssm_dict)
             except ClientError as e:
                 raise AnsibleError("SSM lookup exception: {0}".format(to_native(e)))
-            if len(response['Parameters']) == len(terms):
-                ret = [p['Value'] for p in response['Parameters']]
+            params = boto3_tag_list_to_ansible_dict(response['Parameters'], tag_name_key_name="Name",
+                                                    tag_value_key_name="Value")
+            for i in terms:
+                if i in params:
+                    ret.append(params[i])
+                elif i in response['InvalidParameters']:
+                    ret.append(None)
                 else:
-                raise AnsibleError('Undefined AWS SSM parameter: %s ' % str(response['InvalidParameters']))
+                    raise AnsibleError("Ansible internal error: aws_ssm lookup failed to understand boto3 return value: {0}".format(str(response)))
+            return ret
 
         display.vvvv("AWS_ssm path lookup returning: %s " % str(ret))
         return ret

test/integration/targets/aws_ssm_parameters/tasks/main.yml

@@ -74,17 +74,10 @@
         - "'{{lookup('aws_ssm', '/' ~ ssm_key_prefix ~ '/path', region=ec2_region, aws_access_key=ec2_access_key, aws_secret_key=ec2_secret_key, aws_security_token=security_token, bypath=True, shortnames=true ) | to_json }}' == '{\"toovar\": \"too value\", \"wonvar\": \"won value\"}'"
 
     # ============================================================
-    - name: Error in case we don't find a named parameter
-      debug:
-       msg: "'{{lookup('aws_ssm', '/' ~ ssm_key_prefix ~ '/Goodbye', region=ec2_region, aws_access_key=ec2_access_key, aws_secret_key=ec2_secret_key, aws_security_token=security_token )}}' == 'World'"
-      register: result
-      ignore_errors: true
-
-    - name: assert failure from failure to find parameter
+    - name: Returns empty value in case we don't find a named parameter
       assert:
        that:
-           - 'result.failed'
-           - "'Undefined AWS SSM parameter' in result.msg"
+        - "'{{lookup('aws_ssm', '/' ~ ssm_key_prefix ~ '/Goodbye', region=ec2_region, aws_access_key=ec2_access_key, aws_secret_key=ec2_secret_key, aws_security_token=security_token )}}' == ''"
 
     # ============================================================
     - name: Handle multiple paths with one that doesn't exist - default to full names.
@@ -134,3 +127,4 @@
         - "/{{ssm_key_prefix}}/path/wonvar"
         - "/{{ssm_key_prefix}}/path/toovar"
         - "/{{ssm_key_prefix}}/path/tree/treevar"
+        - "/{{ssm_key_prefix}}/deeppath/wondir/samevar"

test/units/plugins/lookup/test_aws_ssm.py

@@ -61,9 +61,16 @@ path_success_response['Parameters'] = [
     {'Name': '/testpath/won', 'Type': 'String', 'Value': 'simple_value_won', 'Version': 1}
 ]
 
-missing_variable_fail_response = copy(simple_variable_success_response)
-missing_variable_fail_response['Parameters'] = []
-missing_variable_fail_response['InvalidParameters'] = ['missing_variable']
+missing_variable_response = copy(simple_variable_success_response)
+missing_variable_response['Parameters'] = []
+missing_variable_response['InvalidParameters'] = ['missing_variable']
+
+some_missing_variable_response = copy(simple_variable_success_response)
+some_missing_variable_response['Parameters'] = [
+    {'Name': 'simple', 'Type': 'String', 'Value': 'simple_value', 'Version': 1},
+    {'Name': '/testpath/won', 'Type': 'String', 'Value': 'simple_value_won', 'Version': 1}
+]
+some_missing_variable_response['InvalidParameters'] = ['missing_variable']
 
 
 dummy_credentials = {}
@@ -109,16 +116,37 @@ def test_path_lookup_variable(mocker):
     get_path_fn.assert_called_with(Path="/testpath", Recursive=False, WithDecryption=True)
 
 
-def test_warn_missing_variable(mocker):
+def test_return_none_for_missing_variable(mocker):
+    """
+    during jinja2 templates, we can't shouldn't normally raise exceptions since this blocks the ability to use defaults.
+
+    for this reason we return ```None``` for missing variables
+    """
     lookup = aws_ssm.LookupModule()
     lookup._load_name = "aws_ssm"
 
     boto3_double = mocker.MagicMock()
-    boto3_double.Session.return_value.client.return_value.get_parameters.return_value = missing_variable_fail_response
+    boto3_double.Session.return_value.client.return_value.get_parameters.return_value = missing_variable_response
+
+    with mocker.patch.object(boto3, 'session', boto3_double):
+        retval = lookup.run(["missing_variable"], {}, **dummy_credentials)
+    assert(retval[0] is None)
+
+
+def test_match_retvals_to_call_params_even_with_some_missing_variables(mocker):
+    """
+    If we get a complex list of variables with some missing and some not, we still have to return a
+    list which matches with the original variable list.
+    """
+    lookup = aws_ssm.LookupModule()
+    lookup._load_name = "aws_ssm"
+
+    boto3_double = mocker.MagicMock()
+    boto3_double.Session.return_value.client.return_value.get_parameters.return_value = some_missing_variable_response
 
-    with pytest.raises(AnsibleError):
     with mocker.patch.object(boto3, 'session', boto3_double):
-            lookup.run(["missing_variable"], {}, **dummy_credentials)
+        retval = lookup.run(["simple", "missing_variable", "/testpath/won", "simple"], {}, **dummy_credentials)
+    assert(retval == ["simple_value", None, "simple_value_won", "simple_value"])
 
 
 error_response = {'Error': {'Code': 'ResourceNotFoundException', 'Message': 'Fake Testing Error'}}