[cloud]Add aws_ses_identity_policy module for managing SES sending policies (#36623)

* Add aws_ses_identity_policy module for managing SES sending policies

* Add option to AnsibleAWSModule for applying a retry decorator to all calls.

* Add per-callsite opt in to retry behaviours in AnsibleAWSModule

* Update aws_ses_identity_policy module to opt in to retries at all callsites.

* Add test for aws_ses_identity_policy module with inline policy.

* Remove implicit retrys on boto resources since they're not working yet.

CHANGELOG.md

@@ -25,6 +25,7 @@ See [Porting Guide](http://docs.ansible.com/ansible/devel/porting_guides/porting
 #### Cloud
 - amazon
   * aws_caller_facts
+  * aws_ses_identity_policy
 
 
 <a id="2.5"></a>

hacking/aws_config/testing_policies/compute-policy.json

@@ -261,7 +261,11 @@
             "ses:VerifyDomainIdentity",
             "ses:SetIdentityNotificationTopic",
             "ses:SetIdentityHeadersInNotificationsEnabled",
-            "ses:SetIdentityFeedbackForwardingEnabled"
+            "ses:SetIdentityFeedbackForwardingEnabled",
+            "ses:GetIdentityPolicies",
+            "ses:PutIdentityPolicy",
+            "ses:DeleteIdentityPolicy",
+            "ses:ListIdentityPolicies"
           ],
           "Resource": [
             "*"

lib/ansible/module_utils/aws/core.py

@@ -47,6 +47,7 @@ or
 
 """
 
+from functools import wraps
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
 from ansible.module_utils.ec2 import HAS_BOTO3, camel_dict_to_snake_dict, ec2_argument_spec, boto3_conn, get_aws_connection_info
@@ -119,10 +120,11 @@ class AnsibleAWSModule(object):
     def warn(self, *args, **kwargs):
         return self._module.warn(*args, **kwargs)
 
-    def client(self, service):
+    def client(self, service, retry_decorator=None):
         region, ec2_url, aws_connect_kwargs = get_aws_connection_info(self, boto3=True)
-        return boto3_conn(self, conn_type='client', resource=service,
+        conn = boto3_conn(self, conn_type='client', resource=service,
                           region=region, endpoint=ec2_url, **aws_connect_kwargs)
+        return conn if retry_decorator is None else _RetryingBotoClientWrapper(conn, retry_decorator)
 
     def resource(self, service):
         region, ec2_url, aws_connect_kwargs = get_aws_connection_info(self, boto3=True)
@@ -159,3 +161,29 @@ class AnsibleAWSModule(object):
         else:
             self._module.fail_json(msg=message, exception=last_traceback,
                                    **camel_dict_to_snake_dict(response))
+
+
+class _RetryingBotoClientWrapper(object):
+    def __init__(self, client, retry):
+        self.client = client
+        self.retry = retry
+
+    def _create_optional_retry_wrapper_function(self, unwrapped):
+        retrying_wrapper = self.retry(unwrapped)
+
+        @wraps(unwrapped)
+        def deciding_wrapper(aws_retry=False, *args, **kwargs):
+            if aws_retry:
+                return retrying_wrapper(*args, **kwargs)
+            else:
+                return unwrapped(*args, **kwargs)
+        return deciding_wrapper
+
+    def __getattr__(self, name):
+        unwrapped = getattr(self.client, name)
+        if callable(unwrapped):
+            wrapped = self._create_optional_retry_wrapper_function(unwrapped)
+            setattr(self, name, wrapped)
+            return wrapped
+        else:
+            return unwrapped

lib/ansible/modules/cloud/amazon/aws_ses_identity_policy.py

@@ -0,0 +1,194 @@
+#!/usr/bin/python
+# Copyright (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+ANSIBLE_METADATA = {
+    'metadata_version': '1.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+DOCUMENTATION = '''
+---
+module: aws_ses_identity_policy
+short_description: Manages SES sending authorization policies
+description:
+    - This module allows the user to manage sending authorization policies associated with an SES identity (email or domain).
+    - SES authorization sending policies can be used to control what actors are able to send email
+      on behalf of the validated identity and what conditions must be met by the sent emails.
+version_added: "2.6"
+author: Ed Costello    (@orthanc)
+
+options:
+    identity:
+        description: |
+            The SES identity to attach or remove a policy from. This can be either the full ARN or just
+            the verified email or domain.
+        required: true
+    policy_name:
+        description: The name used to identify the policy within the scope of the identity it's attached to.
+        required: true
+    policy:
+        description: A properly formated JSON sending authorization policy. Required when I(state=present).
+    state:
+        description: Whether to create(or update) or delete the authorization policy on the identity.
+        default: present
+        choices: [ 'present', 'absent' ]
+requirements: [ 'botocore', 'boto3' ]
+extends_documentation_fragment:
+    - aws
+    - ec2
+'''
+
+EXAMPLES = '''
+# Note: These examples do not set authentication details, see the AWS Guide for details.
+
+- name: add sending authorization policy to domain identity
+  aws_ses_identity_policy:
+    identity: example.com
+    policy_name: ExamplePolicy
+    policy: "{{ lookup('template', 'policy.json.j2') }}"
+    state: present
+
+- name: add sending authorization policy to email identity
+  aws_ses_identity_policy:
+    identity: example@example.com
+    policy_name: ExamplePolicy
+    policy: "{{ lookup('template', 'policy.json.j2') }}"
+    state: present
+
+- name: add sending authorization policy to identity using ARN
+  aws_ses_identity_policy:
+    identity: "arn:aws:ses:us-east-1:12345678:identity/example.com"
+    policy_name: ExamplePolicy
+    policy: "{{ lookup('template', 'policy.json.j2') }}"
+    state: present
+
+- name: remove sending authorization policy
+  aws_ses_identity_policy:
+    identity: example.com
+    policy_name: ExamplePolicy
+    state: absent
+'''
+
+RETURN = '''
+policies:
+    description: A list of all policies present on the identity after the operation.
+    returned: success
+    type: list
+    sample: [ExamplePolicy]
+'''
+
+from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils.ec2 import compare_policies, AWSRetry
+
+import json
+
+try:
+    from botocore.exceptions import BotoCoreError, ClientError
+except ImportError:
+    pass  # caught by imported HAS_BOTO3
+
+
+def get_identity_policy(connection, module, identity, policy_name):
+    try:
+        response = connection.get_identity_policies(Identity=identity, PolicyNames=[policy_name], aws_retry=True)
+    except (BotoCoreError, ClientError) as e:
+        module.fail_json_aws(e, msg='Failed to retrieve identity policy {policy}'.format(policy=policy_name))
+    policies = response['Policies']
+    if policy_name in policies:
+        return policies[policy_name]
+    return None
+
+
+def create_or_update_identity_policy(connection, module):
+    identity = module.params.get('identity')
+    policy_name = module.params.get('policy_name')
+    required_policy = module.params.get('policy')
+    required_policy_dict = json.loads(required_policy)
+
+    changed = False
+    policy = get_identity_policy(connection, module, identity, policy_name)
+    policy_dict = json.loads(policy) if policy else None
+    if compare_policies(policy_dict, required_policy_dict):
+        changed = True
+        try:
+            if not module.check_mode:
+                connection.put_identity_policy(Identity=identity, PolicyName=policy_name, Policy=required_policy, aws_retry=True)
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e, msg='Failed to put identity policy {policy}'.format(policy=policy_name))
+
+    # Load the list of applied policies to include in the response.
+    # In principle we should be able to just return the response, but given
+    # eventual consistency behaviours in AWS it's plausible that we could
+    # end up with a list that doesn't contain the policy we just added.
+    # So out of paranoia check for this case and if we're missing the policy
+    # just make sure it's present.
+    #
+    # As a nice side benefit this also means the return is correct in check mode
+    try:
+        policies_present = connection.list_identity_policies(Identity=identity, aws_retry=True)['PolicyNames']
+    except (BotoCoreError, ClientError) as e:
+        module.fail_json_aws(e, msg='Failed to list identity policies')
+    if policy_name is not None and policy_name not in policies_present:
+        policies_present = list(policies_present)
+        policies_present.append(policy_name)
+    module.exit_json(
+        changed=changed,
+        policies=policies_present,
+    )
+
+
+def delete_identity_policy(connection, module):
+    identity = module.params.get('identity')
+    policy_name = module.params.get('policy_name')
+
+    changed = False
+    try:
+        policies_present = connection.list_identity_policies(Identity=identity, aws_retry=True)['PolicyNames']
+    except (BotoCoreError, ClientError) as e:
+        module.fail_json_aws(e, msg='Failed to list identity policies')
+    if policy_name in policies_present:
+        try:
+            if not module.check_mode:
+                connection.delete_identity_policy(Identity=identity, PolicyName=policy_name, aws_retry=True)
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e, msg='Failed to delete identity policy {policy}'.format(policy=policy_name))
+        changed = True
+        policies_present = list(policies_present)
+        policies_present.remove(policy_name)
+
+    module.exit_json(
+        changed=changed,
+        policies=policies_present,
+    )
+
+
+def main():
+    module = AnsibleAWSModule(
+        argument_spec={
+            'identity': dict(required=True, type='str'),
+            'state': dict(default='present', choices=['present', 'absent']),
+            'policy_name': dict(required=True, type='str'),
+            'policy': dict(type='json', default=None),
+        },
+        required_if=[['state', 'present', ['policy']]],
+        supports_check_mode=True,
+    )
+
+    # SES APIs seem to have a much lower throttling threshold than most of the rest of the AWS APIs.
+    # Docs say 1 call per second. This shouldn't actually be a big problem for normal usage, but
+    # the ansible build runs multiple instances of the test in parallel that's caused throttling
+    # failures so apply a jittered backoff to call SES calls.
+    connection = module.client('ses', retry_decorator=AWSRetry.jittered_backoff())
+
+    state = module.params.get("state")
+
+    if state == 'present':
+        create_or_update_identity_policy(connection, module)
+    else:
+        delete_identity_policy(connection, module)
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/aws_ses_identity_policy/aliases

@@ -0,0 +1,2 @@
+cloud/aws
+posix/ci/cloud/group4/aws

test/integration/targets/aws_ses_identity_policy/defaults/main.yaml

@@ -0,0 +1,3 @@
+---
+domain_identity: "{{ resource_prefix }}.example.com"
+policy_name: "TestPolicy"

test/integration/targets/aws_ses_identity_policy/tasks/main.yaml

@@ -0,0 +1,334 @@
+---
+# ============================================================
+- name: set up aws connection info
+  set_fact:
+    aws_connection_info: &aws_connection_info
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token }}"
+      region: "{{ aws_region }}"
+  no_log: yes
+# ============================================================
+- name: test add identity policy
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: register identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        policy: "{{ lookup('template', 'policy.json.j2') }}"
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - name: assert result.changed == True
+      assert:
+        that:
+          - result.changed == True
+
+    - name: assert result.policies contains only policy
+      assert:
+        that:
+          - result.policies|length == 1
+          - result.policies|select('equalto', policy_name)|list|length == 1
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info
+# ============================================================
+- name: test add duplicate identity policy
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: register identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        policy: "{{ lookup('template', 'policy.json.j2') }}"
+        state: present
+        <<: *aws_connection_info
+
+    - name: register duplicate identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        policy: "{{ lookup('template', 'policy.json.j2') }}"
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - name: assert result.changed == False
+      assert:
+        that:
+          - result.changed == False
+
+    - name: assert result.policies contains only policy
+      assert:
+        that:
+          - result.policies|length == 1
+          - result.policies|select('equalto', policy_name)|list|length == 1
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info
+# ============================================================
+- name: test add identity policy by identity arn
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: register identity policy
+      aws_ses_identity_policy:
+        identity: "{{ identity_info.identity_arn }}"
+        policy_name: "{{ policy_name }}"
+        policy: "{{ lookup('template', 'policy.json.j2') }}"
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - name: assert result.changed == True
+      assert:
+        that:
+          - result.changed == True
+
+    - name: assert result.policies contains only policy
+      assert:
+        that:
+          - result.policies|length == 1
+          - result.policies|select('equalto', policy_name)|list|length == 1
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info
+# ============================================================
+- name: test add multiple identity policies
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: register identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}-{{ item }}"
+        policy: "{{ lookup('template', 'policy.json.j2') }}"
+        state: present
+        <<: *aws_connection_info
+      with_items:
+        - 1
+        - 2
+      register: result
+
+    - name: assert result.policies contains policies
+      assert:
+        that:
+          - result.results[1].policies|length == 2
+          - result.results[1].policies|select('equalto', policy_name + '-1')|list|length == 1
+          - result.results[1].policies|select('equalto', policy_name + '-2')|list|length == 1
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info
+# ============================================================
+- name: test add inline identity policy
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: register identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        policy:
+          Id: SampleAuthorizationPolicy
+          Version: "2012-10-17"
+          Statement:
+            - Sid: DenyAll
+              Effect: Deny
+              Resource: "{{ identity_info.identity_arn }}"
+              Principal: "*"
+              Action: "*"
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - name: assert result.changed == True
+      assert:
+        that:
+          - result.changed == True
+
+    - name: assert result.policies contains only policy
+      assert:
+        that:
+          - result.policies|length == 1
+          - result.policies|select('equalto', policy_name)|list|length == 1
+
+    - name: register duplicate identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        policy:
+          Id: SampleAuthorizationPolicy
+          Version: "2012-10-17"
+          Statement:
+            - Sid: DenyAll
+              Effect: Deny
+              Resource: "{{ identity_info.identity_arn }}"
+              Principal: "*"
+              Action: "*"
+        state: present
+        <<: *aws_connection_info
+      register: result
+
+    - name: assert result.changed == False
+      assert:
+        that:
+          - result.changed == False
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info
+# ============================================================
+- name: test remove identity policy
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: register identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        policy: "{{ lookup('template', 'policy.json.j2') }}"
+        state: present
+        <<: *aws_connection_info
+
+    - name: delete identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+
+    - name: assert result.changed == True
+      assert:
+        that:
+          - result.changed == True
+
+    - name: assert result.policies empty
+      assert:
+        that:
+          - result.policies|length == 0
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info
+# ============================================================
+- name: test remove missing identity policy
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: delete identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        state: absent
+        <<: *aws_connection_info
+      register: result
+
+    - name: assert result.changed == False
+      assert:
+        that:
+          - result.changed == False
+
+    - name: assert result.policies empty
+      assert:
+        that:
+          - result.policies|length == 0
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info
+# ============================================================
+- name: test add identity policy with invalid policy
+  block:
+    - name: register identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: present
+        <<: *aws_connection_info
+      register: identity_info
+
+    - name: register identity policy
+      aws_ses_identity_policy:
+        identity: "{{ domain_identity }}"
+        policy_name: "{{ policy_name }}"
+        policy: '{"noSuchAttribute": 2}'
+        state: present
+        <<: *aws_connection_info
+      register: result
+      failed_when: result.failed == False
+
+    - name: assert error.code == InvalidPolicy
+      assert:
+        that:
+          - result.error.code == 'InvalidPolicy'
+
+  always:
+    - name: clean-up identity
+      aws_ses_identity:
+        identity: "{{ domain_identity }}"
+        state: absent
+        <<: *aws_connection_info

test/integration/targets/aws_ses_identity_policy/templates/policy.json.j2

@@ -0,0 +1,13 @@
+{
+  "Id": "SampleAuthorizationPolicy",
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "DenyAll",
+      "Effect": "Deny",
+      "Resource": "{{ identity_info.identity_arn }}",
+      "Principal": "*",
+      "Action": "*"
+    }
+  ]
+}