[cloud] add summary feature and refactor AWS cloudfront_facts module (#20791)

* initial commit of cloudfront_facts.py * modification as per review from @georgepsarakis * fixed shippable build error * fixed shippable build error * removed wildcard imports and replaced with specific imports from ansible.module_utils.ec2 as advised by @georgepsarakis * renamed all instances of cloud_front_origin_access_identity to origin_access_identity as advised by @georgepsarakis * show facts based on alias and distribution id for easy referencing as advised by @ryansb. have done for both distribution and distribution_config * fixed incorrect logic for default list_distributions, fixed list_distributions_by_web_acl - wasn't passing web_acl_id, fixed list_invalidations keyword args missing DistributionId * fixed last fact added clobbering all previous facts for an alias or an id * removed list_ prefix from list keys * removed unnecessary boto fields. made list_distributions and list_streaming_distributions dictionaries with id/alias as key. fixed list_invalidations. * initial commit of cloudfront_facts summary * checks for empty list returned from boto, standardised list naming * neatened up parameters * added summary documentation * refactoring of functions for modularity * refactoring, neatening code, fix for if cname not present, added try-catch blocks * more refactoring, cleaning * more cleaning, allowed streaming distributions to be found by domain name alias * removed unnecessary line * fixed tabs * fixed indentation * removed trailing whitespace * removed more whitespace * more refactoring, modified where invalidations dict is set * added ETag to summary * refactored summary list * moved list out of for loop * trailing white space * reverted line lengthening as advised by @willthames * modified exceptions to multi line as advised by @willthames * reverted variable spacing to be more pythonic' * reverted spacing in YAML as advised by @ryansb * reverted line spacing for parameters for correct blame attribution as advised by @ryansb * removed white space * more white space * reverted line spacings for parameters as advised by @ryansb * removed spaces between parameters as advised by @ryansb
9 years ago · 93a689958e
parent f824b2ce8d
commit 93a689958e
1 changed files with 196 additions and 83 deletions
--- a/lib/ansible/modules/cloud/amazon/cloudfront_facts.py
+++ b/lib/ansible/modules/cloud/amazon/cloudfront_facts.py
@ -115,6 +115,11 @@ options:
            - Get a list of streaming distributions.
        required: false
        default: false
+    summary:
+        description:
+            - Returns a summary of all distributions, streaming distributions and origin_access_identities. This is the default behaviour if no option is selected.
+        required: false
+        default: false

 extends_documentation_fragment:
    - aws
@ -124,6 +129,10 @@ extends_documentation_fragment:
 EXAMPLES = '''
 # Note: These examples do not set authentication details, see the AWS Guide for details.

+# Get a summary of distributions
+- cloudfront_facts:
+    summary: true
+
 # Get information about a distribution
 - cloudfront_facts:
    distribution: true
@ -152,6 +161,11 @@ EXAMPLES = '''
    origin_access_identity: true
    origin_access_identity_id: my-cloudfront-origin-access-identity-id

+# Get all information about lists not requiring parameters (ie. list_origin_access_identities, list_distributions, list_streaming_distributions)
+- cloudfront_facts:
+    origin_access_identity: true
+    origin_access_identity_id: my-cloudfront-origin-access-identity-id
+
 # Get all information about lists not requiring parameters (ie. list_origin_access_identities, list_distributions, list_streaming_distributions)
 - cloudfront_facts:
    all_lists: true
@ -186,6 +200,10 @@ streaming_distribution_configuration:
    description: Describes the streaming configuration information for the distribution. Requires distribution_id or domain_name_alias to be specified.
    returned: only if streaming_distribution_configuration is true
    type: dict
+summary:
+    description: Gives a summary of distributions, streaming distributions and origin access identities.
+    returned: as default or if summary is true
+    type: dict
 '''

 try:
@ -217,122 +235,225 @@ class CloudFrontServiceManager:
        except botocore.exceptions.NoRegionError:
            self.module.fail_json(msg="Region must be specified as a parameter, in AWS_DEFAULT_REGION environment variable or in boto configuration file")
        except Exception as e:
-            self.module.fail_json(msg="Can't establish connection - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Can't establish connection - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_distribution(self, distribution_id):
        try:
            func = partial(self.client.get_distribution,Id=distribution_id)
            return self.paginated_response(func)
        except Exception as e:
-            self.module.fail_json(msg="Error describing distribution - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing distribution - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_distribution_config(self, distribution_id):
        try:
            func = partial(self.client.get_distribution_config,Id=distribution_id)
            return self.paginated_response(func)
        except Exception as e:
-            self.module.fail_json(msg="Error describing distribution configuration - " + str(e), exception=traceback.format_exec(e))
+            self.module.fail_json(msg="Error describing distribution configuration - " + str(e),
+                                  exception=traceback.format_exec(e))

    def get_origin_access_identity(self, origin_access_identity_id):
        try:
            func = partial(self.client.get_cloud_front_origin_access_identity,Id=origin_access_identity_id)
            return self.paginated_response(func)
        except Exception as e:
-            self.module.fail_json(msg="Error describing origin access identity - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing origin access identity - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_origin_access_identity_config(self, origin_access_identity_id):
        try:
            func = partial(self.client.get_cloud_front_origin_access_identity_config,Id=origin_access_identity_id)
            return self.paginated_response(func)
        except Exception as e:
-            self.module.fail_json(msg="Error describing origin access identity configuration - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing origin access identity configuration - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_invalidation(self, distribution_id, invalidation_id):
        try:
            func = partial(self.client.get_invalidation,DistributionId=distribution_id,Id=invalidation_id)
            return self.paginated_response(func)
        except Exception as e:
-            self.module.fail_json(msg="Error describing invalidation - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing invalidation - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_streaming_distribution(self, distribution_id):
        try:
            func = partial(self.client.get_streaming_distribution,Id=distribution_id)
            return self.paginated_response(func)
        except Exception as e:
-            self.module.fail_json(msg="Error describing streaming distribution - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing streaming distribution - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_streaming_distribution_config(self, distribution_id):
        try:
            func = partial(self.client.get_streaming_distribution_config,Id=distribution_id)
            return self.paginated_response(func)
        except Exception as e:
-            self.module.fail_json(msg="Error describing streaming distribution - " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error describing streaming distribution - " + str(e),
+                                  exception=traceback.format_exc(e))

    def list_origin_access_identities(self):
        try:
            func = partial(self.client.list_cloud_front_origin_access_identities)
-            return self.paginated_response(func, 'CloudFrontOriginAccessIdentityList')['Items']
+            origin_access_identity_list = self.paginated_response(func, 'CloudFrontOriginAccessIdentityList')
+            if origin_access_identity_list['Quantity'] > 0:
+              return origin_access_identity_list['Items']
+            return {}
        except Exception as e:
-            self.module.fail_json(msg="Error listing cloud front origin access identities = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing cloud front origin access identities - " + str(e),
+                                  exception=traceback.format_exc(e))

    def list_distributions(self, keyed=True):
        try:
            func = partial(self.client.list_distributions)
-            distribution_list = self.paginated_response(func, 'DistributionList')['Items']
+            distribution_list = self.paginated_response(func, 'DistributionList')
+            if distribution_list['Quantity'] == 0:
+                return {}
+            else:
+                distribution_list = distribution_list['Items']
            if not keyed:
                return distribution_list
            return self.keyed_list_helper(distribution_list)
        except Exception as e:
-            self.module.fail_json(msg="Error listing distributions = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing distributions - " + str(e),
+                                  exception=traceback.format_exc(e))

    def list_distributions_by_web_acl_id(self, web_acl_id):
        try:
            func = partial(self.client.list_distributions_by_web_acl_id, WebAclId=web_acl_id)
-            distributions = self.paginated_response(func, 'DistributionList')['Items']
-            return self.keyed_list_helper(distributions)
+            distribution_list = self.paginated_response(func, 'DistributionList')
+            if distribution_list['Quantity'] == 0:
+                return {}
+            else:
+                distribution_list = distribution_list['Items']
+            return self.keyed_list_helper(distribution_list)
        except Exception as e:
-            self.module.fail_json(msg="Error listing distributions by web acl id = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing distributions by web acl id - " + str(e),
+                                  exception=traceback.format_exc(e))

    def list_invalidations(self, distribution_id):
        try:
            func = partial(self.client.list_invalidations, DistributionId=distribution_id)
-            return self.paginated_response(func, 'InvalidationList')['Items']
+            invalidation_list = self.paginated_response(func, 'InvalidationList')
+            if invalidation_list['Quantity'] > 0:
+                return invalidation_list['Items']
+            return {}
        except Exception as e:
-            self.module.fail_json(msg="Error listing invalidations = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error listing invalidations - " + str(e),
+                                  exception=traceback.format_exc(e))

-    def list_streaming_distributions(self):
+    def list_streaming_distributions(self, keyed=True):
        try:
            func = partial(self.client.list_streaming_distributions)
-            streaming_distributions = self.paginated_response(func, 'StreamingDistributionList')['Items']
-            return self.keyed_list_helper(streaming_distributions)
+            streaming_distribution_list = self.paginated_response(func, 'StreamingDistributionList')
+            if streaming_distribution_list['Quantity'] == 0:
+                return {}
+            else:
+                streaming_distribution_list = streaming_distribution_list['Items']
+            if not keyed:
+                return streaming_distribution_list
+            return self.keyed_list_helper(streaming_distribution_list)
+        except Exception as e:
+            self.module.fail_json(msg="Error listing streaming distributions - " + str(e),
+                                  exception=traceback.format_exc(e))
+
+    def summary(self):
+        summary_dict = {}
+        summary_dict.update(self.summary_get_distribution_list(False))
+        summary_dict.update(self.summary_get_distribution_list(True))
+        summary_dict.update(self.summary_get_origin_access_identity_list())
+        return summary_dict
+
+    def summary_get_origin_access_identity_list(self):
+        try:
+            origin_access_identity_list = { 'origin_access_identities': [] }
+            origin_access_identities = self.list_origin_access_identities()
+            for origin_access_identity in origin_access_identities:
+                oai_id = origin_access_identity['Id']
+                oai_full_response = self.get_origin_access_identity(oai_id)
+                oai_summary = { 'Id': oai_id, 'ETag': oai_full_response['ETag'] }
+                origin_access_identity_list['origin_access_identities'].append( oai_summary )
+            return origin_access_identity_list
        except Exception as e:
-            self.module.fail_json(msg="Error listing streaming distributions = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error generating summary of origin access identities - " + str(e),
+                                  exception=traceback.format_exc(e))
+
+    def summary_get_distribution_list(self, streaming=False):
+        try:
+            list_name = 'streaming_distributions' if streaming else 'distributions'
+            key_list = ['Id', 'ARN', 'Status', 'LastModifiedTime', 'DomainName', 'Comment', 'PriceClass', 'Enabled' ]
+            distribution_list = { list_name: [] }
+            distributions = self.list_streaming_distributions(False) if streaming else self.list_distributions(False)
+            for dist in distributions:
+                temp_distribution = {}
+                for key_name in key_list:
+                    temp_distribution.update( { key_name: dist[key_name] } )
+                temp_distribution.update( { 'Aliases': [] } )
+                temp_distribution.update( { 'ETag': self.get_etag_from_distribution_id(dist['Id'], streaming) } )
+                if 'Items' in dist['Aliases']:
+                    for alias in dist['Aliases']['Items']:
+                        temp_distribution['Aliases'].append(alias)
+                if not streaming:
+                    temp_distribution.update( { 'WebACLId': dist['WebACLId'] } )
+                    invalidation_ids = self.get_list_of_invalidation_ids_from_distribution_id(dist['Id'])
+                    if invalidation_ids:
+                        temp_distribution.update( { 'Invalidations': invalidation_ids } )
+                distribution_list[list_name].append(temp_distribution)
+            return distribution_list
+        except Exception as e:
+            self.module.fail_json(msg="Error generating summary of distributions - " + str(e),
+                                  exception=traceback.format_exc(e))
+
+    def get_etag_from_distribution_id(self, distribution_id, streaming):
+        distribution = {}
+        if not streaming:
+            distribution = self.get_distribution(distribution_id)
+        else:
+            distribution = self.get_streaming_distribution(distribution_id)
+        return distribution['ETag']
+
+    def get_list_of_invalidation_ids_from_distribution_id(self, distribution_id):
+        try:
+            invalidation_ids = []
+            invalidations = self.list_invalidations(distribution_id)
+            for invalidation in invalidations:
+                invalidation_ids.append(invalidation['Id'])
+            return invalidation_ids
+        except Exception as e:
+            self.module.fail_json(msg="Error getting list of invalidation ids - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_distribution_id_from_domain_name(self, domain_name):
        try:
            distribution_id = ""
            distributions = self.list_distributions(False)
+            distributions += self.list_streaming_distributions(False)
            for dist in distributions:
-                for alias in dist['Aliases']['Items']:
-                    if str(alias).lower() == domain_name.lower():
-                        distribution_id = str(dist['Id'])
-                        break
+                if 'Items' in dist['Aliases']:
+                    for alias in dist['Aliases']['Items']:
+                        if str(alias).lower() == domain_name.lower():
+                            distribution_id = dist['Id']
+                            break
            return distribution_id
        except Exception as e:
-            self.module.fail_json(msg="Error getting distribution id from domain name = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error getting distribution id from domain name - " + str(e),
+                                  exception=traceback.format_exc(e))

    def get_aliases_from_distribution_id(self, distribution_id):
        aliases = []
        try:
            distributions = self.list_distributions(False)
            for dist in distributions:
-                if dist['Id'] == distribution_id:
+                if dist['Id'] == distribution_id and 'Items' in dist['Aliases']:
                    for alias in dist['Aliases']['Items']:
                        aliases.append(alias)
                    break
            return aliases
        except Exception as e:
-            self.module.fail_json(msg="Error getting list of aliases from distribution_id = " + str(e), exception=traceback.format_exc(e))
+            self.module.fail_json(msg="Error getting list of aliases from distribution_id - " + str(e),
+                                  exception=traceback.format_exc(e))

    def paginated_response(self, func, result_key=""):
        '''
@ -357,13 +478,20 @@ class CloudFrontServiceManager:
    def keyed_list_helper(self, list_to_key):
        keyed_list = dict()
        for item in list_to_key:
-            aliases = item['Aliases']['Items']
            distribution_id = item['Id']
-            keyed_list.update({distribution_id: item})
-            for alias in aliases:
-                keyed_list.update({alias: item})
+            if 'Items' in item['Aliases']:
+                aliases = item['Aliases']['Items']
+                for alias in aliases:
+                    keyed_list.update( { alias: item } )
+            keyed_list.update( { distribution_id: item } )
        return keyed_list

+def set_facts_for_distribution_id_and_alias(details, facts, distribution_id, aliases):
+    facts[distribution_id].update(details)
+    for alias in aliases:
+        facts[alias].update(details)
+    return facts
+
 def main():
    argument_spec = ec2_argument_spec()
    argument_spec.update(dict(
@ -383,7 +511,8 @@ def main():
        list_distributions=dict(required=False, default=False, type='bool'),
        list_distributions_by_web_acl_id=dict(required=False, default=False, type='bool'),
        list_invalidations=dict(required=False, default=False, type='bool'),
-        list_streaming_distributions=dict(required=False, default=False, type='bool')
+        list_streaming_distributions=dict(required=False, default=False, type='bool'),
+        summary=dict(required=False, default=False, type='bool')
    ))

    module = AnsibleModule(argument_spec=argument_spec, supports_check_mode=False)
@ -398,9 +527,7 @@ def main():
    origin_access_identity_id = module.params.get('origin_access_identity_id')
    web_acl_id = module.params.get('web_acl_id')
    domain_name_alias = module.params.get('domain_name_alias')
-
    all_lists = module.params.get('all_lists')
-
    distribution = module.params.get('distribution')
    distribution_config = module.params.get('distribution_config')
    origin_access_identity = module.params.get('origin_access_identity')
@ -408,23 +535,25 @@ def main():
    invalidation = module.params.get('invalidation')
    streaming_distribution = module.params.get('streaming_distribution')
    streaming_distribution_config = module.params.get('streaming_distribution_config')
-
    list_origin_access_identities = module.params.get('list_origin_access_identities')
    list_distributions = module.params.get('list_distributions')
    list_distributions_by_web_acl_id = module.params.get('list_distributions_by_web_acl_id')
    list_invalidations = module.params.get('list_invalidations')
    list_streaming_distributions = module.params.get('list_streaming_distributions')
+    summary = module.params.get('summary')

    aliases = []
+    result = { 'cloudfront': {} }
+    facts = {}

-    require_distribution_id = (distribution or distribution_config or invalidation or
-            streaming_distribution or streaming_distribution_config or list_invalidations)
+    require_distribution_id = (distribution or distribution_config or invalidation or streaming_distribution or
+        streaming_distribution_config or list_invalidations)

-    # set default to list_distributions if no option specified
-    list_distributions = list_distributions or not (distribution or distribution_config or
-            origin_access_identity or origin_access_identity_config or invalidation or
-            streaming_distribution or streaming_distribution_config or list_origin_access_identities or
-            list_distributions_by_web_acl_id or list_invalidations or list_streaming_distributions)
+    # set default to summary if no option specified
+    summary = summary or not (distribution or distribution_config or origin_access_identity or
+        origin_access_identity_config or invalidation or streaming_distribution or streaming_distribution_config or
+        list_origin_access_identities or list_distributions_by_web_acl_id or list_invalidations or
+        list_streaming_distributions or list_distributions)

    # validations
    if require_distribution_id and distribution_id is None and domain_name_alias is None:
@ -444,65 +573,42 @@ def main():

    # set appropriate cloudfront id
    if distribution_id and not list_invalidations:
-        result = { 'cloudfront': { distribution_id: {} } }
+        facts = { distribution_id: {} }
        aliases = service_mgr.get_aliases_from_distribution_id(distribution_id)
        for alias in aliases:
-            result['cloudfront'].update( { alias: {} } )
+            facts.update( { alias: {} } )
        if invalidation_id:
-            result['cloudfront'].update( { invalidation_id: {} } )
-        facts = result['cloudfront']
-    elif list_invalidations:
-        result = { 'cloudfront': { 'invalidations': {} } }
-        facts = result['cloudfront']['invalidations']
+            facts.update( { invalidation_id: {} } )
+    elif distribution_id and list_invalidations:
+        facts = { distribution_id: {} }
        aliases = service_mgr.get_aliases_from_distribution_id(distribution_id)
        for alias in aliases:
-            result['cloudfront']['invalidations'].update( { alias: {} } )
+            facts.update( { alias: {} } )
    elif origin_access_identity_id:
-        result = { 'cloudfront': { origin_access_identity_id: {} } }
-        facts = result['cloudfront'][origin_access_identity_id]
+        facts = { origin_access_identity_id: {} }
    elif web_acl_id:
-        result = { 'cloudfront': { web_acl_id: {} } }
-        facts = result['cloudfront'][web_acl_id]
-    else:
-        result = { 'cloudfront': {} }
-        facts = result['cloudfront']
+        facts = { web_acl_id: {} }

    # get details based on options
    if distribution:
-        distribution_details = service_mgr.get_distribution(distribution_id)
-        facts[distribution_id].update(distribution_details)
-        for alias in aliases:
-            facts[alias].update(distribution_details)
+        facts_to_set = service_mgr.get_distribution(distribution_id)
    if distribution_config:
-        distribution_config_details = service_mgr.get_distribution_config(distribution_id)
-        facts[distribution_id].update(distribution_config_details)
-        for alias in aliases:
-            facts[alias].update(distribution_config_details)
+        facts_to_set = service_mgr.get_distribution_config(distribution_id)
    if origin_access_identity:
        facts[origin_access_identity_id].update(service_mgr.get_origin_access_identity(origin_access_identity_id))
    if origin_access_identity_config:
        facts[origin_access_identity_id].update(service_mgr.get_origin_access_identity_config(origin_access_identity_id))
    if invalidation:
-        invalidation = service_mgr.get_invalidation(distribution_id, invalidation_id)
-        facts[invalidation_id].update(invalidation)
-        facts[distribution_id].update(invalidation)
-        for alias in aliases:
-            facts[alias].update(invalidation)
+        facts_to_set = service_mgr.get_invalidation(distribution_id, invalidation_id)
+        facts[invalidation_id].update(facts_to_set)
    if streaming_distribution:
-        streaming_distribution_details = service_mgr.get_streaming_distribution(distribution_id)
-        facts[distribution_id].update(streaming_distribution_details)
-        for alias in aliases:
-            facts[alias].update(streaming_distribution_details)
+        facts_to_set = service_mgr.get_streaming_distribution(distribution_id)
    if streaming_distribution_config:
-        streaming_distribution_config_details = service_mgr.get_streaming_distribution_config(distribution_id)
-        facts[distribution_id].update(streaming_distribution_config_details)
-        for alias in aliases:
-            facts[alias].update(streaming_distribution_config_details)
+        facts_to_set = service_mgr.get_streaming_distribution_config(distribution_id)
    if list_invalidations:
-        invalidations = service_mgr.list_invalidations(distribution_id)
-        facts[distribution_id].update(invalidations)
-        for alias in aliases:
-            facts[alias].update(invalidations)
+        facts_to_set = {'invalidations': service_mgr.list_invalidations(distribution_id) }
+    if 'facts_to_set' in vars():
+        facts = set_facts_for_distribution_id_and_alias(facts_to_set, facts, distribution_id, aliases)

    # get list based on options
    if all_lists or list_origin_access_identities:
@ -513,8 +619,15 @@ def main():
        facts['streaming_distributions'] = service_mgr.list_streaming_distributions()
    if list_distributions_by_web_acl_id:
        facts['distributions_by_web_acl_id'] = service_mgr.list_distributions_by_web_acl_id(web_acl_id)
+    if list_invalidations:
+        facts['invalidations'] = service_mgr.list_invalidations(distribution_id)
+
+    # default summary option
+    if summary:
+        facts['summary'] = service_mgr.summary()

    result['changed'] = False
+    result['cloudfront'].update(facts)
    module.exit_json(msg="Retrieved cloudfront facts.", ansible_facts=result)

 if __name__ == '__main__':