Feature/aws helper function for tags (#23387)

* Add new helper function for comparing AWS tag key pair dicts. Also modify boto3_tag_list_to_ansible_dict function to be more generic when looking for key names because AWS sometimes uses 'Key', sometimes 'TagKey' and who knows what the future holds! Fixed modules to work with changes.

* Review changes

* Add some more doc to GUIDELINES for tags and fix var name for snaked values in ec2_group_facts

lib/ansible/module_utils/ec2.py

@@ -415,11 +415,13 @@ def ansible_dict_to_boto3_filter_list(filters_dict):
     return filters_list
 
 
-def boto3_tag_list_to_ansible_dict(tags_list):
+def boto3_tag_list_to_ansible_dict(tags_list, tag_name_key_name='Key', tag_value_key_name='Value'):
 
     """ Convert a boto3 list of resource tags to a flat dict of key:value pairs
     Args:
         tags_list (list): List of dicts representing AWS tags.
+        tag_name_key_name (str): Value to use as the key for all tag keys (useful because boto3 doesn't always use "Key")
+        tag_value_key_name (str): Value to use as the key for all tag values (useful because boto3 doesn't always use "Value")
     Basic Usage:
         >>> tags_list = [{'Key': 'MyTagKey', 'Value': 'MyTagValue'}]
         >>> boto3_tag_list_to_ansible_dict(tags_list)
@@ -438,19 +440,19 @@ def boto3_tag_list_to_ansible_dict(tags_list):
 
     tags_dict = {}
     for tag in tags_list:
-        if 'key' in tag:
-            tags_dict[tag['key']] = tag['value']
-        elif 'Key' in tag:
-            tags_dict[tag['Key']] = tag['Value']
+        if tag_name_key_name in tag:
+            tags_dict[tag[tag_name_key_name]] = tag[tag_value_key_name]
 
     return tags_dict
 
 
-def ansible_dict_to_boto3_tag_list(tags_dict):
+def ansible_dict_to_boto3_tag_list(tags_dict, tag_name_key_name='Key', tag_value_key_name='Value'):
 
     """ Convert a flat dict of key:value pairs representing AWS resource tags to a boto3 list of dicts
     Args:
         tags_dict (dict): Dict representing AWS resource tags.
+        tag_name_key_name (str): Value to use as the key for all tag keys (useful because boto3 doesn't always use "Key")
+        tag_value_key_name (str): Value to use as the key for all tag values (useful because boto3 doesn't always use "Value")
     Basic Usage:
         >>> tags_dict = {'MyTagKey': 'MyTagValue'}
         >>> ansible_dict_to_boto3_tag_list(tags_dict)
@@ -469,7 +471,7 @@ def ansible_dict_to_boto3_tag_list(tags_dict):
 
     tags_list = []
     for k,v in tags_dict.items():
-        tags_list.append({'Key': k, 'Value': v})
+        tags_list.append({tag_name_key_name: k, tag_value_key_name: v})
 
     return tags_list
 
@@ -623,3 +625,29 @@ def map_complex_type(complex_type, type_map):
     elif type_map:
         return globals()['__builtins__'][type_map](complex_type)
     return new_type
+
+
+def compare_aws_tags(current_tags_dict, new_tags_dict, purge_tags=True):
+    """
+    Compare two dicts of AWS tags. Dicts are expected to of been created using 'boto3_tag_list_to_ansible_dict' helper function.
+    Two dicts are returned - the first is tags to be set, the second is any tags to remove
+
+    :param current_tags_dict:
+    :param new_tags_dict:
+    :param purge_tags:
+    :return: tag_key_value_pairs_to_set: a dict of key value pairs that need to be set in AWS. If all tags are identical this dict will be empty
+    :return: tag_keys_to_unset: a list of key names that need to be unset in AWS. If no tags need to be unset this list will be empty
+    """
+
+    tag_key_value_pairs_to_set = {}
+    tag_keys_to_unset = []
+
+    for key in current_tags_dict.keys():
+        if key not in new_tags_dict and purge_tags:
+            tag_keys_to_unset.append(key)
+
+    for key in set(new_tags_dict.keys()) - set(tag_keys_to_unset):
+        if new_tags_dict[key] != current_tags_dict.get(key):
+            tag_key_value_pairs_to_set[key] = new_tags_dict[key]
+
+    return tag_key_value_pairs_to_set, tag_keys_to_unset

lib/ansible/modules/cloud/amazon/GUIDELINES.md

@@ -255,6 +255,19 @@ else:
     aws_object.set_policy(user_policy)
 ```
 
+### Dealing with tags
+
+AWS has a concept of resource tags. Usually the boto3 API has separate calls for tagging and
+untagging a resource.  For example, the ec2 API has a create_tags and delete_tags call.
+
+It is common practice in Ansible AWS modules to have a 'purge_tags' parameter that defaults to true.
+
+The purge_tags parameter means that existing tags will be deleted if they are not specified in
+by the Ansible playbook.
+
+There is a helper function 'compare_aws_tags' to ease dealing with tags. It can compare two dicts and
+return the tags to set and the tags to delete.  See the Helper function section below for more detail.
+
 ### Helper functions
 
 Along with the connection functions in Ansible ec2.py module_utils, there are some other useful functions detailed below.
@@ -272,12 +285,15 @@ any boto3 _facts modules.
 #### boto3_tag_list_to_ansible_dict
 
 Converts a boto3 tag list to an Ansible dict. Boto3 returns tags as a list of dicts containing keys called
-'Key' and 'Value'. This function converts this list in to a single dict where the dict key is the tag
-key and the dict value is the tag value.
+'Key' and 'Value' by default.  This key names can be overriden when calling the function.  For example, if you have already
+camel_cased your list of tags you may want to pass lowercase key names instead i.e. 'key' and 'value'.
+
+This function converts the list in to a single dict where the dict key is the tag key and the dict value is the tag value.
 
 #### ansible_dict_to_boto3_tag_list
 
-Opposite of above. Converts an Ansible dict to a boto3 tag list of dicts.
+Opposite of above. Converts an Ansible dict to a boto3 tag list of dicts. You can again override the key names used if 'Key'
+and 'Value' is not suitable.
 
 #### get_ec2_security_group_ids_from_names
 
@@ -290,3 +306,12 @@ across VPCs.
 Pass any JSON policy dict to this function in order to sort any list contained therein. This is useful
 because AWS rarely return lists in the same order that they were submitted so without this function, comparison
 of identical policies returns false.
+
+### compare_aws_tags
+
+Pass two dicts of tags and an optional purge parameter and this function will return a dict containing key pairs you need
+to modify and a list of tag key names that you need to remove.  Purge is True by default.  If purge is False then any
+existing tags will not be modified.
+
+This function is useful when using boto3 'add_tags' and 'remove_tags' functions. Be sure to use the other helper function
+'boto3_tag_list_to_ansible_dict' to get an appropriate tag dict before calling this function.

lib/ansible/modules/cloud/amazon/ec2_eni_facts.py

@@ -73,7 +73,7 @@ from ansible.module_utils.ec2 import (AnsibleAWSError,
         connect_to_aws, ec2_argument_spec, get_aws_connection_info)
 
 
-def list_ec2_snapshots_boto3(connection, module):
+def list_ec2_eni_boto3(connection, module):
 
     if module.params.get("filters") is None:
         filters = []
@@ -81,16 +81,17 @@ def list_ec2_snapshots_boto3(connection, module):
         filters = ansible_dict_to_boto3_filter_list(module.params.get("filters"))
 
     try:
-        network_interfaces_result = connection.describe_network_interfaces(Filters=filters)
+        network_interfaces_result = connection.describe_network_interfaces(Filters=filters)['NetworkInterfaces']
     except (ClientError, NoCredentialsError) as e:
         module.fail_json(msg=e.message)
 
-    # Turn the boto3 result in to ansible_friendly_snaked_names
-    snaked_network_interfaces_result = camel_dict_to_snake_dict(network_interfaces_result)
-    for network_interfaces in snaked_network_interfaces_result['network_interfaces']:
-        network_interfaces['tag_set'] = boto3_tag_list_to_ansible_dict(network_interfaces['tag_set'])
+    # Modify boto3 tags list to be ansible friendly dict and then camel_case
+    camel_network_interfaces = []
+    for network_interface in network_interfaces_result:
+        network_interface['TagSet'] = boto3_tag_list_to_ansible_dict(network_interface['TagSet'])
+        camel_network_interfaces.append(camel_dict_to_snake_dict(network_interface))
 
-    module.exit_json(**snaked_network_interfaces_result)
+    module.exit_json(network_interfaces=camel_network_interfaces)
 
 
 def get_eni_info(interface):
@@ -168,7 +169,7 @@ def main():
         else:
             module.fail_json(msg="region must be specified")
 
-        list_ec2_snapshots_boto3(connection, module)
+        list_ec2_eni_boto3(connection, module)
     else:
         region, ec2_url, aws_connect_params = get_aws_connection_info(module)
 

lib/ansible/modules/cloud/amazon/ec2_group_facts.py

@@ -153,16 +153,12 @@ def main():
     except ClientError as e:
         module.fail_json(msg=e.message, exception=traceback.format_exc())
 
-    # Turn the boto3 result in to ansible_friendly_snaked_names
+    # Modify boto3 tags list to be ansible friendly dict and then camel_case
     snaked_security_groups = []
     for security_group in security_groups['SecurityGroups']:
+        security_group['Tags'] = boto3_tag_list_to_ansible_dict(security_group['Tags'])
         snaked_security_groups.append(camel_dict_to_snake_dict(security_group))
 
-    # Turn the boto3 result in to ansible friendly tag dictionary
-    for security_group in snaked_security_groups:
-        if 'tags' in security_group:
-            security_group['tags'] = boto3_tag_list_to_ansible_dict(security_group['tags'])
-
     module.exit_json(security_groups=snaked_security_groups)
 
 

lib/ansible/modules/cloud/amazon/ec2_snapshot_facts.py

@@ -210,7 +210,7 @@ def list_ec2_snapshots(connection, module):
     # Turn the boto3 result in to ansible friendly tag dictionary
     for snapshot in snaked_snapshots:
         if 'tags' in snapshot:
-            snapshot['tags'] = boto3_tag_list_to_ansible_dict(snapshot['tags'])
+            snapshot['tags'] = boto3_tag_list_to_ansible_dict(snapshot['tags'], 'key', 'value')
 
     module.exit_json(snapshots=snaked_snapshots)
 

lib/ansible/modules/cloud/amazon/ec2_vpc_nacl_facts.py

@@ -132,7 +132,7 @@ def list_ec2_vpc_nacls(connection, module):
     # Turn the boto3 result in to ansible friendly tag dictionary
     for nacl in snaked_nacls:
         if 'tags' in nacl:
-            nacl['tags'] = boto3_tag_list_to_ansible_dict(nacl['tags'])
+            nacl['tags'] = boto3_tag_list_to_ansible_dict(nacl['tags'], 'key', 'value')
         if 'entries' in nacl:
             nacl['egress'] = [nacl_entry_to_list(e) for e in nacl['entries']
                               if e['rule_number'] != 32767 and e['egress']]