mirror of https://github.com/ansible/ansible.git
parse botocore.endpoint logs into a list of AWS actions (#49312)
* Add an option to parse botocore.endpoint logs for the AWS actions performed during a task Add a callback to consolidate all AWS actions used by modules Added some documentation to the AWS guidelines * Enable aws_resource_actions callback only for AWS tests * Add script to help generate policies * Set debug_botocore_endpoint_logs via environment variable for all AWS integration tests Ensure AWS tests inherit environment (also remove AWS CLI in aws_rds inventory tests and use the module)pull/53996/head
Sloane Hertel
5 years ago
committed by
GitHub
parent
eb790cd3c6
commit
7da565b3ae
@ -0,0 +1,327 @@
|
||||
# Requires pandas, bs4, html5lib, and lxml
|
||||
#
|
||||
# Call script with the output from aws_resource_actions callback, e.g.
|
||||
# python build_iam_policy_framework.py ['ec2:AuthorizeSecurityGroupEgress', 'ec2:AuthorizeSecurityGroupIngress', 'sts:GetCallerIdentity']
|
||||
#
|
||||
# The sample output:
|
||||
# {
|
||||
# "Version": "2012-10-17",
|
||||
# "Statement": [
|
||||
# {
|
||||
# "Sid": "AnsibleEditor0",
|
||||
# "Effect": "Allow",
|
||||
# "Action": [
|
||||
# "ec2:AuthorizeSecurityGroupEgress",
|
||||
# "ec2:AuthorizeSecurityGroupIngress"
|
||||
# ],
|
||||
# "Resource": "arn:aws:ec2:${Region}:${Account}:security-group/${SecurityGroupId}"
|
||||
# },
|
||||
# {
|
||||
# "Sid": "AnsibleEditor1",
|
||||
# "Effect": "Allow",
|
||||
# "Action": [
|
||||
# "sts:GetCallerIdentity"
|
||||
# ],
|
||||
# "Resource": "*"
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
#
|
||||
# Policy troubleshooting:
|
||||
# - If there are more actions in the policy than you provided, AWS has documented dependencies for some of your actions and
|
||||
# those have been added to the policy.
|
||||
# - If there are fewer actions in the policy than you provided, some of your actions are not in the IAM table of actions for
|
||||
# that service. For example, the API call s3:DeleteObjects does not actually correlate to the permission needed in a policy.
|
||||
# In this case s3:DeleteObject is the permission required to allow both the s3:DeleteObjects action and the s3:DeleteObject action.
|
||||
# - The policies output are only as accurate as the AWS documentation. If the policy does not permit the
|
||||
# necessary actions, look for undocumented dependencies. For example, redshift:CreateCluster requires ec2:DescribeVpcs,
|
||||
# ec2:DescribeSubnets, ec2:DescribeSecurityGroups, and ec2:DescribeInternetGateways, but AWS does not document this.
|
||||
#
|
||||
|
||||
import json
|
||||
import requests
|
||||
import sys
|
||||
|
||||
missing_dependencies = []
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
missing_dependencies.append('pandas')
|
||||
try:
|
||||
import bs4
|
||||
except ImportError:
|
||||
missing_dependencies.append('bs4')
|
||||
try:
|
||||
import html5lib
|
||||
except ImportError:
|
||||
missing_dependencies.append('html5lib')
|
||||
try:
|
||||
import lxml
|
||||
except ImportError:
|
||||
missing_dependencies.append('lxml')
|
||||
|
||||
|
||||
irregular_service_names = {
|
||||
'a4b': 'alexaforbusiness',
|
||||
'appstream': 'appstream2.0',
|
||||
'acm': 'certificatemanager',
|
||||
'acm-pca': 'certificatemanagerprivatecertificateauthority',
|
||||
'aws-marketplace-management': 'marketplacemanagementportal',
|
||||
'ce': 'costexplorerservice',
|
||||
'cognito-identity': 'cognitoidentity',
|
||||
'cognito-sync': 'cognitosync',
|
||||
'cognito-idp': 'cognitouserpools',
|
||||
'cur': 'costandusagereport',
|
||||
'dax': 'dynamodbacceleratordax',
|
||||
'dlm': 'datalifecyclemanager',
|
||||
'dms': 'databasemigrationservice',
|
||||
'ds': 'directoryservice',
|
||||
'ec2messages': 'messagedeliveryservice',
|
||||
'ecr': 'ec2containerregistry',
|
||||
'ecs': 'elasticcontainerservice',
|
||||
'eks': 'elasticcontainerserviceforkubernetes',
|
||||
'efs': 'elasticfilesystem',
|
||||
'es': 'elasticsearchservice',
|
||||
'events': 'cloudwatchevents',
|
||||
'firehose': 'kinesisfirehose',
|
||||
'fms': 'firewallmanager',
|
||||
'health': 'healthapisandnotifications',
|
||||
'importexport': 'importexportdiskservice',
|
||||
'iot1click': 'iot1-click',
|
||||
'kafka': 'managedstreamingforkafka',
|
||||
'kinesisvideo': 'kinesisvideostreams',
|
||||
'kms': 'keymanagementservice',
|
||||
'license-manager': 'licensemanager',
|
||||
'logs': 'cloudwatchlogs',
|
||||
'opsworks-cm': 'opsworksconfigurationmanagement',
|
||||
'mediaconnect': 'elementalmediaconnect',
|
||||
'mediaconvert': 'elementalmediaconvert',
|
||||
'medialive': 'elementalmedialive',
|
||||
'mediapackage': 'elementalmediapackage',
|
||||
'mediastore': 'elementalmediastore',
|
||||
'mgh': 'migrationhub',
|
||||
'mobiletargeting': 'pinpoint',
|
||||
'pi': 'performanceinsights',
|
||||
'pricing': 'pricelist',
|
||||
'ram': 'resourceaccessmanager',
|
||||
'resource-groups': 'resourcegroups',
|
||||
'sdb': 'simpledb',
|
||||
'servicediscovery': 'cloudmap',
|
||||
'serverlessrepo': 'serverlessapplicationrepository',
|
||||
'sms': 'servermigrationservice',
|
||||
'sms-voice': 'pinpointsmsandvoiceservice',
|
||||
'sso-directory': 'ssodirectory',
|
||||
'ssm': 'systemsmanager',
|
||||
'ssmmessages': 'sessionmanagermessagegatewayservice',
|
||||
'states': 'stepfunctions',
|
||||
'sts': 'securitytokenservice',
|
||||
'swf': 'simpleworkflowservice',
|
||||
'tag': 'resourcegrouptaggingapi',
|
||||
'transfer': 'transferforsftp',
|
||||
'waf-regional': 'wafregional',
|
||||
'wam': 'workspacesapplicationmanager',
|
||||
'xray': 'x-ray'
|
||||
}
|
||||
|
||||
irregular_service_links = {
|
||||
'apigateway': [
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_manageamazonapigateway.html'
|
||||
],
|
||||
'aws-marketplace': [
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplace.html',
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplacemeteringservice.html',
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsprivatemarketplace.html'
|
||||
],
|
||||
'discovery': [
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_applicationdiscovery.html'
|
||||
],
|
||||
'elasticloadbalancing': [
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancing.html',
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancingv2.html'
|
||||
],
|
||||
'globalaccelerator': [
|
||||
'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_globalaccelerator.html'
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def get_docs_by_prefix(prefix):
|
||||
amazon_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazon{0}.html'
|
||||
aws_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_aws{0}.html'
|
||||
|
||||
if prefix in irregular_service_links:
|
||||
links = irregular_service_links[prefix]
|
||||
else:
|
||||
if prefix in irregular_service_names:
|
||||
prefix = irregular_service_names[prefix]
|
||||
links = [amazon_link_form.format(prefix), aws_link_form.format(prefix)]
|
||||
|
||||
return links
|
||||
|
||||
|
||||
def get_html(links):
|
||||
html_list = []
|
||||
for link in links:
|
||||
html = requests.get(link).content
|
||||
try:
|
||||
parsed_html = pd.read_html(html)
|
||||
html_list.append(parsed_html)
|
||||
except ValueError as e:
|
||||
if 'No tables found' in str(e):
|
||||
pass
|
||||
else:
|
||||
raise e
|
||||
|
||||
return html_list
|
||||
|
||||
|
||||
def get_tables(service):
|
||||
links = get_docs_by_prefix(service)
|
||||
html_list = get_html(links)
|
||||
action_tables = []
|
||||
arn_tables = []
|
||||
for df_list in html_list:
|
||||
for df in df_list:
|
||||
table = json.loads(df.to_json(orient='split'))
|
||||
table_data = table['data'][0]
|
||||
if 'Actions' in table_data and 'Resource Types (*required)' in table_data:
|
||||
action_tables.append(table['data'][1::])
|
||||
elif 'Resource Types' in table_data and 'ARN' in table_data:
|
||||
arn_tables.append(table['data'][1::])
|
||||
|
||||
# Action table indices:
|
||||
# 0: Action, 1: Description, 2: Access level, 3: Resource type, 4: Condition keys, 5: Dependent actions
|
||||
# ARN tables indices:
|
||||
# 0: Resource type, 1: ARN template, 2: Condition keys
|
||||
return action_tables, arn_tables
|
||||
|
||||
|
||||
def add_dependent_action(resources, dependency):
|
||||
resource, action = dependency.split(':')
|
||||
if resource in resources:
|
||||
resources[resource].append(action)
|
||||
else:
|
||||
resources[resource] = [action]
|
||||
return resources
|
||||
|
||||
|
||||
def get_dependent_actions(resources):
|
||||
for service in dict(resources):
|
||||
action_tables, arn_tables = get_tables(service)
|
||||
for found_action_table in action_tables:
|
||||
for action_stuff in found_action_table:
|
||||
if action_stuff is None:
|
||||
continue
|
||||
if action_stuff[0] in resources[service] and action_stuff[5]:
|
||||
dependencies = action_stuff[5].split()
|
||||
if isinstance(dependencies, list):
|
||||
for dependency in dependencies:
|
||||
resources = add_dependent_action(resources, dependency)
|
||||
else:
|
||||
resources = add_dependent_action(resources, dependencies)
|
||||
return resources
|
||||
|
||||
|
||||
def get_actions_by_service(resources):
|
||||
service_action_dict = {}
|
||||
dependencies = {}
|
||||
for service in resources:
|
||||
action_tables, arn_tables = get_tables(service)
|
||||
|
||||
# Create dict of the resource type to the corresponding ARN
|
||||
arn_dict = {}
|
||||
for found_arn_table in arn_tables:
|
||||
for arn_stuff in found_arn_table:
|
||||
arn_dict["{0}*".format(arn_stuff[0])] = arn_stuff[1]
|
||||
|
||||
# Create dict of the action to the corresponding ARN
|
||||
action_dict = {}
|
||||
for found_action_table in action_tables:
|
||||
for action_stuff in found_action_table:
|
||||
if action_stuff[0] is None:
|
||||
continue
|
||||
if arn_dict.get(action_stuff[3]):
|
||||
action_dict[action_stuff[0]] = arn_dict[action_stuff[3]]
|
||||
else:
|
||||
action_dict[action_stuff[0]] = None
|
||||
service_action_dict[service] = action_dict
|
||||
return service_action_dict
|
||||
|
||||
|
||||
def get_resource_arns(aws_actions, action_dict):
|
||||
resource_arns = {}
|
||||
for resource_action in aws_actions:
|
||||
resource, action = resource_action.split(':')
|
||||
if action not in action_dict:
|
||||
continue
|
||||
if action_dict[action] is None:
|
||||
resource = "*"
|
||||
else:
|
||||
resource = action_dict[action].replace("${Partition}", "aws")
|
||||
if resource not in resource_arns:
|
||||
resource_arns[resource] = []
|
||||
resource_arns[resource].append(resource_action)
|
||||
return resource_arns
|
||||
|
||||
|
||||
def get_resources(actions):
|
||||
resources = {}
|
||||
for action in actions:
|
||||
resource, action = action.split(':')
|
||||
if resource not in resources:
|
||||
resources[resource] = []
|
||||
resources[resource].append(action)
|
||||
return resources
|
||||
|
||||
|
||||
def combine_arn_actions(resources, service_action_arn_dict):
|
||||
arn_actions = {}
|
||||
for service in service_action_arn_dict:
|
||||
service_arn_actions = get_resource_arns(aws_actions, service_action_arn_dict[service])
|
||||
for resource in service_arn_actions:
|
||||
if resource in arn_actions:
|
||||
arn_actions[resource].extend(service_arn_actions[resource])
|
||||
else:
|
||||
arn_actions[resource] = service_arn_actions[resource]
|
||||
return arn_actions
|
||||
|
||||
|
||||
def combine_actions_and_dependent_actions(resources):
|
||||
aws_actions = []
|
||||
for resource in resources:
|
||||
for action in resources[resource]:
|
||||
aws_actions.append('{0}:{1}'.format(resource, action))
|
||||
return set(aws_actions)
|
||||
|
||||
|
||||
def get_actions_restricted_by_arn(aws_actions):
|
||||
resources = get_resources(aws_actions)
|
||||
resources = get_dependent_actions(resources)
|
||||
service_action_arn_dict = get_actions_by_service(resources)
|
||||
aws_actions = combine_actions_and_dependent_actions(resources)
|
||||
return combine_arn_actions(aws_actions, service_action_arn_dict)
|
||||
|
||||
|
||||
def main(aws_actions):
|
||||
arn_actions = get_actions_restricted_by_arn(aws_actions)
|
||||
statement = []
|
||||
for resource_restriction in arn_actions:
|
||||
statement.append({
|
||||
"Sid": "AnsibleEditor{0}".format(len(statement)),
|
||||
"Effect": "Allow",
|
||||
"Action": arn_actions[resource_restriction],
|
||||
"Resource": resource_restriction
|
||||
})
|
||||
|
||||
policy = {"Version": "2012-10-17", "Statement": statement}
|
||||
print(json.dumps(policy, indent=4))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if missing_dependencies:
|
||||
sys.exit('Missing Python libraries: {0}'.format(', '.join(missing_dependencies)))
|
||||
actions = sys.argv[1:]
|
||||
if len(actions) == 1:
|
||||
actions = sys.argv[1].split(',')
|
||||
aws_actions = [action.strip('[], "\'') for action in actions]
|
||||
main(aws_actions)
|
||||
@ -0,0 +1,72 @@
|
||||
# (C) 2018 Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
# Make coding more python3-ish
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
DOCUMENTATION = '''
|
||||
callback: aws_resource_actions
|
||||
type: aggregate
|
||||
short_description: summarizes all "resource:actions" completed
|
||||
version_added: "2.8"
|
||||
description:
|
||||
- Ansible callback plugin for collecting the AWS actions completed by all boto3 modules using
|
||||
AnsibleAWSModule in a playbook. Botocore endpoint logs need to be enabled for those modules, which can
|
||||
be done easily by setting debug_botocore_endpoint_logs to True for group/aws using module_defaults.
|
||||
requirements:
|
||||
- whitelisting in configuration - see examples section below for details.
|
||||
'''
|
||||
|
||||
EXAMPLES = '''
|
||||
example: >
|
||||
To enable, add this to your ansible.cfg file in the defaults block
|
||||
[defaults]
|
||||
callback_whitelist = aws_resource_actions
|
||||
sample output: >
|
||||
#
|
||||
# AWS ACTIONS: ['s3:PutBucketAcl', 's3:HeadObject', 's3:DeleteObject', 's3:PutObjectAcl', 's3:CreateMultipartUpload',
|
||||
# 's3:DeleteBucket', 's3:GetObject', 's3:DeleteObjects', 's3:CreateBucket', 's3:CompleteMultipartUpload',
|
||||
# 's3:ListObjectsV2', 's3:HeadBucket', 's3:UploadPart', 's3:PutObject']
|
||||
#
|
||||
sample output: >
|
||||
#
|
||||
# AWS ACTIONS: ['ec2:DescribeVpcAttribute', 'ec2:DescribeVpcClassicLink', 'ec2:ModifyVpcAttribute', 'ec2:CreateTags',
|
||||
# 'sts:GetCallerIdentity', 'ec2:DescribeSecurityGroups', 'ec2:DescribeTags', 'ec2:DescribeVpcs', 'ec2:CreateVpc']
|
||||
#
|
||||
'''
|
||||
|
||||
from ansible.plugins.callback import CallbackBase
|
||||
from ansible.module_utils._text import to_native
|
||||
|
||||
|
||||
class CallbackModule(CallbackBase):
|
||||
CALLBACK_VERSION = 2.8
|
||||
CALLBACK_TYPE = 'aggregate'
|
||||
CALLBACK_NAME = 'aws_resource_actions'
|
||||
CALLBACK_NEEDS_WHITELIST = True
|
||||
|
||||
def __init__(self):
|
||||
self.aws_resource_actions = []
|
||||
super(CallbackModule, self).__init__()
|
||||
|
||||
def extend_aws_resource_actions(self, result):
|
||||
if result.get('resource_actions'):
|
||||
self.aws_resource_actions.extend(result['resource_actions'])
|
||||
|
||||
def runner_on_ok(self, host, res):
|
||||
self.extend_aws_resource_actions(res)
|
||||
|
||||
def runner_on_failed(self, host, res, ignore_errors=False):
|
||||
self.extend_aws_resource_actions(res)
|
||||
|
||||
def v2_runner_item_on_ok(self, result):
|
||||
self.extend_aws_resource_actions(result._result)
|
||||
|
||||
def v2_runner_item_on_failed(self, result):
|
||||
self.extend_aws_resource_actions(result._result)
|
||||
|
||||
def playbook_on_stats(self, stats):
|
||||
if self.aws_resource_actions:
|
||||
self.aws_resource_actions = sorted(list(to_native(action) for action in set(self.aws_resource_actions)))
|
||||
self._display.display("AWS ACTIONS: {0}".format(self.aws_resource_actions))
|
||||
@ -1,5 +1,6 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
environment: "{{ ansible_test.environment }}"
|
||||
|
||||
roles:
|
||||
- aws_eks
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
environment: "{{ ansible_test.environment }}"
|
||||
|
||||
roles:
|
||||
- ../../cloudformation_stack_set
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
environment: "{{ ansible_test.environment }}"
|
||||
|
||||
roles:
|
||||
- ec2_instance
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
environment: "{{ ansible_test.environment }}"
|
||||
roles:
|
||||
- ec2_launch_template
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
environment: "{{ ansible_test.environment }}"
|
||||
|
||||
roles:
|
||||
- ecs_cluster
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
environment: "{{ ansible_test.environment }}"
|
||||
|
||||
roles:
|
||||
- elb_target
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
environment: "{{ ansible_test.environment }}"
|
||||
|
||||
roles:
|
||||
- elb_target_facts
|
||||
|
||||
Loading…
Reference in New Issue