aws.core in new aws dir in module utils - module with AnsibleAWSModule class and fail_json_aws (#25780)

* aws module utils including AnsibleAWSModule
* fail_json_aws method on AnsibleAWSModule to do fail_json nicely with AWS exceptions
* aws module util - feedback - rename to aws/core.py & improve doc strings

lib/ansible/module_utils/aws/core.py

@@ -0,0 +1,143 @@
+#
+#  Copyright 2017 Michael De La Rue | Ansible
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""This module adds shared support for generic Amazon AWS modules
+
+**This code is not yet ready for use in user modules.  As of 2017**
+**and through to 2018, the interface is likely to change**
+**aggressively as the exact correct interface for ansible AWS modules**
+**is identified.  In particular, until this notice goes away or is**
+**changed, methods may disappear from the interface.  Please don't**
+**publish modules using this except directly to the main Ansible**
+**development repository.**
+
+In order to use this module, include it as part of a custom
+module as shown below.
+
+  from ansible.module_utils.aws import AnsibleAWSModule
+  module = AnsibleAWSModule(argument_spec=dictionary, supports_check_mode=boolean
+                            mutually_exclusive=list1, required_together=list2)
+
+The 'AnsibleAWSModule' module provides similar, but more restricted,
+interfaces to the normal Ansible module.  It also includes the
+additional methods for connecting to AWS using the standard module arguments
+
+  try:
+      m.aws_connect(resource='lambda') # - get an AWS connection.
+  except Exception:
+      m.fail_json_aws(Exception, msg="trying to connect") # - take an exception and make a decent failure
+
+
+"""
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils._text import to_native
+from ansible.module_utils.ec2 import HAS_BOTO3, camel_dict_to_snake_dict, ec2_argument_spec
+import traceback
+
+# We will also export HAS_BOTO3 so end user modules can use it.
+HAS_BOTO3 = HAS_BOTO3
+
+
+class AnsibleAWSModule(object):
+    """An ansible module class for AWS modules
+
+    AnsibleAWSModule provides an a class for building modules which
+    connect to Amazon Web Services.  The interface is currently more
+    restricted than the basic module class with the aim that later the
+    basic module class can be reduced.  If you find that any key
+    feature is missing please contact the author/Ansible AWS team
+    (available on #ansible-aws on IRC) to request the additional
+    features needed.
+    """
+    default_settings = {
+        "default_args": True,
+        "check_boto3": True,
+        "auto_retry": True,
+        "module_class": AnsibleModule
+    }
+
+    def __init__(self, **kwargs):
+        local_settings = {}
+        for key in AnsibleAWSModule.default_settings:
+            try:
+                local_settings[key] = kwargs.pop(key)
+            except KeyError:
+                local_settings[key] = AnsibleAWSModule.default_settings[key]
+        self.settings = local_settings
+
+        if local_settings["default_args"]:
+            # ec2_argument_spec contains the region so we use that; there's a patch coming which
+            # will add it to aws_argument_spec so if that's accepted then later we should change
+            # over
+            argument_spec_full = ec2_argument_spec()
+            try:
+                argument_spec_full.update(kwargs["argument_spec"])
+            except (TypeError, NameError):
+                pass
+            kwargs["argument_spec"] = argument_spec_full
+
+        self._module = AnsibleAWSModule.default_settings["module_class"](**kwargs)
+
+        if local_settings["check_boto3"] and not HAS_BOTO3:
+            self._module.fail_json(
+                msg='Python modules "botocore" or "boto3" are missing, please install both')
+
+        self.check_mode = self._module.check_mode
+
+    @property
+    def params(self):
+        return self._module.params
+
+    def exit_json(self, *args, **kwargs):
+        return self._module.exit_json(*args, **kwargs)
+
+    def fail_json(self, *args, **kwargs):
+        return self._module.fail_json(*args, **kwargs)
+
+    def fail_json_aws(self, exception, msg=None):
+        """call fail_json with processed exception
+
+        function for converting exceptions thrown by AWS SDK modules,
+        botocore, boto3 and boto, into nice error messages.
+        """
+        last_traceback = traceback.format_exc()
+
+        # to_native is trusted to handle exceptions that str() could
+        # convert to text.
+        try:
+            except_msg = to_native(exception.message)
+        except AttributeError:
+            except_msg = to_native(exception)
+
+        if msg is not None:
+            message = '{0}: {1}'.format(msg, except_msg)
+        else:
+            message = except_msg
+
+        try:
+            response = exception.response
+        except AttributeError:
+            response = None
+
+        if response is None:
+            self._module.fail_json(msg=message, exception=last_traceback)
+        else:
+            self._module.fail_json(msg=message, exception=last_traceback,
+                                   **camel_dict_to_snake_dict(response))

test/units/module_utils/aws/test_aws_module.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+# (c) 2017, Michael De La Rue
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+from pytest import importorskip
+import unittest
+from ansible.module_utils import basic
+from ansible.module_utils.aws.core import AnsibleAWSModule
+from ansible.module_utils._text import to_bytes
+from ansible.compat.tests.mock import Mock, patch
+import json
+
+importorskip("boto3")
+botocore = importorskip("botocore")
+
+
+class AWSModuleTestCase(unittest.TestCase):
+
+    basic._ANSIBLE_ARGS = to_bytes(json.dumps({'ANSIBLE_MODULE_ARGS': {}}))
+
+    def test_create_aws_module_should_set_up_params(self):
+        m = AnsibleAWSModule(argument_spec=dict(
+            win_string_arg=dict(type='list', default=['win'])
+        ))
+        m_noretry_no_customargs = AnsibleAWSModule(
+            auto_retry=False, default_args=False,
+            argument_spec=dict(
+                success_string_arg=dict(type='list', default=['success'])
+            )
+        )
+        assert m, "module wasn't true!!"
+        assert m_noretry_no_customargs, "module wasn't true!!"
+
+        m_params = m.params
+        m_no_defs_params = m_noretry_no_customargs.params
+        assert 'region' in m_params
+        assert 'win' in m_params["win_string_arg"]
+        assert 'success' in m_no_defs_params["success_string_arg"]
+        assert 'aws_secret_key' not in m_no_defs_params
+
+
+class ErrorReportingTestcase(unittest.TestCase):
+
+    def test_botocore_exception_reports_nicely_via_fail_json_aws(self):
+
+        basic._ANSIBLE_ARGS = to_bytes(json.dumps({'ANSIBLE_MODULE_ARGS': {}}))
+        module = AnsibleAWSModule(argument_spec=dict(
+            fail_mode=dict(type='list', default=['success'])
+        ))
+
+        fail_json_double = Mock()
+        err_msg = {'Error': {'Code': 'FakeClass.FakeError'}}
+        with patch.object(basic.AnsibleModule, 'fail_json', fail_json_double):
+            try:
+                raise botocore.exceptions.ClientError(err_msg, 'Could not find you')
+            except Exception as e:
+                print("exception is " + str(e))
+                module.fail_json_aws(e, msg="Fake failure for testing boto exception messages")
+
+        assert(len(fail_json_double.mock_calls) >
+               0), "failed to call fail_json when should have"
+        assert(len(fail_json_double.mock_calls) <
+               2), "called fail_json multiple times when once would do"
+        assert("test_botocore_exception_reports_nicely"
+               in fail_json_double.mock_calls[0][2]["exception"]), \
+            "exception traceback doesn't include correct function, fail call was actually: " \
+            + str(fail_json_double.mock_calls[0])
+
+        assert("Fake failure for testing boto exception messages:"
+               in fail_json_double.mock_calls[0][2]["msg"]), \
+            "error message doesn't include the local message; was: " \
+            + str(fail_json_double.mock_calls[0])
+        assert("Could not find you" in fail_json_double.mock_calls[0][2]["msg"]), \
+            "error message doesn't include the botocore exception message; was: " \
+            + str(fail_json_double.mock_calls[0])
+        try:
+            fail_json_double.mock_calls[0][2]["error"]
+        except KeyError:
+            raise Exception("error was missing; call was: " + str(fail_json_double.mock_calls[0]))
+        assert("FakeClass.FakeError" == fail_json_double.mock_calls[0][2]["error"]["code"]), \
+            "Failed to find error/code; was: " + str(fail_json_double.mock_calls[0])
+
+    def test_botocore_exception_without_response_reports_nicely_via_fail_json_aws(self):
+        basic._ANSIBLE_ARGS = to_bytes(json.dumps({'ANSIBLE_MODULE_ARGS': {}}))
+        module = AnsibleAWSModule(argument_spec=dict(
+            fail_mode=dict(type='list', default=['success'])
+        ))
+
+        fail_json_double = Mock()
+        err_msg = None
+        with patch.object(basic.AnsibleModule, 'fail_json', fail_json_double):
+            try:
+                raise botocore.exceptions.ClientError(err_msg, 'Could not find you')
+            except Exception as e:
+                print("exception is " + str(e))
+                module.fail_json_aws(e, msg="Fake failure for testing boto exception messages")
+
+        assert(len(fail_json_double.mock_calls) > 0), "failed to call fail_json when should have"
+        assert(len(fail_json_double.mock_calls) < 2), "called fail_json multiple times"
+
+        assert("test_botocore_exception_without_response_reports_nicely_via_fail_json_aws"
+               in fail_json_double.mock_calls[0][2]["exception"]), \
+            "exception traceback doesn't include correct function, fail call was actually: " \
+            + str(fail_json_double.mock_calls[0])
+
+        assert("Fake failure for testing boto exception messages"
+               in fail_json_double.mock_calls[0][2]["msg"]), \
+            "error message doesn't include the local message; was: " \
+            + str(fail_json_double.mock_calls[0])
+
+        # I would have thought this should work, however the botocore exception comes back with
+        # "argument of type 'NoneType' is not iterable" so it's probably not really designed
+        # to handle "None" as an error response.
+        #
+        # assert("Could not find you" in fail_json_double.mock_calls[0][2]["msg"]), \
+        #    "error message doesn't include the botocore exception message; was: " \
+        #    + str(fail_json_double.mock_calls[0])
+
+
+# TODO:
+#  - an exception without a message
+#  - plain boto exception
+#  - socket errors and other standard things.