Creates base Sophos UTM module (#45781)

* Fixes #18568
* Commit of the first set of utm modules
* added documentation line for module_utils file
* removed other utm modules for the first pr
* added maintainers to botmeta
* implemented fixes for shippable
* fixed whitespaces and newlines in included doc fragment
* added types and choices to documentation
* fix for E501
* Implemented change requests
* changed utm_utils license to BSD
* changed str() to to_native()
* added a status state that will just return information about my object
* renamed state 'status' to 'info'
* added team_e-spirit to botmeta and added the team as maintainer for the utm_utils
* only return a result if the lookup was not empty. Do not return a null result
* removed info state
* added boilerplate
* made preparation for info-only modules

.github/BOTMETA.yml

@@ -353,6 +353,11 @@ files:
   $modules/utilities/logic/pause.py: samdoran
   $modules/utilities/logic/wait_for.py: gregswift
   $modules/web_infrastructure/ansible_tower/: $team_tower
+  $modules/web_infrastructure/sophos_utm/:
+    maintainers: $team_e-spirit
+    keywords:
+      - utm
+      - sophos
   $modules/web_infrastructure/django_manage.py: scottanderson42
   $modules/web_infrastructure/htpasswd.py: $team_ansible
   $modules/web_infrastructure/jboss: $team_jboss
@@ -604,6 +609,8 @@ files:
     maintainers: $team_vultr
     labels:
       - cloud
+  $module_utils/utm_utils.py:
+    maintainers: $team_e-spirit
   $module_utils/vmware:
     support: core
     maintainers: $team_vmware
@@ -1126,6 +1133,7 @@ macros:
   team_scaleway: sieben hekonsek Spredzy abarbare anthony25 pilou-
   team_tower: ghjm jlaska matburt wwitzel3 simfarm ryanpetrello rooftopcellist AlanCoding
   team_ucs: dsoper2 johnamcdonough SDBrett vallard vvb dagwieers
+  team_e-spirit: MatrixCrawler getjack
   team_vmware: Akasurde dav1x warthog9
   team_vultr: resmo Spredzy
   team_windows: dagwieers jborean93 jhawkesworth nitzmahone

docs/docsite/rst/dev_guide/developing_module_utilities.rst

@@ -75,5 +75,6 @@ The following is a list of ``module_utils`` files and a general description. The
 - six/__init__.py - Bundled copy of the `Six Python library <https://pythonhosted.org/six/>`_ to aid in writing code compatible with both Python 2 and Python 3.
 - splitter.py - String splitting and manipulation utilities for working with Jinja2 templates
 - urls.py - Utilities for working with http and https requests
+- utm_utils.py - Contains base class for creating new Sophos UTM Modules and helper functions for handling the rest interface of Sophos UTM
 - vca.py - Contains utilities for modules that work with VMware vCloud Air
 - vmware.py - Contains utilities for modules that work with VMware vSphere VMs

lib/ansible/module_utils/utm_utils.py

@@ -0,0 +1,218 @@
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Copyright: (c) 2018, Johannes Brunswicker <johannes.brunswicker@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+import json
+
+from ansible.module_utils._text import to_native
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.urls import fetch_url
+
+
+class UTMModuleConfigurationError(Exception):
+
+    def __init__(self, msg, **args):
+        super(UTMModuleConfigurationError, self).__init__(self, msg)
+        self.msg = msg
+        self.module_fail_args = args
+
+    def do_fail(self, module):
+        module.fail_json(msg=self.msg, other=self.module_fail_args)
+
+
+class UTMModule(AnsibleModule):
+    """
+    This is a helper class to construct any UTM Module. This will automatically add the utm host, port, token,
+    protocol, validate_certs and state field to the module. If you want to implement your own sophos utm module
+    just initialize this UTMModule class and define the Payload fields that are needed for your module.
+    See the other modules like utm_aaa_group for example.
+    """
+
+    def __init__(self, argument_spec, bypass_checks=False, no_log=False, check_invalid_arguments=None,
+                 mutually_exclusive=None, required_together=None, required_one_of=None, add_file_common_args=False,
+                 supports_check_mode=False, required_if=None):
+        default_specs = dict(
+            utm_host=dict(type='str', required=True),
+            utm_port=dict(type='int', default=4444),
+            utm_token=dict(type='str', required=True, no_log=True),
+            utm_protocol=dict(type='str', required=False, default="https", choices=["https", "http"]),
+            validate_certs=dict(type='bool', required=False, default=True),
+            state=dict(default='present', choices=['present', 'absent', 'info'])
+        )
+        super(UTMModule, self).__init__(self._merge_specs(default_specs, argument_spec), bypass_checks, no_log,
+                                        check_invalid_arguments, mutually_exclusive, required_together, required_one_of,
+                                        add_file_common_args, supports_check_mode, required_if)
+
+    def _merge_specs(self, default_specs, custom_specs):
+        result = default_specs.copy()
+        result.update(custom_specs)
+        return result
+
+
+class UTM:
+
+    def __init__(self, module, endpoint, change_relevant_keys, info_only=False):
+        """
+        Initialize UTM Class
+        :param module: The Ansible module
+        :param endpoint: The corresponding endpoint to the module
+        :param change_relevant_keys: The keys of the object to check for changes
+        :param info_only: When implementing an info module, set this to true. Will allow access to the info method only
+        """
+        self.info_only = info_only
+        self.module = module
+        self.request_url = module.params.get('utm_protocol') + "://" + module.params.get('utm_host') + ":" + to_native(
+            module.params.get('utm_port')) + "/api/objects/" + endpoint + "/"
+
+        """
+        The change_relevant_keys will be checked for changes to determine whether the object needs to be updated
+        """
+        self.change_relevant_keys = change_relevant_keys
+        self.module.params['url_username'] = 'token'
+        self.module.params['url_password'] = module.params.get('utm_token')
+        if all(elem in self.change_relevant_keys for elem in module.params.keys()):
+            raise UTMModuleConfigurationError(
+                "The keys " + to_native(
+                    self.change_relevant_keys) + " to check are not in the modules keys:\n" + to_native(
+                    module.params.keys()))
+
+    def execute(self):
+        try:
+            if not self.info_only:
+                if self.module.params.get('state') == 'present':
+                    self._add()
+                elif self.module.params.get('state') == 'absent':
+                    self._remove()
+            else:
+                self._info()
+        except Exception as e:
+            self.module.fail_json(msg=to_native(e))
+
+    def _info(self):
+        """
+        returns the info for an object in utm
+        """
+        info, result = self._lookup_entry(self.module, self.request_url)
+        if info["status"] >= 400:
+            self.module.fail_json(result=json.loads(info))
+        else:
+            if result is None:
+                self.module.exit_json(changed=False)
+            else:
+                self.module.exit_json(result=result, changed=False)
+
+    def _add(self):
+        """
+        adds or updates a host object on utm
+        """
+        is_changed = False
+        info, result = self._lookup_entry(self.module, self.request_url)
+        if info["status"] >= 400:
+            self.module.fail_json(result=json.loads(info))
+        else:
+            data_as_json_string = self.module.jsonify(self.module.params)
+            if result is None:
+                response, info = fetch_url(self.module, self.request_url, method="POST",
+                                           headers={"Accept": "application/json", "Content-type": "application/json"},
+                                           data=data_as_json_string)
+                if info["status"] >= 400:
+                    self.module.fail_json(msg=json.loads(info["body"]))
+                is_changed = True
+                result = self._clean_result(json.loads(response.read()))
+            else:
+                if self._is_object_changed(self.change_relevant_keys, self.module, result):
+                    response, info = fetch_url(self.module, self.request_url + result['_ref'], method="PUT",
+                                               headers={"Accept": "application/json",
+                                                        "Content-type": "application/json"},
+                                               data=data_as_json_string)
+                    if info['status'] >= 400:
+                        self.module.fail_json(msg=json.loads(info["body"]))
+                    is_changed = True
+                    result = self._clean_result(json.loads(response.read()))
+            self.module.exit_json(result=result, changed=is_changed)
+
+    def _remove(self):
+        """
+        removes an object from utm
+        """
+        is_changed = False
+        info, result = self._lookup_entry(self.module, self.request_url)
+        if result is not None:
+            response, info = fetch_url(self.module, self.request_url + result['_ref'], method="DELETE",
+                                       headers={"Accept": "application/json", "X-Restd-Err-Ack": "all"},
+                                       data=self.module.jsonify(self.module.params))
+            if info["status"] >= 400:
+                self.module.fail_json(msg=json.loads(info["body"]))
+            else:
+                is_changed = True
+        self.module.exit_json(changed=is_changed)
+
+    def _lookup_entry(self, module, request_url):
+        """
+        Lookup for existing entry
+        :param module:
+        :param request_url:
+        :return:
+        """
+        response, info = fetch_url(module, request_url, method="GET", headers={"Accept": "application/json"})
+        result = None
+        if response is not None:
+            results = json.loads(response.read())
+            result = next(iter(filter(lambda d: d['name'] == module.params.get('name'), results)), None)
+        return info, result
+
+    def _clean_result(self, result):
+        """
+        Will clean the result from irrelevant fields
+        :param result: The result from the query
+        :return: The modified result
+        """
+        del result['utm_host']
+        del result['utm_port']
+        del result['utm_token']
+        del result['utm_protocol']
+        del result['validate_certs']
+        del result['url_username']
+        del result['url_password']
+        del result['state']
+        return result
+
+    def _is_object_changed(self, keys, module, result):
+        """
+        Check if my object is changed
+        :param keys: The keys that will determine if an object is changed
+        :param module: The module
+        :param result: The result from the query
+        :return:
+        """
+        for key in keys:
+            if module.params.get(key) != result[key]:
+                return True
+        return False

lib/ansible/modules/web_infrastructure/sophos_utm/utm_dns_host.py

@@ -0,0 +1,156 @@
+#!/usr/bin/python
+
+# Copyright: (c) 2018, Johannes Brunswicker <johannes.brunswicker@gmail.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+ANSIBLE_METADATA = {
+    'metadata_version': '1.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+DOCUMENTATION = """
+---
+module: utm_dns_host
+
+author:
+    - Johannes Brunswicker (@MatrixCrawler)
+
+short_description: create, update or destroy dns entry in Sophos UTM
+
+description:
+    - Create, update or destroy a dns entry in SOPHOS UTM.
+    - This module needs to have the REST Ability of the UTM to be activated.
+
+version_added: "2.8"
+
+options:
+    name:
+        description:
+          - The name of the object. Will be used to identify the entry
+        required: true
+    address:
+        description:
+          - The IPV4 Address of the entry. Can be left empty for automatic resolving.
+        default: 0.0.0.0
+    address6:
+        description:
+          - The IPV6 Address of the entry. Can be left empty for automatic resolving.
+        default: "::"
+    comment:
+        description:
+          - An optional comment to add to the dns host object
+    hostname:
+        description:
+          - The hostname for the dns host object
+    interface:
+        description:
+          - The reference name of the interface to use. If not provided the default interface will be used
+    resolved:
+        description:
+          - whether the hostname's ipv4 address is already resolved or not
+        default: False
+        type: bool
+    resolved6:
+        description:
+          - whether the hostname's ipv6 address is already resolved or not
+        default: False
+        type: bool
+    timeout:
+        description:
+          - the timeout for the utm to resolve the ip address for the hostname again
+        default: 0
+
+extends_documentation_fragment:
+    - utm
+"""
+
+EXAMPLES = """
+- name: Create UTM dns host entry
+  utm_dns_host:
+    utm_host: sophos.host.name
+    utm_token: abcdefghijklmno1234
+    name: TestDNSEntry
+    hostname: testentry.some.tld
+    state: present
+
+- name: Remove UTM dns host entry
+  utm_dns_host:
+    utm_host: sophos.host.name
+    utm_token: abcdefghijklmno1234
+    name: TestDNSEntry
+    state: absent
+"""
+
+RETURN = """
+result:
+    description: The utm object that was created
+    returned: success
+    type: complex
+    contains:
+        _ref:
+            description: The reference name of the object
+            type: string
+        _locked:
+            description: Whether or not the object is currently locked
+            type: boolean
+        name:
+            description: The name of the object
+            type: string
+        address:
+            description: The ipv4 address of the object
+            type: string
+        address6:
+            description: The ipv6 adress of the object
+            type: string
+        comment:
+            description: The comment string
+            type: string
+        hostname:
+            description: The hostname of the object
+            type: string
+        interface:
+            description: The reference name of the interface the object is associated with
+            type: string
+        resolved:
+            description: Whether the ipv4 address is resolved or not
+            type: boolean
+        resolved6:
+            description: Whether the ipv6 address is resolved or not
+            type: boolean
+        timeout:
+            description: The timeout until a new resolving will be attempted
+            type: int
+"""
+
+from ansible.module_utils.utm_utils import UTM, UTMModule
+from ansible.module_utils._text import to_native
+
+
+def main():
+    endpoint = "network/dns_host"
+    key_to_check_for_changes = ["comment", "hostname", "interface"]
+    module = UTMModule(
+        argument_spec=dict(
+            name=dict(type='str', required=True),
+            address=dict(type='str', required=False, default='0.0.0.0'),
+            address6=dict(type='str', required=False, default='::'),
+            comment=dict(type='str', required=False, default=""),
+            hostname=dict(type='str', required=False),
+            interface=dict(type='str', required=False, default=""),
+            resolved=dict(type='bool', required=False, default=False),
+            resolved6=dict(type='bool', required=False, default=False),
+            timeout=dict(type='int', required=False, default=0),
+        )
+    )
+    try:
+        UTM(module, endpoint, key_to_check_for_changes).execute()
+    except Exception as e:
+        module.fail_json(msg=to_native(e))
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/utils/module_docs_fragments/utm.py

@@ -0,0 +1,44 @@
+# Copyright: (c) 2018, Johannes Brunswicker <johannes.brunswicker@gmail.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+
+class ModuleDocFragment(object):
+    DOCUMENTATION = """
+options:
+    utm_host:
+        description:
+          - The REST Endpoint of the Sophos UTM.
+        required: true
+    utm_port:
+        description:
+          - The port of the REST interface.
+        default: 4444
+    utm_token:
+        description:
+          - "The token used to identify at the REST-API. See U(https://www.sophos.com/en-us/medialibrary/\
+            PDFs/documentation/UTMonAWS/Sophos-UTM-RESTful-API.pdf?la=en), Chapter 2.4.2."
+        required: true
+    utm_protocol:
+        description:
+          - The protocol of the REST Endpoint.
+        choices:
+          - https
+          - http
+        default: https
+    validate_certs:
+        description:
+          - Whether the REST interface's ssl certificate should be verified or not.
+        default: True
+        type: bool
+    state:
+        description:
+          - The desired state of the object.
+          - C(present) will create or update an object
+          - C(absent) will delete an object if it was present
+          - C(info) will return the object details
+        choices:
+          - present
+          - absent
+          - info
+        default: present
+"""