New module: routeros_facts – collect facts from RouterOS devices (#46114)

* feat(routeros): implement routeros_facts module

* fix(routeros): review by felixfontein

* fix(routeros): review by ganeshrn

* fix(routeros): review by felixfontein

lib/ansible/modules/network/routeros/routeros_facts.py

@@ -0,0 +1,435 @@
+#!/usr/bin/python
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'community'}
+
+
+DOCUMENTATION = """
+---
+module: routeros_facts
+version_added: "2.8"
+author: "Egor Zaitsev (@heuels)"
+short_description: Collect facts from remote devices running MikroTik RouterOS
+description:
+  - Collects a base set of device facts from a remote device that
+    is running RotuerOS.  This module prepends all of the
+    base network fact keys with C(ansible_net_<fact>).  The facts
+    module will always collect a base set of facts from the device
+    and can enable or disable collection of additional facts.
+options:
+  gather_subset:
+    description:
+      - When supplied, this argument will restrict the facts collected
+        to a given subset.  Possible values for this argument include
+        C(all), C(hardware), C(config), and C(interfaces).  Can specify a list of
+        values to include a larger subset.  Values can also be used
+        with an initial C(!) to specify that a specific subset should
+        not be collected.
+    required: false
+    default: '!config'
+"""
+
+EXAMPLES = """
+# Collect all facts from the device
+- routeros_facts:
+    gather_subset: all
+
+# Collect only the config and default facts
+- routeros_facts:
+    gather_subset:
+      - config
+
+# Do not collect hardware facts
+- routeros_facts:
+    gather_subset:
+      - "!hardware"
+"""
+
+RETURN = """
+ansible_net_gather_subset:
+  description: The list of fact subsets collected from the device
+  returned: always
+  type: list
+
+# default
+ansible_net_model:
+  description: The model name returned from the device
+  returned: always
+  type: string
+ansible_net_serialnum:
+  description: The serial number of the remote device
+  returned: always
+  type: string
+ansible_net_version:
+  description: The operating system version running on the remote device
+  returned: always
+  type: string
+ansible_net_hostname:
+  description: The configured hostname of the device
+  returned: always
+  type: string
+
+# hardware
+ansible_net_spacefree_mb:
+  description: The available disk space on the remote device in MiB
+  returned: when hardware is configured
+  type: dict
+ansible_net_spacetotal_mb:
+  description: The total disk space on the remote device in MiB
+  returned: when hardware is configured
+  type: dict
+ansible_net_memfree_mb:
+  description: The available free memory on the remote device in MiB
+  returned: when hardware is configured
+  type: int
+ansible_net_memtotal_mb:
+  description: The total memory on the remote device in MiB
+  returned: when hardware is configured
+  type: int
+
+# config
+ansible_net_config:
+  description: The current active config from the device
+  returned: when config is configured
+  type: string
+
+# interfaces
+ansible_net_all_ipv4_addresses:
+  description: All IPv4 addresses configured on the device
+  returned: when interfaces is configured
+  type: list
+ansible_net_all_ipv6_addresses:
+  description: All IPv6 addresses configured on the device
+  returned: when interfaces is configured
+  type: list
+ansible_net_interfaces:
+  description: A hash of all interfaces running on the system
+  returned: when interfaces is configured
+  type: dict
+ansible_net_neighbors:
+  description: The list of neighbors from the remote device
+  returned: when interfaces is configured
+  type: dict
+"""
+import re
+
+from ansible.module_utils.network.routeros.routeros import run_commands
+from ansible.module_utils.network.routeros.routeros import routeros_argument_spec
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.six import iteritems
+
+
+class FactsBase(object):
+
+    COMMANDS = list()
+
+    def __init__(self, module):
+        self.module = module
+        self.facts = dict()
+        self.responses = None
+
+    def populate(self):
+        self.responses = run_commands(self.module, commands=self.COMMANDS, check_rc=False)
+
+    def run(self, cmd):
+        return run_commands(self.module, commands=cmd, check_rc=False)
+
+
+class Default(FactsBase):
+
+    COMMANDS = [
+        '/system identity print without-paging',
+        '/system resource print without-paging',
+        '/system routerboard print without-paging'
+    ]
+
+    def populate(self):
+        super(Default, self).populate()
+        data = self.responses[0]
+        if data:
+            self.facts['hostname'] = self.parse_hostname(data)
+
+        data = self.responses[1]
+        if data:
+            self.facts['version'] = self.parse_version(data)
+
+        data = self.responses[2]
+        if data:
+            self.facts['model'] = self.parse_model(data)
+            self.facts['serialnum'] = self.parse_serialnum(data)
+
+    def parse_hostname(self, data):
+        match = re.search(r'name:\s(.*)\s*$', data, re.M)
+        if match:
+            return match.group(1)
+
+    def parse_version(self, data):
+        match = re.search(r'version:\s(.*)\s*$', data, re.M)
+        if match:
+            return match.group(1)
+
+    def parse_model(self, data):
+        match = re.search(r'model:\s(.*)\s*$', data, re.M)
+        if match:
+            return match.group(1)
+
+    def parse_serialnum(self, data):
+        match = re.search(r'serial-number:\s(.*)\s*$', data, re.M)
+        if match:
+            return match.group(1)
+
+
+class Hardware(FactsBase):
+
+    COMMANDS = [
+        '/system resource print without-paging'
+    ]
+
+    def populate(self):
+        super(Hardware, self).populate()
+        data = self.responses[0]
+        if data:
+            self.parse_filesystem_info(data)
+            self.parse_memory_info(data)
+
+    def parse_filesystem_info(self, data):
+        match = re.search(r'free-hdd-space:\s(.*)([KMG]iB)', data, re.M)
+        if match:
+            self.facts['spacefree_mb'] = self.to_megabytes(match)
+        match = re.search(r'total-hdd-space:\s(.*)([KMG]iB)', data, re.M)
+        if match:
+            self.facts['spacetotal_mb'] = self.to_megabytes(match)
+
+    def parse_memory_info(self, data):
+        match = re.search(r'free-memory:\s(\d+\.?\d*)([KMG]iB)', data, re.M)
+        if match:
+            self.facts['memfree_mb'] = self.to_megabytes(match)
+        match = re.search(r'total-memory:\s(\d+\.?\d*)([KMG]iB)', data, re.M)
+        if match:
+            self.facts['memtotal_mb'] = self.to_megabytes(match)
+
+    def to_megabytes(self, data):
+        if data.group(2) == 'KiB':
+            return float(data.group(1)) / 1024
+        elif data.group(2) == 'MiB':
+            return float(data.group(1))
+        elif data.group(2) == 'GiB':
+            return float(data.group(1)) * 1024
+        else:
+            return None
+
+
+class Config(FactsBase):
+
+    COMMANDS = ['/export']
+
+    def populate(self):
+        super(Config, self).populate()
+        data = self.responses[0]
+        if data:
+            self.facts['config'] = data
+
+
+class Interfaces(FactsBase):
+
+    COMMANDS = [
+        '/interface print detail without-paging',
+        '/ip address print detail without-paging',
+        '/ipv6 address print detail without-paging',
+        '/ip neighbor print detail without-paging'
+    ]
+
+    DETAIL_RE = re.compile(r'([\w\d\-]+)=\"?(\w{3}/\d{2}/\d{4}\s\d{2}:\d{2}:\d{2}|[\w\d\-\.:/]+)')
+
+    def populate(self):
+        super(Interfaces, self).populate()
+
+        self.facts['interfaces'] = dict()
+        self.facts['all_ipv4_addresses'] = list()
+        self.facts['all_ipv6_addresses'] = list()
+        self.facts['neighbors'] = dict()
+
+        data = self.responses[0]
+        if data:
+            interfaces = self.parse_interfaces(data)
+            self.populate_interfaces(interfaces)
+
+        data = self.responses[1]
+        if data:
+            data = self.parse_addresses(data)
+            self.populate_ipv4_interfaces(data)
+
+        data = self.responses[2]
+        if data:
+            data = self.parse_addresses(data)
+            self.populate_ipv6_interfaces(data)
+
+        data = self.responses[3]
+        if data:
+            self.facts['neighbors'] = self.parse_neighbors(data)
+
+    def populate_interfaces(self, data):
+        for key, value in iteritems(data):
+            self.facts['interfaces'][key] = value
+
+    def populate_ipv4_interfaces(self, data):
+        for key, value in iteritems(data):
+            if 'ipv4' not in self.facts['interfaces'][key]:
+                self.facts['interfaces'][key]['ipv4'] = list()
+            addr, subnet = value['address'].split("/")
+            ipv4 = dict(address=addr.strip(), subnet=subnet.strip())
+            self.add_ip_address(addr.strip(), 'ipv4')
+            self.facts['interfaces'][key]['ipv4'].append(ipv4)
+
+    def populate_ipv6_interfaces(self, data):
+        for key, value in iteritems(data):
+            if 'ipv6' not in self.facts['interfaces'][key]:
+                self.facts['interfaces'][key]['ipv6'] = list()
+            addr, subnet = value['address'].split("/")
+            ipv6 = dict(address=addr.strip(), subnet=subnet.strip())
+            self.add_ip_address(addr.strip(), 'ipv6')
+            self.facts['interfaces'][key]['ipv6'].append(ipv6)
+
+    def add_ip_address(self, address, family):
+        if family == 'ipv4':
+            self.facts['all_ipv4_addresses'].append(address)
+        else:
+            self.facts['all_ipv6_addresses'].append(address)
+
+    def preprocess(self, data):
+        preprocessed = list()
+        for line in data.split('\n'):
+            if len(line) == 0 or line[:5] == 'Flags':
+                continue
+            elif re.match(r'\s\d', line[:2]):
+                preprocessed.append(line[2:])
+            else:
+                preprocessed[-1] += line
+        return preprocessed
+
+    def parse_interfaces(self, data):
+        facts = dict()
+        data = self.preprocess(data)
+        for line in data:
+            name = self.parse_name(line)
+            facts[name] = dict()
+            for (key, value) in re.findall(self.DETAIL_RE, line):
+                facts[name][key] = value
+        return facts
+
+    def parse_addresses(self, data):
+        facts = dict()
+        data = self.preprocess(data)
+        for line in data:
+            name = self.parse_interface(line)
+            facts[name] = dict()
+            for (key, value) in re.findall(self.DETAIL_RE, line):
+                facts[name][key] = value
+        return facts
+
+    def parse_neighbors(self, data):
+        facts = dict()
+        data = self.preprocess(data)
+        for line in data:
+            name = self.parse_interface(line)
+            facts[name] = dict()
+            for (key, value) in re.findall(self.DETAIL_RE, line):
+                facts[name][key] = value
+        return facts
+
+    def parse_name(self, data):
+        match = re.search(r'name=\"([\w\d\-]+)\"', data, re.M)
+        if match:
+            return match.group(1)
+
+    def parse_interface(self, data):
+        match = re.search(r'interface=([\w\d\-]+)', data, re.M)
+        if match:
+            return match.group(1)
+
+
+FACT_SUBSETS = dict(
+    default=Default,
+    hardware=Hardware,
+    interfaces=Interfaces,
+    config=Config,
+)
+
+VALID_SUBSETS = frozenset(FACT_SUBSETS.keys())
+
+global warnings
+warnings = list()
+
+
+def main():
+    """main entry point for module execution
+    """
+    argument_spec = dict(
+        gather_subset=dict(default=['!config'], type='list')
+    )
+
+    argument_spec.update(routeros_argument_spec)
+
+    module = AnsibleModule(argument_spec=argument_spec,
+                           supports_check_mode=True)
+
+    gather_subset = module.params['gather_subset']
+
+    runable_subsets = set()
+    exclude_subsets = set()
+
+    for subset in gather_subset:
+        if subset == 'all':
+            runable_subsets.update(VALID_SUBSETS)
+            continue
+
+        if subset.startswith('!'):
+            subset = subset[1:]
+            if subset == 'all':
+                exclude_subsets.update(VALID_SUBSETS)
+                continue
+            exclude = True
+        else:
+            exclude = False
+
+        if subset not in VALID_SUBSETS:
+            module.fail_json(msg='Bad subset: %s' % subset)
+
+        if exclude:
+            exclude_subsets.add(subset)
+        else:
+            runable_subsets.add(subset)
+
+    if not runable_subsets:
+        runable_subsets.update(VALID_SUBSETS)
+
+    runable_subsets.difference_update(exclude_subsets)
+    runable_subsets.add('default')
+
+    facts = dict()
+    facts['gather_subset'] = list(runable_subsets)
+
+    instances = list()
+    for key in runable_subsets:
+        instances.append(FACT_SUBSETS[key](module))
+
+    for inst in instances:
+        inst.populate()
+        facts.update(inst.facts)
+
+    ansible_facts = dict()
+    for key, value in iteritems(facts):
+        key = 'ansible_net_%s' % key
+        ansible_facts[key] = value
+
+    module.exit_json(ansible_facts=ansible_facts, warnings=warnings)
+
+
+if __name__ == '__main__':
+    main()

test/units/modules/network/routeros/fixtures/routeros_facts/export

@@ -0,0 +1,26 @@
+# sep/25/2018 10:10:52 by RouterOS 6.42.5
+# software id = 9EER-511K
+#
+#
+#
+/interface wireless security-profiles
+set [ find default=yes ] supplicant-identity=MikroTik
+/tool user-manager customer
+set admin access=own-routers,own-users,own-profiles,own-limits,config-payment-gw
+/ip address
+add address=192.168.88.1/24 comment=defconf interface=ether1 network=192.168.88.0
+/ip dhcp-client
+add dhcp-options=hostname,clientid disabled=no interface=ether1
+/system lcd
+set contrast=0 enabled=no port=parallel type=24x4
+/system lcd page
+set time disabled=yes display-time=5s
+set resources disabled=yes display-time=5s
+set uptime disabled=yes display-time=5s
+set packets disabled=yes display-time=5s
+set bits disabled=yes display-time=5s
+set version disabled=yes display-time=5s
+set identity disabled=yes display-time=5s
+set ether1 disabled=yes display-time=5s
+/tool user-manager database
+set db-path=user-manager

test/units/modules/network/routeros/fixtures/routeros_facts/interface_print_detail_without-paging

@@ -0,0 +1,7 @@
+Flags: D - dynamic, X - disabled, R - running, S - slave
+ 0  R  name="ether1" default-name="ether1" type="ether" mtu=1500 actual-mtu=1500
+       mac-address=00:1C:42:36:52:90 last-link-up-time=sep/25/2018 06:30:04
+       link-downs=0
+ 1  R  name="ether2" default-name="ether2" type="ether" mtu=1500 actual-mtu=1500
+       mac-address=00:1C:42:36:52:90 last-link-up-time=sep/25/2018 06:30:04
+       link-downs=0

test/units/modules/network/routeros/fixtures/routeros_facts/ip_address_print_detail_without-paging

@@ -0,0 +1,7 @@
+Flags: X - disabled, I - invalid, D - dynamic
+ 0   ;;; defconf
+     address=192.168.88.1/24 network=192.168.88.0 interface=ether1
+     actual-interface=ether1
+
+ 1 D address=10.37.129.3/24 network=10.37.129.0 interface=ether1
+     actual-interface=ether1

test/units/modules/network/routeros/fixtures/routeros_facts/ip_neighbor_print_detail_without-paging

@@ -0,0 +1,15 @@
+ 0 interface=ether2-master address=10.37.129.3 address4=10.37.129.3 mac-address=D4:CA:6D:C6:16:4C identity="router1" platform="MikroTik" version="6.42.2 (stable)" unpack=none age=59s
+   uptime=3w19h11m36s software-id="1234-1234" board="RBwAPG-5HacT2HnD" interface-name="bridge" system-description="MikroTik RouterOS 6.42.2 (stable) RBwAPG-5HacT2HnD"
+   system-caps="" system-caps-enabled=""
+
+ 1 interface=ether3 address=10.37.129.4 address4=10.37.129.4 mac-address=D4:CA:6D:C6:18:2F identity="router2" platform="MikroTik" version="6.42.2 (stable)" unpack=none age=54s
+   uptime=3w19h11m30s software-id="1234-1234" board="RBwAPG-5HacT2HnD" ipv6=no interface-name="bridge" system-description="MikroTik RouterOS 6.42.2 (stable) RBwAPG-5HacT2HnD"
+   system-caps="" system-caps-enabled=""
+
+ 2 interface=ether5 address=10.37.129.5 address4=10.37.129.5 mac-address=B8:69:F4:37:F0:C8 identity="router3" platform="MikroTik" version="6.40.8 (bugfix)" unpack=none age=43s
+   uptime=3d14h25m31s software-id="1234-1234" board="RB960PGS" interface-name="ether1" system-description="MikroTik RouterOS 6.40.8 (bugfix) RB960PGS" system-caps=""
+   system-caps-enabled=""
+
+ 3 interface=ether10 address=10.37.129.6 address4=10.37.129.6 mac-address=6C:3B:6B:A1:0B:63 identity="router4" platform="MikroTik" version="6.42.2 (stable)" unpack=none age=54s
+   uptime=3w6d1h11m44s software-id="1234-1234" board="RBSXTLTE3-7" interface-name="bridge" system-description="MikroTik RouterOS 6.42.2 (stable) RBSXTLTE3-7" system-caps=""
+   system-caps-enabled=""

test/units/modules/network/routeros/fixtures/routeros_facts/ipv6_address_print_detail_without-paging

@@ -0,0 +1,3 @@
+Flags: X - disabled, I - invalid, D - dynamic, G - global, L - link-local
+ 0 DL address=fe80::21c:42ff:fe36:5290/64 from-pool="" interface=ether1
+      actual-interface=ether1 eui-64=no advertise=no no-dad=no

test/units/modules/network/routeros/fixtures/routeros_facts/system_identity_print_without-paging

@@ -0,0 +1 @@
+  name: MikroTik

test/units/modules/network/routeros/fixtures/routeros_facts/system_resource_print_without-paging

@@ -0,0 +1,16 @@
+                   uptime: 3h28m52s
+                  version: 6.42.5 (stable)
+               build-time: Jun/26/2018 12:12:08
+              free-memory: 988.3MiB
+             total-memory: 1010.8MiB
+                      cpu: Intel(R)
+                cpu-count: 2
+            cpu-frequency: 2496MHz
+                 cpu-load: 0%
+           free-hdd-space: 63.4GiB
+          total-hdd-space: 63.5GiB
+  write-sect-since-reboot: 4576
+         write-sect-total: 4576
+        architecture-name: x86
+               board-name: x86
+                 platform: MikroTik

test/units/modules/network/routeros/fixtures/routeros_facts/system_routerboard_print_without-paging

@@ -0,0 +1,7 @@
+       routerboard: yes
+             model: RouterBOARD 3011UiAS
+     serial-number: 1234567890
+     firmware-type: ipq8060
+  factory-firmware: 3.41
+  current-firmware: 3.41
+  upgrade-firmware: 6.42.2

test/units/modules/network/routeros/test_routeros_facts.py

@@ -0,0 +1,113 @@
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.compat.tests.mock import patch
+from ansible.modules.network.routeros import routeros_facts
+from units.modules.utils import set_module_args
+from .routeros_module import TestRouterosModule, load_fixture
+
+
+class TestRouterosFactsModule(TestRouterosModule):
+
+    module = routeros_facts
+
+    def setUp(self):
+        super(TestRouterosFactsModule, self).setUp()
+        self.mock_run_commands = patch('ansible.modules.network.routeros.routeros_facts.run_commands')
+        self.run_commands = self.mock_run_commands.start()
+
+    def tearDown(self):
+        super(TestRouterosFactsModule, self).tearDown()
+        self.mock_run_commands.stop()
+
+    def load_fixtures(self, commands=None):
+        def load_from_file(*args, **kwargs):
+            module = args
+            commands = kwargs['commands']
+            output = list()
+
+            for command in commands:
+                filename = str(command).split(' | ')[0].replace(' ', '_')
+                output.append(load_fixture('routeros_facts%s' % filename))
+            return output
+
+        self.run_commands.side_effect = load_from_file
+
+    def test_routeros_facts_default(self):
+        set_module_args(dict(gather_subset='default'))
+        result = self.execute_module()
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_hostname'], 'MikroTik'
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_version'], '6.42.5 (stable)'
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_model'], 'RouterBOARD 3011UiAS'
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_serialnum'], '1234567890'
+        )
+
+    def test_routeros_facts_hardware(self):
+        set_module_args(dict(gather_subset='hardware'))
+        result = self.execute_module()
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_spacefree_mb'], 64921.6
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_spacetotal_mb'], 65024.0
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_memfree_mb'], 988.3
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_memtotal_mb'], 1010.8
+        )
+
+    def test_routeros_facts_config(self):
+        set_module_args(dict(gather_subset='config'))
+        result = self.execute_module()
+        self.assertIsInstance(
+            result['ansible_facts']['ansible_net_config'], str
+        )
+
+    def test_routeros_facts_interfaces(self):
+        set_module_args(dict(gather_subset='interfaces'))
+        result = self.execute_module()
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_all_ipv4_addresses'][0], '10.37.129.3'
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_all_ipv4_addresses'][0],
+            result['ansible_facts']['ansible_net_interfaces']['ether1']['ipv4'][0]['address']
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_all_ipv6_addresses'], ['fe80::21c:42ff:fe36:5290']
+        )
+        self.assertEqual(
+            result['ansible_facts']['ansible_net_all_ipv6_addresses'][0],
+            result['ansible_facts']['ansible_net_interfaces']['ether1']['ipv6'][0]['address']
+        )
+        self.assertEqual(
+            len(result['ansible_facts']['ansible_net_interfaces'].keys()), 2
+        )
+        self.assertEqual(
+            len(result['ansible_facts']['ansible_net_neighbors'].keys()), 4
+        )