Simplified sensor plugins, round 2

app/config.py

@@ -248,3 +248,13 @@ class ExperimentConfig():
             return self.raw_config["attacks"]["nap_time"]
         except KeyError:
             return 0
+
+    def get_sensor_config(self, name):
+        """ Return the config for a specific sensor
+
+        @param name: name of the sensor
+        """
+        if "sensors" not in self.raw_config:
+            return {}
+        if name not in self.raw_config["sensors"]:
+            return self.raw_config["sensors"][name]

app/machinecontrol.py

@@ -201,7 +201,8 @@ class Machine():
                       }
             plugin.set_sysconf(syscon)
             plugin.set_machine_plugin(self.vm_manager)
-            plugin.process_config(self.config.raw_config.get(name, {}))    # plugin specific configuration
+            # TODO: Process experiment config to get sensor configuration
+            plugin.process_config({})    # plugin specific configuration
             plugin.setup()
             reboot |= plugin.prime()
             self.sensors.append(plugin)

experiment_control.py

@@ -6,6 +6,9 @@ import argparse
 from app.experimentcontrol import Experiment
 
 
+# TODO: Add verbose settings: -v to -vvv
+# TODO: Name experiments. Name will be written to the log
+
 def explain(args):  # pylint: disable=unused-argument
     """ Explain the tool"""
 

plugins/base/machinery.py

@@ -64,6 +64,7 @@ class MachineryPlugin(BasePlugin):
     def remote_run(self, cmd, disown=False):
         """ Connects to the machine and runs a command there
 
+
         @param cmd: command to run int he machine's shell
         @param disown: Send the connection into background
         """

plugins/base/plugin_base.py

@@ -135,10 +135,14 @@ class BasePlugin():
     def load_default_config(self):
         """ Reads and returns the default config as dict """
 
-        if not os.path.isfile(self.get_default_config_filename()):
+        filename = self.get_default_config_filename()
+
+        if not os.path.isfile(filename):
+            print(f"Did not find default config {filename}")
             self.conf = {}
         else:
-            with open(self.get_default_config_filename()) as fh:
+            with open(filename) as fh:
+                print(f"Loading default config {filename}")
                 self.conf = yaml.safe_load(fh)
             if self.conf is None:
                 self.conf = {}

plugins/base/ssh_features.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+""" A class you can use to add SSH features to you plugin. Useful for vm_controller/machinery classes """
+
+from fabric import Connection
+from app.exceptions import NetworkError
+from invoke.exceptions import UnexpectedExit
+import paramiko
+import time
+import socket
+from plugins.base.plugin_base import BasePlugin
+
+
+class SSHFeatures(BasePlugin):
+
+    def __init__(self):
+        super().__init__()
+        self.c = None
+
+    def connect(self):
+        """ Connect to a machine """
+
+        if self.c:
+            return self.c
+
+        retries = 10
+        retry_sleep = 10
+        timeout = 30
+        while retries:
+            try:
+                if self.config.os() == "linux":
+                    uhp = self.get_ip()
+                    print(f"Connecting to {uhp}")
+                    self.c = Connection(uhp, connect_timeout=timeout)
+
+                if self.config.os() == "windows":
+                    args = {}
+                    # args = {"key_filename": self.config.ssh_keyfile() or self.v.keyfile(vm_name=self.config.vmname())}
+                    if self.config.ssh_keyfile():
+                        args["key_filename"] = self.config.ssh_keyfile()
+                    if self.config.ssh_password():
+                        args["password"] = self.config.ssh_password()
+                    print(args)
+                    uhp = self.get_ip()
+                    print(uhp)
+                    self.c = Connection(uhp, connect_timeout=timeout, user=self.config.ssh_user(), connect_kwargs=args)
+            except (paramiko.ssh_exception.SSHException, socket.timeout):
+                print(f"Failed to connect, will retry {retries} times. Timeout: {timeout}")
+                retries -= 1
+                timeout += 10
+                time.sleep(retry_sleep)
+            else:
+                print(f"Connection: {self.c}")
+                return self.c
+
+        print("SSH network error")
+        raise NetworkError
+
+    def remote_run(self, cmd, disown=False):
+        """ Connects to the machine and runs a command there
+
+        @param disown: Send the connection into background
+        """
+
+        if cmd is None:
+            return ""
+
+        self.connect()
+        cmd = cmd.strip()
+
+        print("Running SSH remote run: " + cmd)
+        print("Disown: " + str(disown))
+        result = None
+        retry = 2
+        while retry > 0:
+            try:
+                result = self.c.run(cmd, disown=disown)
+                print(result)
+                # paramiko.ssh_exception.SSHException in the next line is needed for windows openssh
+            except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit, paramiko.ssh_exception.SSHException):
+                if retry <= 0:
+                    raise NetworkError
+                else:
+                    self.disconnect()
+                    self.connect()
+                    retry -= 1
+                    print("Got some SSH errors. Retrying")
+            else:
+                break
+
+        if result and result.stderr:
+            print("Debug: Stderr: " + str(result.stderr.strip()))
+
+        if result:
+            return result.stdout.strip()
+
+        return ""
+
+    def put(self, src, dst):
+        """ Send a file to a machine
+
+        @param src: source dir
+        @param dst: destination
+        """
+        self.connect()
+
+        print(f"PUT {src} -> {dst}")
+
+        res = ""
+        retries = 10
+        retry_sleep = 10
+        timeout = 30
+        while retries:
+            try:
+                res = self.c.put(src, dst)
+            except (paramiko.ssh_exception.SSHException, socket.timeout, UnexpectedExit):
+                print(f"PUT Failed to connect, will retry {retries} times. Timeout: {timeout}")
+                retries -= 1
+                timeout += 10
+                time.sleep(retry_sleep)
+                self.disconnect()
+                self.connect()
+            except FileNotFoundError as e:
+                print(f"File not found: {e}")
+                break
+            else:
+                return res
+        print("SSH network error on PUT command")
+        raise NetworkError
+
+    def get(self, src, dst):
+        """ Get a file to a machine
+
+        @param src: source dir
+        @param dst: destination
+        """
+        self.connect()
+
+        retry = 2
+        while retry > 0:
+            try:
+                res = self.c.get(src, dst)
+            except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit):
+                if retry <= 0:
+                    raise NetworkError
+                else:
+                    self.disconnect()
+                    self.connect()
+                    retry -= 1
+                    print("Got some SSH errors. Retrying")
+            except FileNotFoundError as e:
+                print(e)
+                break
+            else:
+                break
+
+        return res
+
+    def disconnect(self):
+        """ Disconnect from a machine """
+        if self.c:
+            self.c.close()
+            self.c = None

plugins/default/kali/nmap/nmap_plugin.py

@@ -46,6 +46,9 @@ from plugins.base.kali import KaliPlugin
 # TODO firewall evasion : -sS and -f for fragmented. old tech. But good for basic NDS tests
 # TODO decoy scan: -D RND:5 to generate 5 decoys
 # TODO spoof mac: --spoof-mac with 0, Apple, Dell, Cisco or fake MAC the first parameters in this list will generate random mac
+# TODO: Use timing settings: -T0-T5 (paranoid, sneaky, polite, default, aggressive, insane). --min-parallelism 100 (for crashes) and use --scan-delay 10s or similar
+# By that: crash sensors (most aggressive) or be under the detection threshold
+
 
 class NmapPlugin(KaliPlugin):
 

plugins/default/vm_controller/running_vm/running_vm_plugin.py

@@ -3,15 +3,10 @@
 # A plugin to control already running vms
 
 from plugins.base.machinery import MachineryPlugin, MachineStates
-from fabric import Connection
+from plugins.base.ssh_features import SSHFeatures
-from app.exceptions import NetworkError
-from invoke.exceptions import UnexpectedExit
-import paramiko
-import time
-import socket
 
 
-class RunningVMPlugin(MachineryPlugin):
+class RunningVMPlugin(SSHFeatures, MachineryPlugin):
 
     # Boilerplate
     name = "running_vm"
@@ -21,8 +16,9 @@ class RunningVMPlugin(MachineryPlugin):
 
     def __init__(self):
         super().__init__()
+        # super(SSHFeatures).__init__()
+        # super(MachineryPlugin).__init__()
         self.plugin_path = __file__
-        self.c = None
         self.vagrantfilepath = None
         self.vagrantfile = None
 
@@ -45,145 +41,6 @@ class RunningVMPlugin(MachineryPlugin):
         """ Destroy a machine """
         return
 
-    def connect(self):
-        """ Connect to a machine """
-
-        if self.c:
-            return self.c
-
-        retries = 10
-        retry_sleep = 10
-        timeout = 30
-        while retries:
-            try:
-                if self.config.os() == "linux":
-                    uhp = self.get_ip()
-                    print(f"Connecting to {uhp}")
-                    self.c = Connection(uhp, connect_timeout=timeout)
-
-                if self.config.os() == "windows":
-                    args = {}
-                    # args = {"key_filename": self.config.ssh_keyfile() or self.v.keyfile(vm_name=self.config.vmname())}
-                    if self.config.ssh_keyfile():
-                        args["key_filename"] = self.config.ssh_keyfile()
-                    if self.config.ssh_password():
-                        args["password"] = self.config.ssh_password()
-                    uhp = self.get_ip()
-                    self.c = Connection(uhp, connect_timeout=timeout, user=self.config.ssh_user(), connect_kwargs=args)
-            except (paramiko.ssh_exception.SSHException, socket.timeout):
-                print(f"Failed to connect, will retry {retries} times. Timeout: {timeout}")
-                retries -= 1
-                timeout += 10
-                time.sleep(retry_sleep)
-            else:
-                print(f"Connection: {self.c}")
-                return self.c
-
-        print("SSH network error")
-        raise NetworkError
-
-    def remote_run(self, cmd, disown=False):
-        """ Connects to the machine and runs a command there
-
-        @param disown: Send the connection into background
-        """
-
-        if cmd is None:
-            return ""
-
-        self.connect()
-        cmd = cmd.strip()
-
-        print("Running VM plugin remote run: " + cmd)
-        print("Disown: " + str(disown))
-        result = None
-        retry = 2
-        while retry > 0:
-            try:
-                result = self.c.run(cmd, disown=disown)
-                print(result)
-            except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit, paramiko.ssh_exception.SSHException):
-                if retry <= 0:
-                    raise NetworkError
-                else:
-                    self.disconnect()
-                    self.connect()
-                    retry -= 1
-                    print("Got some SSH errors. Retrying")
-            else:
-                break
-
-        if result and result.stderr:
-            print("Debug: Stderr: " + str(result.stderr.strip()))
-
-        if result:
-            return result.stdout.strip()
-
-        return ""
-
-    def put(self, src, dst):
-        """ Send a file to a machine
-
-        @param src: source dir
-        @param dst: destination
-        """
-        self.connect()
-
-        print(f"PUT {src} -> {dst}")
-
-        res = ""
-        retries = 10
-        retry_sleep = 10
-        timeout = 30
-        while retries:
-            try:
-                res = self.c.put(src, dst)
-            except (paramiko.ssh_exception.SSHException, socket.timeout, UnexpectedExit):
-                print(f"PUT Failed to connect, will retry {retries} times. Timeout: {timeout}")
-                retries -= 1
-                timeout += 10
-                time.sleep(retry_sleep)
-                self.disconnect()
-                self.connect()
-            else:
-                return res
-        print("SSH network error on PUT command")
-        raise NetworkError
-
-    def get(self, src, dst):
-        """ Get a file to a machine
-
-        @param src: source dir
-        @param dst: destination
-        """
-        self.connect()
-
-        retry = 2
-        while retry > 0:
-            try:
-                res = self.c.get(src, dst)
-            except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit):
-                if retry <= 0:
-                    raise NetworkError
-                else:
-                    self.disconnect()
-                    self.connect()
-                    retry -= 1
-                    print("Got some SSH errors. Retrying")
-            except FileNotFoundError as e:
-                print(e)
-                break
-            else:
-                break
-
-        return res
-
-    def disconnect(self):
-        """ Disconnect from a machine """
-        if self.c:
-            self.c.close()
-            self.c = None
-
     def get_state(self):
         """ Get detailed state of a machine """
 

plugins/default/vm_controller/vagrant/vagrant_plugin.py

@@ -8,14 +8,15 @@ import vagrant
 from fabric import Connection
 import os
 from app.exceptions import ConfigurationError
-from app.exceptions import NetworkError
+# from app.exceptions import NetworkError
-from invoke.exceptions import UnexpectedExit
+# from invoke.exceptions import UnexpectedExit
-import paramiko
+# import paramiko
+from plugins.base.ssh_features import SSHFeatures
 
 # Experiment with paramiko instead of fabric. Seems fabric has some issues with the "put" command to Windows. There seems no fix (just my workarounds). Maybe paramiko is better.
 
 
-class VagrantPlugin(MachineryPlugin):
+class VagrantPlugin(SSHFeatures, MachineryPlugin):
 
     # Boilerplate
     name = "vagrant"
@@ -30,6 +31,7 @@ class VagrantPlugin(MachineryPlugin):
         self.c = None
         self.vagrantfilepath = None
         self.vagrantfile = None
+        self.sysconf = {}
 
     def process_config(self, config):
         """ Machine specific processing of configuration """
@@ -76,131 +78,33 @@ class VagrantPlugin(MachineryPlugin):
     def connect(self):
         """ Connect to a machine. If there is already a connection we keep it """
 
-        if self.c:
+        # For linux we are using Vagrant style
-            return self.c
-
         if self.config.os() == "linux":
+            if self.c:
+                return self.c
+
             uhp = self.v.user_hostname_port(vm_name=self.config.vmname())
             print(f"Connecting to {uhp}")
             self.c = Connection(uhp, connect_kwargs={"key_filename": self.v.keyfile(vm_name=self.config.vmname())})
-            print(self.c)
             return self.c
 
-        if self.config.os() == "windows":
+        else:
-            args = {"key_filename": os.path.join(self.sysconf["abs_machinepath_external"], self.config.ssh_keyfile())}
+            return super().connect()
-            if self.config.ssh_password():
+
-                args["password"] = self.config.ssh_password()
+        # if self.config.os() == "linux":
-            uhp = self.get_ip()
+        #     super
-            print(uhp)
+
-            print(args)
+        # if self.config.os() == "windows":
-            print(self.config.ssh_user())
+        #     args = {"key_filename": os.path.join(self.sysconf["abs_machinepath_external"], self.config.ssh_keyfile())}
-            self.c = Connection(uhp, user=self.config.ssh_user(), connect_kwargs=args)
+        #     if self.config.ssh_password():
-            print(self.c)
+        #         args["password"] = self.config.ssh_password()
-            return self.c
+        #     uhp = self.get_ip()
-
+        #     print(uhp)
-    def remote_run(self, cmd, disown=False):
+        #     print(args)
-        """ Connects to the machine and runs a command there
+        #     print(self.config.ssh_user())
-
+        #     self.c = Connection(uhp, user=self.config.ssh_user(), connect_kwargs=args)
-        @param disown: Send the connection into background
+        #     print(self.c)
-        """
+        #     return self.c
-
-        if cmd is None:
-            return ""
-
-        self.connect()
-        cmd = cmd.strip()
-
-        print("Vagrant plugin remote run: " + cmd)
-        print("Disown: " + str(disown))
-        result = None
-        retry = 2
-        while retry > 0:
-            try:
-                result = self.c.run(cmd, disown=disown)
-                print(result)
-            except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit, paramiko.ssh_exception.SSHException):
-                if retry <= 0:
-                    raise(NetworkError)
-                else:
-                    self.disconnect()
-                    self.connect()
-                    retry -= 1
-                    print("Got some SSH errors. Retrying")
-            else:
-                break
-
-        if result and result.stderr:
-            print("Debug: Stderr: " + str(result.stderr.strip()))
-
-        if result:
-            return result.stdout.strip()
-
-        return ""
-
-    def put(self, src, dst):
-        """ Send a file to a machine
-
-        @param src: source dir
-        @param dst: destination
-        """
-        self.connect()
-
-        print(f"{src} -> {dst}")
-
-        res = ""
-        retry = 2
-        while retry > 0:
-            try:
-                res = self.c.put(src, dst)
-            except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit):
-                if retry <= 0:
-                    raise (NetworkError)
-                else:
-                    self.disconnect()
-                    self.connect()
-                    retry -= 1
-                    print("Got some SSH errors. Retrying")
-            except FileNotFoundError as e:
-                print(e)
-                break
-            else:
-                break
-
-        return res
-
-    def get(self, src, dst):
-        """ Get a file to a machine
-
-        @param src: source dir
-        @param dst: destination
-        """
-        self.connect()
-
-        retry = 2
-        while retry > 0:
-            try:
-                res = self.c.get(src, dst)
-            except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit):
-                if retry <= 0:
-                    raise (NetworkError)
-                else:
-                    self.disconnect()
-                    self.connect()
-                    retry -= 1
-                    print("Got some SSH errors. Retrying")
-            except FileNotFoundError as e:
-                print(e)
-                break
-            else:
-                break
-
-        return res
-
-    def disconnect(self):
-        """ Disconnect from a machine """
-        if self.c:
-            self.c.close()
-        self.c = None
 
     def get_state(self):
         """ Get detailed state of a machine """

tools/shipit.py

@@ -41,6 +41,7 @@ globs = ["TODO.md",
          "plugins/default/*/*/*.txt",
          "plugins/default/*/*/*.md",
          "plugins/default/*/*/*.exe",
+         "plugins/default/*/*/*.yaml",
          "plugins/default/*/*/*.dll",
          "plugins/default/*/*/*.dll_*",
          "plugins/default/*/*/*.reg",
@@ -48,6 +49,7 @@ globs = ["TODO.md",
          "plugins/avast_internal_plugins/*/*/*.bat",
          "plugins/avast_internal_plugins/*/*/*.txt",
          "plugins/avast_internal_plugins/*/*/*.md",
+         "plugins/avast_internal_plugins/*/*/*.yaml",
          "plugins/avast_internal_plugins/*/*/*.exe",
          "plugins/avast_internal_plugins/*/*/*.dll",
          "plugins/avast_internal_plugins/*/*/*.dll_*",