From 8025dbcd67c789a83651f538f9f0e3897ad02854 Mon Sep 17 00:00:00 2001 From: Thorsten Sick Date: Mon, 10 May 2021 10:35:52 +0200 Subject: [PATCH] Simplified sensor plugins, round 2 --- app/config.py | 10 ++ app/machinecontrol.py | 3 +- experiment_control.py | 3 + plugins/base/machinery.py | 1 + plugins/base/plugin_base.py | 8 +- plugins/base/ssh_features.py | 162 ++++++++++++++++++ plugins/default/kali/nmap/default_config.yaml | 0 plugins/default/kali/nmap/nmap_plugin.py | 3 + .../running_vm/running_vm_plugin.py | 151 +--------------- .../vm_controller/vagrant/vagrant_plugin.py | 150 +++------------- tools/shipit.py | 2 + 11 files changed, 220 insertions(+), 273 deletions(-) create mode 100644 plugins/base/ssh_features.py delete mode 100644 plugins/default/kali/nmap/default_config.yaml diff --git a/app/config.py b/app/config.py index 1399a70..a2a900c 100644 --- a/app/config.py +++ b/app/config.py @@ -248,3 +248,13 @@ class ExperimentConfig(): return self.raw_config["attacks"]["nap_time"] except KeyError: return 0 + + def get_sensor_config(self, name): + """ Return the config for a specific sensor + + @param name: name of the sensor + """ + if "sensors" not in self.raw_config: + return {} + if name not in self.raw_config["sensors"]: + return self.raw_config["sensors"][name] diff --git a/app/machinecontrol.py b/app/machinecontrol.py index 572d864..1dc23ed 100644 --- a/app/machinecontrol.py +++ b/app/machinecontrol.py @@ -201,7 +201,8 @@ class Machine(): } plugin.set_sysconf(syscon) plugin.set_machine_plugin(self.vm_manager) - plugin.process_config(self.config.raw_config.get(name, {})) # plugin specific configuration + # TODO: Process experiment config to get sensor configuration + plugin.process_config({}) # plugin specific configuration plugin.setup() reboot |= plugin.prime() self.sensors.append(plugin) diff --git a/experiment_control.py b/experiment_control.py index e2b220a..f19423f 100644 --- a/experiment_control.py +++ b/experiment_control.py @@ -6,6 +6,9 @@ import argparse from app.experimentcontrol import Experiment +# TODO: Add verbose settings: -v to -vvv +# TODO: Name experiments. Name will be written to the log + def explain(args): # pylint: disable=unused-argument """ Explain the tool""" diff --git a/plugins/base/machinery.py b/plugins/base/machinery.py index 663ae8a..3804bf2 100644 --- a/plugins/base/machinery.py +++ b/plugins/base/machinery.py @@ -64,6 +64,7 @@ class MachineryPlugin(BasePlugin): def remote_run(self, cmd, disown=False): """ Connects to the machine and runs a command there + @param cmd: command to run int he machine's shell @param disown: Send the connection into background """ diff --git a/plugins/base/plugin_base.py b/plugins/base/plugin_base.py index 97355cd..93fe328 100644 --- a/plugins/base/plugin_base.py +++ b/plugins/base/plugin_base.py @@ -135,10 +135,14 @@ class BasePlugin(): def load_default_config(self): """ Reads and returns the default config as dict """ - if not os.path.isfile(self.get_default_config_filename()): + filename = self.get_default_config_filename() + + if not os.path.isfile(filename): + print(f"Did not find default config {filename}") self.conf = {} else: - with open(self.get_default_config_filename()) as fh: + with open(filename) as fh: + print(f"Loading default config {filename}") self.conf = yaml.safe_load(fh) if self.conf is None: self.conf = {} diff --git a/plugins/base/ssh_features.py b/plugins/base/ssh_features.py new file mode 100644 index 0000000..b29a16e --- /dev/null +++ b/plugins/base/ssh_features.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" A class you can use to add SSH features to you plugin. Useful for vm_controller/machinery classes """ + +from fabric import Connection +from app.exceptions import NetworkError +from invoke.exceptions import UnexpectedExit +import paramiko +import time +import socket +from plugins.base.plugin_base import BasePlugin + + +class SSHFeatures(BasePlugin): + + def __init__(self): + super().__init__() + self.c = None + + def connect(self): + """ Connect to a machine """ + + if self.c: + return self.c + + retries = 10 + retry_sleep = 10 + timeout = 30 + while retries: + try: + if self.config.os() == "linux": + uhp = self.get_ip() + print(f"Connecting to {uhp}") + self.c = Connection(uhp, connect_timeout=timeout) + + if self.config.os() == "windows": + args = {} + # args = {"key_filename": self.config.ssh_keyfile() or self.v.keyfile(vm_name=self.config.vmname())} + if self.config.ssh_keyfile(): + args["key_filename"] = self.config.ssh_keyfile() + if self.config.ssh_password(): + args["password"] = self.config.ssh_password() + print(args) + uhp = self.get_ip() + print(uhp) + self.c = Connection(uhp, connect_timeout=timeout, user=self.config.ssh_user(), connect_kwargs=args) + except (paramiko.ssh_exception.SSHException, socket.timeout): + print(f"Failed to connect, will retry {retries} times. Timeout: {timeout}") + retries -= 1 + timeout += 10 + time.sleep(retry_sleep) + else: + print(f"Connection: {self.c}") + return self.c + + print("SSH network error") + raise NetworkError + + def remote_run(self, cmd, disown=False): + """ Connects to the machine and runs a command there + + @param disown: Send the connection into background + """ + + if cmd is None: + return "" + + self.connect() + cmd = cmd.strip() + + print("Running SSH remote run: " + cmd) + print("Disown: " + str(disown)) + result = None + retry = 2 + while retry > 0: + try: + result = self.c.run(cmd, disown=disown) + print(result) + # paramiko.ssh_exception.SSHException in the next line is needed for windows openssh + except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit, paramiko.ssh_exception.SSHException): + if retry <= 0: + raise NetworkError + else: + self.disconnect() + self.connect() + retry -= 1 + print("Got some SSH errors. Retrying") + else: + break + + if result and result.stderr: + print("Debug: Stderr: " + str(result.stderr.strip())) + + if result: + return result.stdout.strip() + + return "" + + def put(self, src, dst): + """ Send a file to a machine + + @param src: source dir + @param dst: destination + """ + self.connect() + + print(f"PUT {src} -> {dst}") + + res = "" + retries = 10 + retry_sleep = 10 + timeout = 30 + while retries: + try: + res = self.c.put(src, dst) + except (paramiko.ssh_exception.SSHException, socket.timeout, UnexpectedExit): + print(f"PUT Failed to connect, will retry {retries} times. Timeout: {timeout}") + retries -= 1 + timeout += 10 + time.sleep(retry_sleep) + self.disconnect() + self.connect() + except FileNotFoundError as e: + print(f"File not found: {e}") + break + else: + return res + print("SSH network error on PUT command") + raise NetworkError + + def get(self, src, dst): + """ Get a file to a machine + + @param src: source dir + @param dst: destination + """ + self.connect() + + retry = 2 + while retry > 0: + try: + res = self.c.get(src, dst) + except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit): + if retry <= 0: + raise NetworkError + else: + self.disconnect() + self.connect() + retry -= 1 + print("Got some SSH errors. Retrying") + except FileNotFoundError as e: + print(e) + break + else: + break + + return res + + def disconnect(self): + """ Disconnect from a machine """ + if self.c: + self.c.close() + self.c = None diff --git a/plugins/default/kali/nmap/default_config.yaml b/plugins/default/kali/nmap/default_config.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/plugins/default/kali/nmap/nmap_plugin.py b/plugins/default/kali/nmap/nmap_plugin.py index 1c3bfa4..4e00962 100644 --- a/plugins/default/kali/nmap/nmap_plugin.py +++ b/plugins/default/kali/nmap/nmap_plugin.py @@ -46,6 +46,9 @@ from plugins.base.kali import KaliPlugin # TODO firewall evasion : -sS and -f for fragmented. old tech. But good for basic NDS tests # TODO decoy scan: -D RND:5 to generate 5 decoys # TODO spoof mac: --spoof-mac with 0, Apple, Dell, Cisco or fake MAC the first parameters in this list will generate random mac +# TODO: Use timing settings: -T0-T5 (paranoid, sneaky, polite, default, aggressive, insane). --min-parallelism 100 (for crashes) and use --scan-delay 10s or similar +# By that: crash sensors (most aggressive) or be under the detection threshold + class NmapPlugin(KaliPlugin): diff --git a/plugins/default/vm_controller/running_vm/running_vm_plugin.py b/plugins/default/vm_controller/running_vm/running_vm_plugin.py index c729d54..8bfb77e 100644 --- a/plugins/default/vm_controller/running_vm/running_vm_plugin.py +++ b/plugins/default/vm_controller/running_vm/running_vm_plugin.py @@ -3,15 +3,10 @@ # A plugin to control already running vms from plugins.base.machinery import MachineryPlugin, MachineStates -from fabric import Connection -from app.exceptions import NetworkError -from invoke.exceptions import UnexpectedExit -import paramiko -import time -import socket +from plugins.base.ssh_features import SSHFeatures -class RunningVMPlugin(MachineryPlugin): +class RunningVMPlugin(SSHFeatures, MachineryPlugin): # Boilerplate name = "running_vm" @@ -21,8 +16,9 @@ class RunningVMPlugin(MachineryPlugin): def __init__(self): super().__init__() + # super(SSHFeatures).__init__() + # super(MachineryPlugin).__init__() self.plugin_path = __file__ - self.c = None self.vagrantfilepath = None self.vagrantfile = None @@ -45,145 +41,6 @@ class RunningVMPlugin(MachineryPlugin): """ Destroy a machine """ return - def connect(self): - """ Connect to a machine """ - - if self.c: - return self.c - - retries = 10 - retry_sleep = 10 - timeout = 30 - while retries: - try: - if self.config.os() == "linux": - uhp = self.get_ip() - print(f"Connecting to {uhp}") - self.c = Connection(uhp, connect_timeout=timeout) - - if self.config.os() == "windows": - args = {} - # args = {"key_filename": self.config.ssh_keyfile() or self.v.keyfile(vm_name=self.config.vmname())} - if self.config.ssh_keyfile(): - args["key_filename"] = self.config.ssh_keyfile() - if self.config.ssh_password(): - args["password"] = self.config.ssh_password() - uhp = self.get_ip() - self.c = Connection(uhp, connect_timeout=timeout, user=self.config.ssh_user(), connect_kwargs=args) - except (paramiko.ssh_exception.SSHException, socket.timeout): - print(f"Failed to connect, will retry {retries} times. Timeout: {timeout}") - retries -= 1 - timeout += 10 - time.sleep(retry_sleep) - else: - print(f"Connection: {self.c}") - return self.c - - print("SSH network error") - raise NetworkError - - def remote_run(self, cmd, disown=False): - """ Connects to the machine and runs a command there - - @param disown: Send the connection into background - """ - - if cmd is None: - return "" - - self.connect() - cmd = cmd.strip() - - print("Running VM plugin remote run: " + cmd) - print("Disown: " + str(disown)) - result = None - retry = 2 - while retry > 0: - try: - result = self.c.run(cmd, disown=disown) - print(result) - except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit, paramiko.ssh_exception.SSHException): - if retry <= 0: - raise NetworkError - else: - self.disconnect() - self.connect() - retry -= 1 - print("Got some SSH errors. Retrying") - else: - break - - if result and result.stderr: - print("Debug: Stderr: " + str(result.stderr.strip())) - - if result: - return result.stdout.strip() - - return "" - - def put(self, src, dst): - """ Send a file to a machine - - @param src: source dir - @param dst: destination - """ - self.connect() - - print(f"PUT {src} -> {dst}") - - res = "" - retries = 10 - retry_sleep = 10 - timeout = 30 - while retries: - try: - res = self.c.put(src, dst) - except (paramiko.ssh_exception.SSHException, socket.timeout, UnexpectedExit): - print(f"PUT Failed to connect, will retry {retries} times. Timeout: {timeout}") - retries -= 1 - timeout += 10 - time.sleep(retry_sleep) - self.disconnect() - self.connect() - else: - return res - print("SSH network error on PUT command") - raise NetworkError - - def get(self, src, dst): - """ Get a file to a machine - - @param src: source dir - @param dst: destination - """ - self.connect() - - retry = 2 - while retry > 0: - try: - res = self.c.get(src, dst) - except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit): - if retry <= 0: - raise NetworkError - else: - self.disconnect() - self.connect() - retry -= 1 - print("Got some SSH errors. Retrying") - except FileNotFoundError as e: - print(e) - break - else: - break - - return res - - def disconnect(self): - """ Disconnect from a machine """ - if self.c: - self.c.close() - self.c = None - def get_state(self): """ Get detailed state of a machine """ diff --git a/plugins/default/vm_controller/vagrant/vagrant_plugin.py b/plugins/default/vm_controller/vagrant/vagrant_plugin.py index 9722201..4822ddb 100644 --- a/plugins/default/vm_controller/vagrant/vagrant_plugin.py +++ b/plugins/default/vm_controller/vagrant/vagrant_plugin.py @@ -8,14 +8,15 @@ import vagrant from fabric import Connection import os from app.exceptions import ConfigurationError -from app.exceptions import NetworkError -from invoke.exceptions import UnexpectedExit -import paramiko +# from app.exceptions import NetworkError +# from invoke.exceptions import UnexpectedExit +# import paramiko +from plugins.base.ssh_features import SSHFeatures # Experiment with paramiko instead of fabric. Seems fabric has some issues with the "put" command to Windows. There seems no fix (just my workarounds). Maybe paramiko is better. -class VagrantPlugin(MachineryPlugin): +class VagrantPlugin(SSHFeatures, MachineryPlugin): # Boilerplate name = "vagrant" @@ -30,6 +31,7 @@ class VagrantPlugin(MachineryPlugin): self.c = None self.vagrantfilepath = None self.vagrantfile = None + self.sysconf = {} def process_config(self, config): """ Machine specific processing of configuration """ @@ -76,131 +78,33 @@ class VagrantPlugin(MachineryPlugin): def connect(self): """ Connect to a machine. If there is already a connection we keep it """ - if self.c: - return self.c - + # For linux we are using Vagrant style if self.config.os() == "linux": + if self.c: + return self.c + uhp = self.v.user_hostname_port(vm_name=self.config.vmname()) print(f"Connecting to {uhp}") self.c = Connection(uhp, connect_kwargs={"key_filename": self.v.keyfile(vm_name=self.config.vmname())}) - print(self.c) return self.c - if self.config.os() == "windows": - args = {"key_filename": os.path.join(self.sysconf["abs_machinepath_external"], self.config.ssh_keyfile())} - if self.config.ssh_password(): - args["password"] = self.config.ssh_password() - uhp = self.get_ip() - print(uhp) - print(args) - print(self.config.ssh_user()) - self.c = Connection(uhp, user=self.config.ssh_user(), connect_kwargs=args) - print(self.c) - return self.c - - def remote_run(self, cmd, disown=False): - """ Connects to the machine and runs a command there - - @param disown: Send the connection into background - """ - - if cmd is None: - return "" - - self.connect() - cmd = cmd.strip() - - print("Vagrant plugin remote run: " + cmd) - print("Disown: " + str(disown)) - result = None - retry = 2 - while retry > 0: - try: - result = self.c.run(cmd, disown=disown) - print(result) - except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit, paramiko.ssh_exception.SSHException): - if retry <= 0: - raise(NetworkError) - else: - self.disconnect() - self.connect() - retry -= 1 - print("Got some SSH errors. Retrying") - else: - break - - if result and result.stderr: - print("Debug: Stderr: " + str(result.stderr.strip())) - - if result: - return result.stdout.strip() - - return "" - - def put(self, src, dst): - """ Send a file to a machine - - @param src: source dir - @param dst: destination - """ - self.connect() - - print(f"{src} -> {dst}") - - res = "" - retry = 2 - while retry > 0: - try: - res = self.c.put(src, dst) - except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit): - if retry <= 0: - raise (NetworkError) - else: - self.disconnect() - self.connect() - retry -= 1 - print("Got some SSH errors. Retrying") - except FileNotFoundError as e: - print(e) - break - else: - break - - return res - - def get(self, src, dst): - """ Get a file to a machine - - @param src: source dir - @param dst: destination - """ - self.connect() - - retry = 2 - while retry > 0: - try: - res = self.c.get(src, dst) - except (paramiko.ssh_exception.NoValidConnectionsError, UnexpectedExit): - if retry <= 0: - raise (NetworkError) - else: - self.disconnect() - self.connect() - retry -= 1 - print("Got some SSH errors. Retrying") - except FileNotFoundError as e: - print(e) - break - else: - break - - return res - - def disconnect(self): - """ Disconnect from a machine """ - if self.c: - self.c.close() - self.c = None + else: + return super().connect() + + # if self.config.os() == "linux": + # super + + # if self.config.os() == "windows": + # args = {"key_filename": os.path.join(self.sysconf["abs_machinepath_external"], self.config.ssh_keyfile())} + # if self.config.ssh_password(): + # args["password"] = self.config.ssh_password() + # uhp = self.get_ip() + # print(uhp) + # print(args) + # print(self.config.ssh_user()) + # self.c = Connection(uhp, user=self.config.ssh_user(), connect_kwargs=args) + # print(self.c) + # return self.c def get_state(self): """ Get detailed state of a machine """ diff --git a/tools/shipit.py b/tools/shipit.py index c38d7ad..afe371d 100755 --- a/tools/shipit.py +++ b/tools/shipit.py @@ -41,6 +41,7 @@ globs = ["TODO.md", "plugins/default/*/*/*.txt", "plugins/default/*/*/*.md", "plugins/default/*/*/*.exe", + "plugins/default/*/*/*.yaml", "plugins/default/*/*/*.dll", "plugins/default/*/*/*.dll_*", "plugins/default/*/*/*.reg", @@ -48,6 +49,7 @@ globs = ["TODO.md", "plugins/avast_internal_plugins/*/*/*.bat", "plugins/avast_internal_plugins/*/*/*.txt", "plugins/avast_internal_plugins/*/*/*.md", + "plugins/avast_internal_plugins/*/*/*.yaml", "plugins/avast_internal_plugins/*/*/*.exe", "plugins/avast_internal_plugins/*/*/*.dll", "plugins/avast_internal_plugins/*/*/*.dll_*",