Merge pull request #25 from avast/fixing_filebeat

Fixing Hydra vs. filebeat. Issues fixed: Kali linux updated. Hydra de…

app/experimentcontrol.py

@@ -73,6 +73,7 @@ class Experiment():
             if self.machine_needs_caldera(target_1, caldera_attacks):
                 target_1.install_caldera_service()
             target_1.up()
+            target_1.reboot()  # Kernel changes on system creation require a reboot
             needs_reboot = target_1.prime_vulnerabilities()
             needs_reboot |= target_1.prime_sensors()
             if needs_reboot:
@@ -331,7 +332,7 @@ class Experiment():
             except subprocess.CalledProcessError:
                 # Machine does not exist
                 pass
-            self.attacker_1.create(reboot=False)
+            self.attacker_1.create(reboot=True)
             self.attacker_1.up()
             self.attacker_1.install_caldera_server(cleanup=False)
         else:

plugins/default/sensors/linux_filebeat/linux_filebeat_plugin.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 
 # A plugin to experiment with Linux logstash filebeat sensors
+# https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html
 
 from plugins.base.sensor import SensorPlugin
 import os
 from jinja2 import Environment, FileSystemLoader, select_autoescape
+import time
 
 
 class LinuxFilebeatPlugin(SensorPlugin):
@@ -42,12 +44,17 @@ class LinuxFilebeatPlugin(SensorPlugin):
 
         self.vprint("Installing Linux filebeat sensor", 3)
 
-        self.run_cmd("sudo wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -")
-        self.run_cmd('sudo echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-7.x.list')
-        self.run_cmd("sudo apt update")
-        self.run_cmd("sudo apt -y install default-jre")
-        self.run_cmd("sudo apt -y install logstash")
-        self.run_cmd("sudo apt -y install filebeat")
+        # Filebeat
+        fb_file = "filebeat-7.15.2-amd64.deb"
+        self.run_cmd(f"curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/{fb_file}")
+        self.run_cmd(f"sudo dpkg -i {fb_file}")
+
+        # Logstash
+
+        self.run_cmd("wget -qO- https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -")
+        self.run_cmd("sudo apt-get install apt-transport-https")
+        self.run_cmd("echo 'deb https://artifacts.elastic.co/packages/7.x/apt stable main' | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list")
+        self.run_cmd("sudo apt update && sudo apt install logstash")
 
         # Copy config
         self.run_cmd(f"sudo cp {pg}/filebeat.yml /etc/filebeat/filebeat.yml")
@@ -67,12 +74,12 @@ class LinuxFilebeatPlugin(SensorPlugin):
 
     def start(self):
 
-        self.run_cmd("sudo filebeat modules enable system,iptables")
+        self.run_cmd("sudo filebeat modules enable system iptables")
         self.run_cmd("sudo filebeat setup --pipelines --modules iptables,system,")
-        self.run_cmd("sudo systemctl enable filebeat")
-        self.run_cmd("sudo systemctl start filebeat")
-        self.run_cmd("sudo systemctl enable logstash.service")
-        self.run_cmd("sudo systemctl start logstash.service")
+        # self.run_cmd("sudo systemctl start logstash.service")
+        self.run_cmd("sudo nohup /usr/share/logstash/bin/logstash  -f /etc/logstash/conf.d/filebeat.conf &", disown=True)
+        time.sleep(20)
+        self.run_cmd("sudo service filebeat start")
 
         return None
 

systems/Vagrantfile

@@ -224,7 +224,7 @@ Vagrant.configure("2") do |config|
 
 	  # https://app.vagrantup.com/kalilinux/boxes/rolling
 	  attacker.vm.box = "kalilinux/rolling"
-	  # config.vm.box_version = "2020.3.0"
+	  attacker.vm.box_version = "2021.3.0"
 	  #config.vm.base_mac = "080027BB1476"
 	  attacker.vm.hostname = "attacker"
 

systems/attacker1/bootstrap.sh

@@ -11,16 +11,14 @@ echo "Bootstrapping attacker1"
 echo '* libraries/restart-without-asking boolean true' | sudo debconf-set-selections
 
 # Update system
-apt update
+apt -y update
+export DEBIAN_FRONTEND=noninteractive
+yes '' | apt -y -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" dist-upgrade
+
 cd ~
 wget https://bootstrap.pypa.io/get-pip.py
 python3 get-pip.py
 apt -y install golang sphinx-common
-#apt -y upgrade
-
-#apt -y install apt-transport-https
-#apt -y install openssh-server
-#apt -y install whois # for mkpasswd
 
 ip addr show eth1 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1 > /vagrant/attacker1/ip4.txt
 

systems/target3/bootstrap.sh

@@ -27,49 +27,9 @@ apt -y install gdb
 # user with password "passw0rd"
 # useradd -m -p '$6$q5PAnDI5K0uv$hMGMJQleeS9F2yLOiHXs2PxZHEmV.ook8jyWILzDGDxSTJmTTZSe.QgLVrnuwiyAl5PFJVARkMsSnPICSndJR1' -s /bin/bash password
 
-# Install Elastic search debian repo
 
-wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
-echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-7.x.list
-apt update
-
-
-################# This must move into a plugin !!! ###############
-# Install Logstash
-# apt -y install default-jre
-# apt -y install logstash
-
-# Install filebeat
-# apt -y install filebeat
-# Configure logstash as output
-# cp /vagrant/target1/config/filebeat.yml /etc/filebeat/filebeat.yml
-# cp /vagrant/target1/config/caldera_agent.service /etc/systemd/system/
-
-# Config logstash
-# cp /vagrant/target1/logstash_conf/*.conf /etc/logstash/conf.d
-# rm /vagrant/target1/logstash/filebeat.json
-# touch /vagrant/target1/logstash/filebeat.json
-# chmod o+w /vagrant/target1/logstash/filebeat.json
-
-# Start Logstash and filebeat
-# filebeat modules enable system,iptables
-# filebeat setup --pipelines --modules iptables,system,
-# systemctl start logstash.service
-# systemctl enable filebeat
-# systemctl enable logstash.service
-
-# Run logstash manually for debugging:
-# https://www.elastic.co/guide/en/logstash/current/running-logstash-command-line.html
-# /usr/share/logstash/bin/logstash --node-name debug -f /etc/logstash/conf.d/ --log.level debug --config.debug
-
-# To test conf files:
-# /usr/share/logstash/bin/logstash  -f /etc/logstash/conf.d/ -t
-
-# Start Caldera agent service
-# ln -s /vagrant/target1/config/caldera_agent.service /etc/systemd/system
-# chmod 666 /etc/systemd/system
-# systemctl enable caldera_agent.service
-# systemctl start caldera_agent.service
+apt -y update
+apt -y upgrade
 
 
 ip addr show enp0s8 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1 > /vagrant/target3/ip4.txt