Caldera fix for long running experiments

app/calderacontrol.py

@@ -12,6 +12,7 @@ import simplejson
 from app.exceptions import CalderaError
 from app.interface_sfx import CommandlineColors
 from app.attack_log import AttackLog
+from pprint import pprint
 
 
 # TODO: Ability deserves an own class.
@@ -259,7 +260,7 @@ class CalderaControl():
 
         if paw not in orep["steps"]:
             print("Broken operation report:")
-            print(orep)
+            pprint(orep)
             print(f"Could not find {paw} in {orep['steps']}")
             raise CalderaError
         # print("oprep: " + str(orep))
@@ -267,7 +268,8 @@ class CalderaControl():
             if a_step["ability_id"] == ability_id:
                 try:
                     # TODO There is no output if the state is for example -4 (untrusted). Fix that. Why is the caldera implant untrusted ?
-                    print("oprep: " + str(orep))
+                    # print("Operation report: ")
+                    # pprint(orep)
                     return a_step["output"]
                 except KeyError as exception:
                     raise CalderaError from exception
@@ -456,7 +458,7 @@ class CalderaControl():
         if debug:
             print(f"Operation data {operation}")
         try:
-            print(operation[0]["state"])
+            # print(operation[0]["state"])
             if operation[0]["state"] == "finished":
                 return True
         except KeyError as exception:
@@ -534,17 +536,18 @@ class CalderaControl():
 
         print(f"New adversary generated. ID: {adid}, ability: {ability_id} group: {group}")
         res = self.add_operation(operation_name, advid=adid, group=group)
-        print(f"Add operation: {res}")
+        print(f"Add operation: ")
+        pprint(res)
 
         opid = self.get_operation(operation_name)["id"]
         print("New operation created. OpID: " + str(opid))
 
-        res = self.execute_operation(opid)
-        print(f"Execute operation: {res}")
-        retries = 50
+        self.execute_operation(opid)
+        print(f"Execute operation")
+        retries = 30
         print(f"{CommandlineColors.OKGREEN}Executed attack operation{CommandlineColors.ENDC}")
         while not self.is_operation_finished(opid) and retries > 0:
-            print(".... waiting for Caldera to finish")
+            print(f".... waiting for Caldera to finish {retries}")
             time.sleep(10)
             retries -= 1
             if retries <= 0:
@@ -552,22 +555,26 @@ class CalderaControl():
 
         # TODO: Handle outout from several clients
 
-        retries = 0
+        retries = 5
         output = None
-        while retries < 10:
+        while retries > 0:
             try:
+                retries -= 1
+                time.sleep(10)
                 output = self.view_operation_output(opid, paw, ability_id)
+                print(f".... getting Caldera output {retries}")
+                if output:
+                    break
             except CalderaError:
-                retries += 1
-                time.sleep(10)
-            else:
-                break
+                pass
 
         if output is None:
             output = str(self.get_operation_by_id(opid))
             print(f"{CommandlineColors.FAIL}Failed getting operation data. We just have: {output} from get_operation_by_id{CommandlineColors.ENDC}")
         else:
-            print("Output: " + str(output))
+            outp = str(output)
+            print(f"{CommandlineColors.BACKGROUND_BLUE} Output: {outp} {CommandlineColors.ENDC}")
+            pprint(output)
 
         #  ######## Cleanup
         self.execute_operation(opid, "cleanup")

app/experimentcontrol.py

@@ -14,6 +14,7 @@ from app.config import ExperimentConfig
 from app.interface_sfx import CommandlineColors
 from caldera_control import CalderaControl
 from machine_control import Machine
+from app.exceptions import ServerError
 
 
 # TODO: Multi threading at least when starting machines
@@ -112,7 +113,31 @@ class Experiment():
 
                     caldera_control.attack(attack_logger=self.attack_logger, paw=target_1.get_paw(), ability_id=attack, group=target_1.get_group())
 
+                    # Moved to fix section below. If fix works: can be removed
+                    # print(f"Pausing before next attack (config: nap_time): {self.experiment_control.get_nap_time()}")
+                    # time.sleep(self.experiment_control.get_nap_time())
+
+                    # Fix: Caldera sometimes gets stuck. This is why we better re-start the caldera server and wait till all the implants re-connected
+                    # Reason: In some scenarios we keep the infra up for hours or days. No re-creation like intended. This can cause Caldera to hick up
+                    print(f"{CommandlineColors.OKBLUE}Restarting caldera server and waiting for clients to re-connect{CommandlineColors.ENDC}")
+                    self.attacker_1.start_caldera_server()
+                    print(f"Pausing before next attack (config: nap_time): {self.experiment_control.get_nap_time()}")
                     time.sleep(self.experiment_control.get_nap_time())
+                    retries = 100
+                    for target_system in self.targets:
+                        running_agents = caldera_control.list_paws_of_running_agents()
+                        print(f"Agents currently connected to the server: {running_agents}")
+                        while target_system.get_paw() not in running_agents:
+                            time.sleep(1)
+                            running_agents = caldera_control.list_paws_of_running_agents()
+                            retries -= 1
+                            print(f"Waiting for clients to re-connect ({retries}, {running_agents}) ")
+                            if retries <= 0:
+                                raise ServerError
+                    print(f"{CommandlineColors.OKGREEN}Restarted caldera server clients re-connected{CommandlineColors.ENDC}")
+                    # End of fix
+
+
         print(f"{CommandlineColors.OKGREEN}Finished Caldera attacks{CommandlineColors.ENDC}")
 
         # Run Kali attacks
@@ -123,7 +148,7 @@ class Experiment():
                 # TODO: Work with snapshots
                 print(f"Attacking machine with PAW: {target_1.get_paw()} with attack: {attack}")
                 self.attacker_1.kali_attack(attack, target_1.getip(), self.experiment_control)
-
+                print(f"Pausing before next attack (config: nap_time): {self.experiment_control.get_nap_time()}")
                 time.sleep(self.experiment_control.get_nap_time())
 
         print(f"{CommandlineColors.OKGREEN}Finished Kali attacks{CommandlineColors.ENDC}")

app/interface_sfx.py

@@ -4,6 +4,7 @@
 
 # Colors to be used when printing text to the terminal
 # print(f"{CommandlineColors.WARNING}Warning {CommandlineColors.ENDC}")
+# https://dev.to/ifenna__/adding-colors-to-bash-scripts-48g4
 
 
 class CommandlineColors:
@@ -15,6 +16,7 @@ class CommandlineColors:
     OKBLUE = '\033[94m'
     OKCYAN = '\033[96m'
     OKGREEN = '\033[92m'
+    BACKGROUND_BLUE ='\033[104m'
     WARNING = '\033[93m'
     ATTACK = '\033[93m'       # An attack is running
     MACHINE_CREATED = '\033[92m'

app/machinecontrol.py

@@ -431,6 +431,8 @@ class Machine():
 
         print(f"{CommandlineColors.OKBLUE}Starting Caldera server {CommandlineColors.ENDC}")
 
+        # The pkill was added because the server sometimes gets stuck. And we can not re-create the attacking machines in all cases
+        self.vm_manager.__call_remote_run__(" pkill -f server.py;", disown=False)
         cmd = f"cd {self.caldera_basedir}; cd caldera ; nohup python3 server.py --insecure &"
         self.vm_manager.__call_remote_run__(cmd, disown=True)
         self.wait_for_caldera_server()