diff --git a/app/calderacontrol.py b/app/calderacontrol.py index a5f89ec..30ba60a 100644 --- a/app/calderacontrol.py +++ b/app/calderacontrol.py @@ -12,6 +12,7 @@ import simplejson from app.exceptions import CalderaError from app.interface_sfx import CommandlineColors from app.attack_log import AttackLog +from pprint import pprint # TODO: Ability deserves an own class. @@ -259,7 +260,7 @@ class CalderaControl(): if paw not in orep["steps"]: print("Broken operation report:") - print(orep) + pprint(orep) print(f"Could not find {paw} in {orep['steps']}") raise CalderaError # print("oprep: " + str(orep)) @@ -267,7 +268,8 @@ class CalderaControl(): if a_step["ability_id"] == ability_id: try: # TODO There is no output if the state is for example -4 (untrusted). Fix that. Why is the caldera implant untrusted ? - print("oprep: " + str(orep)) + # print("Operation report: ") + # pprint(orep) return a_step["output"] except KeyError as exception: raise CalderaError from exception @@ -456,7 +458,7 @@ class CalderaControl(): if debug: print(f"Operation data {operation}") try: - print(operation[0]["state"]) + # print(operation[0]["state"]) if operation[0]["state"] == "finished": return True except KeyError as exception: @@ -534,17 +536,18 @@ class CalderaControl(): print(f"New adversary generated. ID: {adid}, ability: {ability_id} group: {group}") res = self.add_operation(operation_name, advid=adid, group=group) - print(f"Add operation: {res}") + print(f"Add operation: ") + pprint(res) opid = self.get_operation(operation_name)["id"] print("New operation created. OpID: " + str(opid)) - res = self.execute_operation(opid) - print(f"Execute operation: {res}") - retries = 50 + self.execute_operation(opid) + print(f"Execute operation") + retries = 30 print(f"{CommandlineColors.OKGREEN}Executed attack operation{CommandlineColors.ENDC}") while not self.is_operation_finished(opid) and retries > 0: - print(".... waiting for Caldera to finish") + print(f".... waiting for Caldera to finish {retries}") time.sleep(10) retries -= 1 if retries <= 0: @@ -552,22 +555,26 @@ class CalderaControl(): # TODO: Handle outout from several clients - retries = 0 + retries = 5 output = None - while retries < 10: + while retries > 0: try: + retries -= 1 + time.sleep(10) output = self.view_operation_output(opid, paw, ability_id) + print(f".... getting Caldera output {retries}") + if output: + break except CalderaError: - retries += 1 - time.sleep(10) - else: - break + pass if output is None: output = str(self.get_operation_by_id(opid)) print(f"{CommandlineColors.FAIL}Failed getting operation data. We just have: {output} from get_operation_by_id{CommandlineColors.ENDC}") else: - print("Output: " + str(output)) + outp = str(output) + print(f"{CommandlineColors.BACKGROUND_BLUE} Output: {outp} {CommandlineColors.ENDC}") + pprint(output) # ######## Cleanup self.execute_operation(opid, "cleanup") diff --git a/app/experimentcontrol.py b/app/experimentcontrol.py index 8d05bbd..ab13542 100644 --- a/app/experimentcontrol.py +++ b/app/experimentcontrol.py @@ -14,6 +14,7 @@ from app.config import ExperimentConfig from app.interface_sfx import CommandlineColors from caldera_control import CalderaControl from machine_control import Machine +from app.exceptions import ServerError # TODO: Multi threading at least when starting machines @@ -112,7 +113,31 @@ class Experiment(): caldera_control.attack(attack_logger=self.attack_logger, paw=target_1.get_paw(), ability_id=attack, group=target_1.get_group()) + # Moved to fix section below. If fix works: can be removed + # print(f"Pausing before next attack (config: nap_time): {self.experiment_control.get_nap_time()}") + # time.sleep(self.experiment_control.get_nap_time()) + + # Fix: Caldera sometimes gets stuck. This is why we better re-start the caldera server and wait till all the implants re-connected + # Reason: In some scenarios we keep the infra up for hours or days. No re-creation like intended. This can cause Caldera to hick up + print(f"{CommandlineColors.OKBLUE}Restarting caldera server and waiting for clients to re-connect{CommandlineColors.ENDC}") + self.attacker_1.start_caldera_server() + print(f"Pausing before next attack (config: nap_time): {self.experiment_control.get_nap_time()}") time.sleep(self.experiment_control.get_nap_time()) + retries = 100 + for target_system in self.targets: + running_agents = caldera_control.list_paws_of_running_agents() + print(f"Agents currently connected to the server: {running_agents}") + while target_system.get_paw() not in running_agents: + time.sleep(1) + running_agents = caldera_control.list_paws_of_running_agents() + retries -= 1 + print(f"Waiting for clients to re-connect ({retries}, {running_agents}) ") + if retries <= 0: + raise ServerError + print(f"{CommandlineColors.OKGREEN}Restarted caldera server clients re-connected{CommandlineColors.ENDC}") + # End of fix + + print(f"{CommandlineColors.OKGREEN}Finished Caldera attacks{CommandlineColors.ENDC}") # Run Kali attacks @@ -123,7 +148,7 @@ class Experiment(): # TODO: Work with snapshots print(f"Attacking machine with PAW: {target_1.get_paw()} with attack: {attack}") self.attacker_1.kali_attack(attack, target_1.getip(), self.experiment_control) - + print(f"Pausing before next attack (config: nap_time): {self.experiment_control.get_nap_time()}") time.sleep(self.experiment_control.get_nap_time()) print(f"{CommandlineColors.OKGREEN}Finished Kali attacks{CommandlineColors.ENDC}") diff --git a/app/interface_sfx.py b/app/interface_sfx.py index 849aea5..37cb66e 100644 --- a/app/interface_sfx.py +++ b/app/interface_sfx.py @@ -4,6 +4,7 @@ # Colors to be used when printing text to the terminal # print(f"{CommandlineColors.WARNING}Warning {CommandlineColors.ENDC}") +# https://dev.to/ifenna__/adding-colors-to-bash-scripts-48g4 class CommandlineColors: @@ -15,6 +16,7 @@ class CommandlineColors: OKBLUE = '\033[94m' OKCYAN = '\033[96m' OKGREEN = '\033[92m' + BACKGROUND_BLUE ='\033[104m' WARNING = '\033[93m' ATTACK = '\033[93m' # An attack is running MACHINE_CREATED = '\033[92m' diff --git a/app/machinecontrol.py b/app/machinecontrol.py index 5e84bac..c50b08c 100644 --- a/app/machinecontrol.py +++ b/app/machinecontrol.py @@ -431,6 +431,8 @@ class Machine(): print(f"{CommandlineColors.OKBLUE}Starting Caldera server {CommandlineColors.ENDC}") + # The pkill was added because the server sometimes gets stuck. And we can not re-create the attacking machines in all cases + self.vm_manager.__call_remote_run__(" pkill -f server.py;", disown=False) cmd = f"cd {self.caldera_basedir}; cd caldera ; nohup python3 server.py --insecure &" self.vm_manager.__call_remote_run__(cmd, disown=True) self.wait_for_caldera_server()