Requirements update and fixes

Makefile

@@ -31,3 +31,11 @@ mypy:
 # Fixing mypy file by file
 stepbystep:
 	mypy --strict-optional --disallow-untyped-defs --check-untyped-defs plugins/base/ app/
+
+# Checking dependencies
+check_dependencies:
+	pipdeptree
+
+# Updating dependencies
+update_dependencies:
+	pip-upgrade requirements.txt

app/attack_log.py

@@ -640,7 +640,7 @@ class AttackLog():
 
         :param filename: Name of the json file
         """
-        with open(filename, "wt") as fh:
+        with open(filename, "wt", encoding="utf8") as fh:
             json.dump(self.get_dict(), fh)
 
     def post_process(self) -> None:

app/calderaapi_4.py

@@ -700,7 +700,7 @@ class CalderaAPI():
 
         print(f"Number of abilities: {len(self.list_abilities())}")
 
-        with open("debug_removeme.txt", "wt") as fh:
+        with open("debug_removeme.txt", "wt", encoding="utf8") as fh:
             fh.write(pformat(self.list_abilities()))
 
         for ability in self.list_abilities():
@@ -714,12 +714,12 @@ class CalderaAPI():
         @param abi: A ability dict
         """
 
-        print("""
-        TTP: {technique_id}
-        Technique name: {technique_name}
-        Tactic: {tactic}
-        Name: {name}
-        ID: {ability_id}
-        Description: {description}
+        print(f"""
+        TTP: {abi["technique_id"]}
+        Technique name: {abi["technique_name"]}
+        Tactic: {abi["tactic"]}
+        Name: {abi["name"]}
+        ID: {abi["ability_id"]}
+        Description: {abi["description"]}
 
-        """.format(**abi))
+        """)

app/calderacontrol.py

@@ -130,7 +130,7 @@ class CalderaControl(CalderaAPI):
 
         print(f"Number of abilities: {len(self.list_abilities())}")
 
-        with open("debug_removeme.txt", "wt") as fh:
+        with open("debug_removeme.txt", "wt", encoding="utf8") as fh:
             fh.write(pformat(self.list_abilities()))
 
         for ability in self.list_abilities():
@@ -458,13 +458,13 @@ class CalderaControl(CalderaAPI):
         :param abi: A ability dict
         """
 
-        print("""
-        TTP: {technique_id}
-        Technique name: {technique_name}
-        Tactic: {tactic}
-        Name: {name}
-        ID: {ability_id}
-        Description: {description}
-        Platform: {platform}/{executor}
+        print(f"""
+        TTP: {abi["technique_id"]}
+        Technique name: {abi["technique_name"]}
+        Tactic: {abi["tactic"]}
+        Name: {abi["name"]}
+        ID: {abi["ability_id"]}
+        Description: {abi["description"]}
+        Platform: {abi["platform"]}/{abi["executor"]}
 
-        """.format(**abi))
+        """)

app/config.py

@@ -169,7 +169,7 @@ class ExperimentConfig():
         :param configfile: The configuration file to process
         """
 
-        with open(configfile) as fh:
+        with open(configfile, encoding="utf8") as fh:
             data = yaml.safe_load(fh)
 
         if data is None:

app/doc_generator.py

@@ -32,13 +32,13 @@ class DocGenerator():
         )
         template = env.get_template("attack_description.rst")
 
-        with open(jfile) as fh:
+        with open(jfile, encoding="utf8") as fh:
             attack = json.load(fh)
 
         rendered = template.render(events=attack["attack_log"], systems=attack["system_overview"], boilerplate=attack["boilerplate"])
         print(rendered)
 
-        with open(outfile, "wt") as fh:
+        with open(outfile, "wt", encoding="utf8") as fh:
             fh.write(rendered)
 
     def compile_documentation(self) -> None:

app/machinecontrol.py

@@ -682,7 +682,7 @@ START {playground}{filename} -server {url} -group {self.config.caldera_group()}
             filename = os.path.join(self.abs_machinepath_external, "caldera_agent.sh")
         elif self.get_os() == "windows":
             filename = os.path.join(self.abs_machinepath_external, "caldera_agent.bat")
-        with open(filename, "wt") as fh:
+        with open(filename, "wt", encoding="utf8") as fh:
             fh.write(content)
         if self.attack_logger is not None:
             self.attack_logger.vprint(f"{CommandlineColors.OKGREEN}Installed Caldera service {CommandlineColors.ENDC}", 1)

detect.py

@@ -27,7 +27,7 @@ class Detector():
         as_text = "["
 
         # Filebeat jsons are not valid jsons and have to be fixed
-        with open(args.sensor_log, "rt") as fh:
+        with open(args.sensor_log, "rt", encoding="utf-8") as fh:
             new = fh.read()
             new = new.replace("}{", "},{")
             as_text += new

experiment_control.py

@@ -23,7 +23,7 @@ def run(args):
     """
 
     if args.caldera_attack_file:
-        with open(args.caldera_attack_file, "rt") as fh:
+        with open(args.caldera_attack_file, "rt", encoding="utf8") as fh:
             for line in fh:
                 line = line.strip()
                 print(f"Running calder attack {line}")

machine_control.py

@@ -18,7 +18,7 @@ def create_machines(arguments):
     @param arguments: The arguments from argparse
     """
 
-    with open(arguments.configfile) as fh:
+    with open(arguments.configfile, encoding="utf8") as fh:
         config = yaml.safe_load(fh)
 
     attack_logger = AttackLog(arguments.verbose)
@@ -47,7 +47,7 @@ def create_machines(arguments):
     target_.start_caldera_client()
     print("Target done")
 
-    print("Caldera server running at: http://{}:8888/".format(attacker_1.get_ip()))
+    print(f"Caldera server running at: http://{attacker_1.get_ip()}:8888/")
     # target_.install_caldera_client(attacker_1.getip(), "target1elf")
 
 

plugins/base/plugin_base.py

@@ -258,7 +258,7 @@ class BasePlugin():
         """
 
         if os.path.isfile(self.get_default_config_filename()):
-            with open(self.get_default_config_filename(), "rt") as fh:
+            with open(self.get_default_config_filename(), "rt", encoding="utf8") as fh:
                 return fh.read()
         else:
             return f"# The plugin {self.get_name()} does not support configuration"
@@ -276,7 +276,7 @@ class BasePlugin():
             self.vprint(f"Did not find default config {filename}", 3)
             self.conf = {}
         else:
-            with open(filename) as fh:
+            with open(filename, encoding="utf8") as fh:
                 self.vprint(f"Loading default config {filename}", 3)
                 self.conf = yaml.safe_load(fh)
             if self.conf is None:

pydantic_test.py

@@ -13,7 +13,7 @@ from app.config_verifier import MainConfig
 
 def load(filename):
     """ Loads the config file and feeds it into the built in verifier """
-    with open(filename) as fh:
+    with open(filename, encoding="utf-8") as fh:
         data = yaml.safe_load(fh)
         return MainConfig(**data)
 

requirements.txt

@@ -1,32 +1,36 @@
 python-vagrant==0.5.15
 fabric==2.6.0
-requests==2.25.1
-simplejson==3.17.2
-tox==3.22.0
-coverage==5.4
-PyYAML==5.4.1
+requests==2.27.1
+simplejson==3.17.6
+tox==3.24.5
+coverage==6.3.2
+PyYAML==6.0
 straight.plugin==1.5.0
-paramiko==2.7.2
+paramiko==2.9.2
 pymetasploit3==1.0.3
-pylint==2.9.3
-flask==2.0.2
-pydantic==1.8.2
-dotmap==1.3.25
+pylint==2.12.2
+flask==2.0.3
+pydantic==1.9.0
+dotmap==1.3.26
 
 # Sphinx stuff
-sphinx-argparse==0.2.5
+sphinx-argparse==0.3.1
 sphinxcontrib-autoyaml==0.6.1
-sphinx-pyreverse==0.0.13
-sphinxcontrib.asciinema==0.3.2
+sphinx-pyreverse==0.0.17
+sphinxcontrib.asciinema==0.3.3
 sphinx-revealjs
 # sphinx-pydantic    # This one has issues that must be fixed upstream first
 
 # Mypy stuff
 mypy==0.931
-types-PyYAML==5.4.6
-types-requests==2.25.6
-types-simplejson==3.17.0
-types-paramiko==2.7.0
+types-PyYAML==6.0.4
+types-requests==2.27.11
+types-simplejson==3.17.3
+types-paramiko==2.8.13
+
+# pipdeptree
+pipdeptree
+pip-upgrade
 
 # Argcomplete. See README.md
 argcomplete==2.0.0

setup.py

@@ -7,7 +7,7 @@ import setuptools
 # https://packaging.python.org/tutorials/packaging-projects/
 # https://setuptools.readthedocs.io/en/latest/
 
-with open("README.md", "r") as fh:
+with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setuptools.setup(