Basic doc upgrade

caldera_control.py

@@ -6,6 +6,7 @@ import argparse
 from pprint import pprint
 import argcomplete
 
+# from app.calderacontrol import CalderaControl
 # from app.calderacontrol import CalderaControl
 from app.calderaapi_4 import CalderaAPI
 
@@ -40,36 +41,22 @@ def agents(calcontrol, arguments):  # pylint: disable=unused-argument
         print(calcontrol.kill_agent(arguments.paw))
 
 
-def list_facts(calcontrol, arguments):  # pylint: disable=unused-argument
-    """ Call list fact stores ("sources") in caldera control
+def facts(calcontrol, arguments):
+    """ Deal with fact stores ("sources") in caldera control
 
     @param calcontrol: Connection to the caldera server
     @param arguments: Parser command line arguments
     """
 
-    printme = "No found"
-
-    if arguments.name:
-        printme = calcontrol.list_facts_for_name(arguments.name)
-    else:
-        printme = calcontrol.list_sources()
-
-    print(f"Stored facts: {printme}")
-
-
-def add_facts(calcontrol, arguments):  # pylint: disable=unused-argument
-    """ Generate new facts in caldera
-
-    @param calcontrol: Connection to the caldera server
-    @param arguments: Parser command line arguments
-    """
-    name = "Test"
-    data = {"foo": "bar"}
+    if arguments.list:
+        if arguments.name is None:
+            raise CmdlineArgumentException("Listing facts by name requires a name")
 
-    print(f'Created fact: {calcontrol.add_sources(name, data)}')
+        print_me = calcontrol.list_facts_for_name(arguments.name)
+        print(f"Stored facts: {print_me}")
 
 
-def list_abilities(calcontrol, arguments):
+def abilities(calcontrol, arguments):
     """ Call list abilities in caldera control
 
     @param calcontrol: Connection to the caldera server
@@ -77,11 +64,11 @@ def list_abilities(calcontrol, arguments):
     """
 
     if arguments.list:
-        abilities = calcontrol.list_abilities()
-        abi_ids = [aid.ability_id for aid in abilities]
+        ability_list = calcontrol.list_abilities()
+        abi_ids = [aid.ability_id for aid in ability_list]
         print(abi_ids)
 
-        for abi in abilities:
+        for abi in ability_list:
             for executor in abi.executors:
                 for a_parser in executor.parsers:
                     pprint(a_parser.relationships)
@@ -202,7 +189,7 @@ def operations(calcontrol, arguments):
 
 
 def attack(calcontrol, arguments):
-    """ Calling attack
+    """ Starting an attack
 
     @param calcontrol: Connection to the caldera server
     @param arguments: Parser command line arguments
@@ -217,56 +204,57 @@ def attack(calcontrol, arguments):
 def create_parser():
     """ Creates the parser for the command line arguments"""
 
-    main_parser = argparse.ArgumentParser("Controls a Caldera server to attack other systems")
+    main_parser = argparse.ArgumentParser("Controls a Caldera server. Use this to test your Caldera setup or the Caldera API.")
     main_parser.add_argument('--verbose', '-v', action='count', default=0)
 
     subparsers = main_parser.add_subparsers(help="sub-commands")
 
     # Sub parser for attacks
-    parser_attack = subparsers.add_parser("attack", help="attack system")
+    parser_attack = subparsers.add_parser("attack", help="Attack system")
     parser_attack.set_defaults(func=attack)
-    parser_attack.add_argument("--paw", default="kickme", help="paw to attack and get specific results for")
-    parser_attack.add_argument("--group", default="red", help="target group to attack")
+    parser_attack.add_argument("--paw", default="kickme", help="Paw to attack and get specific results for")
+    parser_attack.add_argument("--group", default="red", help="Target group to attack")
     parser_attack.add_argument("--ability_id", default="bd527b63-9f9e-46e0-9816-b8434d2b8989",
                                help="The ability to use for the attack")
 
     # Sub parser to list abilities
-    parser_abilities = subparsers.add_parser("abilities", help="abilities")
+    parser_abilities = subparsers.add_parser("abilities", help="Control Caldera abilities ( aka exploits)")
     # parser_abilities.add_argument("--abilityid", default=None, help="Id of the ability to list")
-    parser_abilities.set_defaults(func=list_abilities)
-    parser_abilities.add_argument("--ability_ids", default=[], nargs="+",
-                                  help="The abilities to look up. One or more ids")
+    parser_abilities.set_defaults(func=abilities)
+    # parser_abilities.add_argument("--ability_ids", default=[], nargs="+",
+    #                               help="The abilities to look up. One or more ids")
     parser_abilities.add_argument("--list", default=False, action="store_true",
                                   help="List all abilities")
 
-    parser_agents = subparsers.add_parser("agents", help="agents")
+    parser_agents = subparsers.add_parser("agents", help="Control Caldera agents ( aka implants)")
     parser_agents.set_defaults(func=agents)
     parser_agents.add_argument("--list", default=False, action="store_true", help="List all agents")
-    parser_agents.add_argument("--delete", default=False, action="store_true", help="Delete agent")
-    parser_agents.add_argument("--kill", default=False, action="store_true", help="Delete agent")
-    parser_agents.add_argument("--paw", default=None, help="PAW to delete. if not set it will delete all agents")
+    parser_agents.add_argument("--delete", default=False, action="store_true", help="Delete agent from database")
+    parser_agents.add_argument("--kill", default=False, action="store_true", help="Kill agent on target system")
+    parser_agents.add_argument("--paw", default=None, help="PAW to delete or kill. If this is not set it will delete all agents")
 
     parser_facts = subparsers.add_parser("facts", help="facts")
-    parser_facts.set_defaults(func=list_facts)
+    parser_facts.set_defaults(func=facts)
+    parser_facts.add_argument("--list", default=False, action="store_true", help="List facts")
     parser_facts.add_argument("--name", default=None, help="Name of a fact source to focus on")
 
-    parser_facts = subparsers.add_parser("add_facts", help="facts")
-    parser_facts.set_defaults(func=add_facts)
+    # parser_facts = subparsers.add_parser("add_facts", help="facts")
+    # parser_facts.set_defaults(func=add_facts)
 
     # Sub parser for obfuscators
-    parser_obfuscators = subparsers.add_parser("obfuscators", help="obfuscators")
+    parser_obfuscators = subparsers.add_parser("obfuscators", help="Obfuscator interface. Hide the attack")
     parser_obfuscators.set_defaults(func=obfuscators)
     parser_obfuscators.add_argument("--list", default=False, action="store_true",
                                     help="List all obfuscators")
 
     # Sub parser for objectives
-    parser_objectives = subparsers.add_parser("objectives", help="objectives")
+    parser_objectives = subparsers.add_parser("objectives", help="Objectives interface")
     parser_objectives.set_defaults(func=objectives)
     parser_objectives.add_argument("--list", default=False, action="store_true",
                                    help="List all objectives")
 
     # Sub parser for adversaries
-    parser_adversaries = subparsers.add_parser("adversaries", help="adversaries")
+    parser_adversaries = subparsers.add_parser("adversaries", help="Adversary interface. Adversaries are attacker archetypes")
     parser_adversaries.set_defaults(func=adversaries)
     parser_adversaries.add_argument("--list", default=False, action="store_true",
                                     help="List all adversaries")
@@ -279,7 +267,7 @@ def create_parser():
     parser_adversaries.add_argument("--adversary_id", "--advid", default=None, help="Adversary ID")
 
     # Sub parser for operations
-    parser_operations = subparsers.add_parser("operations", help="operations")
+    parser_operations = subparsers.add_parser("operations", help="Attack operation interface")
     parser_operations.set_defaults(func=operations)
     parser_operations.add_argument("--list", default=False, action="store_true",
                                    help="List all operations")
@@ -291,7 +279,7 @@ def create_parser():
                                    help="View the report of a finished operation")
     parser_operations.add_argument("--name", default=None, help="Name of the operation")
     parser_operations.add_argument("--adversary_id", "--advid", default=None, help="Adversary ID")
-    parser_operations.add_argument("--source_id", "--sourceid", default="basic", help="'Source' ID")
+    parser_operations.add_argument("--source_id", "--sourceid", default="basic", help="Source ID")
     parser_operations.add_argument("--planner_id", "--planid", default="atomic", help="Planner ID")
     parser_operations.add_argument("--group", default="", help="Caldera group to run the operation on (we are targeting groups, not PAWs)")
     parser_operations.add_argument("--state", default="running", help="State to start the operation in")
@@ -300,20 +288,20 @@ def create_parser():
     parser_operations.add_argument("--id", default=None, help="ID of operation to delete")
 
     # Sub parser for sources
-    parser_sources = subparsers.add_parser("sources", help="sources")
+    parser_sources = subparsers.add_parser("sources", help="Data source management")
     parser_sources.set_defaults(func=sources)
     parser_sources.add_argument("--list", default=False, action="store_true",
                                 help="List all sources")
 
     # Sub parser for planners
-    parser_sources = subparsers.add_parser("planners", help="planners")
+    parser_sources = subparsers.add_parser("planners", help="Planner management. They define the pattern of attack steps")
     parser_sources.set_defaults(func=planners)
     parser_sources.add_argument("--list", default=False, action="store_true",
                                 help="List all planners")
 
     # For all parsers
-    main_parser.add_argument("--caldera_url", help="caldera url, including port", default="http://localhost:8888/")
-    main_parser.add_argument("--apikey", help="caldera api key", default="ADMIN123")
+    main_parser.add_argument("--caldera_url", help="The Caldera url, including port and protocol (http://)", default="http://localhost:8888/")
+    main_parser.add_argument("--apikey", help="Caldera api key", default="ADMIN123")
 
     return main_parser
 

doc/doc_todo.txt

@@ -0,0 +1,12 @@
+
+
+
+
+TODO: What sensors are pre-installed ?
+TODO: How to attack it ?
+TODO: How to contact the servers (ssh/...) ? Scriptable
+TODO: How to run it without sudo ?
+TODO: Which data is collected ? How to access it ? How to get data dumps out ?
+TODO: Add Linux Server
+TODO: Add Mac Server
+

doc/source/basics/background.rst

@@ -6,15 +6,15 @@ Purple Dome is a simulated and automated environment to experiment with hacking
 
 PurpleDome is relevant for you:
 
-* If you develop sensors for bolt on security
-* If you want to test detection logic for your bolt on security
-* If you want to stress test mitigation around your vulnerable apps
-* Experiment with hardening your OS or software
-* Want to forensically analyse a system after an attack
-* Do some blue team exercises
-* Want to train ML on data from real attacks
+* If you develop **sensors** for bolt on security
+* If you want to test **detection logic** for your bolt on security
+* If you want to stress test **mitigation** around your vulnerable apps
+* Experiment with **hardening** your OS or software
+* Want to **forensically** analyse a system after an attack
+* Do some **blue team exercises**
+* Want to **train ML** on data from real attacks
 
-PurpleDome simulates a small busniess network. It generates an attacker VM and target VMs. Automated attacks are then run against the targets.
+PurpleDome simulates a small business network. It generates an attacker VM and target VMs. Automated attacks are then run against the targets.
 
 Depending on which sensors you picked you will get their logs. And the logs from the attacks. Perfect to compare them side-by-side.
 
@@ -52,54 +52,18 @@ The experiments are configured in YAML files, the format is described in the *co
 
 If you want to modify Purple Dome and contribute to it I can point you to the *Extending* chapter. Thanks to a plugin interface this is quite simple.
 
-
-
-
-
-TODO: What sensors are pre-installed ?
-TODO: How to attack it ?
-TODO: How to contact the servers (ssh/...) ? Scriptable
-TODO: How to run it without sudo ?
-TODO: Which data is collected ? How to access it ? How to get data dumps out ?
-TODO: Add Linux Server
-TODO: Add Mac Server
-
-
-
 Data aggregator
 ---------------
 
 We currently can use logstash
 
-There are several options for data aggregators:
-
-* Fleet OSQuery aggregator: https://github.com/kolide/fleet
-* The Hive
-
-
-Sensors on Targets (most are Windows)
--------------------------------------
-
-Those sensors are not integrated but could be nice to play with:
-
-Palantir Windows Event forwarding: https://github.com/palantir/windows-event-forwarding
-
-Autorun monitoring: https://github.com/palantir/windows-event-forwarding/tree/master/AutorunsToWinEventLog
-
-Palantir OSquery: https://github.com/palantir/osquery-configuration
-
-SwiftOnSecurity Sysmon config: https://github.com/SwiftOnSecurity/sysmon-config
-
-
-Palantir OSQuery is mixed OS: Windows/Mac Endpoints, Linux Servers
-
 
 Caldera
 -------
 
-Attack framework.
+Caldera is an attack framework. Especially useful for pen testing and blue team training.
 
-Starting: *python3 server.py --insecure*
+Starting it: *python3 server.py --insecure*
 
 Web UI on *http://localhost:8888/*
 
@@ -114,40 +78,25 @@ server="http://192.168.178.45:8888";curl -s -X POST -H "file:sandcat.go" -H "pla
 Filebeat
 --------
 
-Filebeat has a set of modules:
+Filebeat collects logs on the target system.
+
+It has a set of modules:
 
 https://www.elastic.co/guide/en/beats/filebeat/6.8/filebeat-modules-overview.html
 
-List modules: *filebeat modules list*
+You can view a list of modules using: *filebeat modules list*
 
-%% TODO: Add OSQueryD https://osquery.readthedocs.io/en/latest/introduction/using-osqueryd/
 
 Logstash
 --------
 
+Logstash is used to aggregate the data from filebeat into a json file.
+
 Logstash uses all .conf files in /etc/logstash/conf.d
 
 https://www.elastic.co/guide/en/logstash/current/config-setting-files.html
 
-Alternative: The Hive
----------------------
-
-Sander Spierenburg (SOC Teamlead) seems to be interested in The Hive. So it is back in the game
-
-
-
 Repos
 -----
 
-* The main part: https://git.int.avast.com/ai-research/purpledome
-* Caldera fork to fix bugs: TBD
-* Caldera Plugin for statistics: <add public git/avast folder>
-
-
-Links
------
-
-* Others detecting this kind of things
-
-    - https://redcanary.com/blog/how-one-hospital-thwarted-a-ryuk-ransomware-outbreak/
-
+PurpleDome can be found on github: https://git.int.avast.com/ai-research/purpledome

doc/source/basics/configuration.rst

@@ -4,6 +4,8 @@ Configuration
 
 Configuration is contained in yaml files. The example shipped with the code is *template.yaml*.
 
+For your first experiments use *hello_world.yaml* which will run a simple attack on a simulated system.
+
 To define the VMs there are also *Vagrantfiles* and associated scripts. The example shipped with the code is in the *systems* folder. Using Vagrant is optional.
 
 Machines
@@ -25,6 +27,8 @@ You can install vulnerabilities and weaknesses in the targets to allow your atta
 Sensors
 =======
 
+Sensors are all kinds of technology monitoring system events and collecting data required to detect an attack. Either while it happens or as a forensic experiment.
+
 Each machine can have a list of sensors to run on it. In addition there is the global *sensor_conf* setting to configure the sensors.
 
 Sensors are implemented as plugins.
@@ -37,7 +41,7 @@ caldera_attacks
 
 Caldera attacks (called abilities) are identified by a unique ID. Some abilities are built to target several OS-es.
 
-All Caldera abilities are available. As some will need parameters and Caldera does not offer the option to configure those in the YAML, some caldera attacks might not work without implementing a plugin.
+All Caldera abilities are available. As some will need parameters and PurpleDome does not offer the option to configure those in the YAML, some caldera attacks might not work without implementing a plugin.
 
 In the YAML file you will find two sub-categories under caldera_attacks: linux and windows. There you just list the ids of the caldera attacks to run on those systems.
 

doc/source/index.rst

@@ -8,8 +8,8 @@
 .. Autodoc part
 .. https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#module-sphinx.ext.autodoc
 
-Welcome to the Purple Dome documentation!
-=========================================
+Welcome to the Purple Dome documentation
+========================================
 
 .. toctree::
    :maxdepth: 3

doc/source/usage/cli.rst

@@ -9,6 +9,10 @@ The central one is Experiment control where you start your experiments:
 .. asciinema:: ./../asciinema/experiment_control.cast
     :speed: 2
 
+
+
+
+
 Experiment control
 ==================
 
@@ -19,6 +23,10 @@ Experiment control is the core tool to run an experiment. It accepts a yaml conf
     :func: create_parser
     :prog: ./experiment_control.py
 
+
+
+
+
 Testing YAML files
 ==================
 
@@ -29,6 +37,10 @@ Configuration can be a bit complex and mistakes can happen. To find them before
     :func: create_parser
     :prog: ./pydantic_test.py
 
+
+
+
+
 Plugin manager
 ==============
 
@@ -39,6 +51,10 @@ List available plugins or a specific plugin config. Most importantly: You can ve
     :func: create_parser
     :prog: ./plugin_manager.py
 
+
+
+
+
 Caldera control
 ===============
 
@@ -49,6 +65,10 @@ Directly control a caldera server. You will need a running caldera server to con
     :func: create_parser
     :prog: ./caldera_control.py
 
+
+
+
+
 Machine control
 ===============
 
@@ -59,6 +79,10 @@ Directly control the machines
     :func: create_parser
     :prog: ./machine_control.py
 
+
+
+
+
 Doc generator
 =============
 

doc_generator.py

@@ -6,15 +6,29 @@ import argparse
 import argcomplete
 from app.doc_generator import DocGenerator
 
-DEFAULT_ATTACK_LOG = "removeme/loot/2021_09_08___07_41_35/attack.json"  # FIN 7 first run on environment
+
+class CmdlineArgumentException(Exception):
+    """ An error in the user supplied command line """
+
+
+def create(arguments):
+    """ Create a document """
+
+    if arguments.attack_log is None:
+        raise CmdlineArgumentException("Creating a new document requires an attack_log")
+
+    doc_get = DocGenerator()
+    doc_get.generate(arguments.attack_log, arguments.outfile)
 
 
 def create_parser():
     """ Creates the parser for the command line arguments"""
-    lparser = argparse.ArgumentParser("Controls an experiment on the configured systems")
-
-    lparser.add_argument("--attack_log", default=DEFAULT_ATTACK_LOG, help="The attack log the document is based on")
-    lparser.add_argument("--outfile", default="tools/human_readable_documentation/source/contents.rst", help="The default output file")
+    lparser = argparse.ArgumentParser("Manage attack documentation")
+    subparsers = lparser.add_subparsers(help="sub-commands")
+    parser_create = subparsers.add_parser("create", help="Create a new human readable document")
+    parser_create.set_defaults(func=create)
+    parser_create.add_argument("--attack_log", default=None, help="The attack log the document is based on")
+    parser_create.add_argument("--outfile", default="tools/human_readable_documentation/source/contents.rst", help="The default output file")
 
     return lparser
 
@@ -22,7 +36,10 @@ def create_parser():
 if __name__ == "__main__":
     parser = create_parser()
     argcomplete.autocomplete(parser)
-    arguments = parser.parse_args()
+    args = parser.parse_args()
 
-    dg = DocGenerator()
-    dg.generate(arguments.attack_log, arguments.outfile)
+    try:
+        str(args.func(args))
+    except CmdlineArgumentException as ex:
+        parser.print_help()
+        print(f"\nCommandline error: {ex}")

experiment_control.py

@@ -42,7 +42,7 @@ def create_parser():
     subparsers = lparser.add_subparsers(help="sub-commands")
 
     lparser.set_defaults(func=explain)
-    lparser.add_argument('--verbose', '-v', action='count', default=0)
+    lparser.add_argument('--verbose', '-v', action='count', default=0, help="Verbosity level")
 
     # Sub parser for machine creation
     parser_run = subparsers.add_parser("run", help="run experiments")

machine_control.py

@@ -13,11 +13,10 @@ from app.attack_log import AttackLog
 
 
 def create_machines(arguments):
-    """
+    """ Create machines based on config
 
     @param arguments: The arguments from argparse
     """
-    # TODO: Add argparse and make it flexible
 
     with open(arguments.configfile) as fh:
         config = yaml.safe_load(fh)
@@ -69,18 +68,18 @@ def download_caldera_client(arguments):
 def create_parser():
     """ Creates the parser for the command line arguments"""
 
-    main_parser = argparse.ArgumentParser("Controls a Caldera server to attack other systems")
-    main_parser.add_argument('--verbose', '-v', action='count', default=0)
+    main_parser = argparse.ArgumentParser("Controls machinery to test VM interaction")
+    main_parser.add_argument('--verbose', '-v', action='count', default=0, help="Verbosity level")
     subparsers = main_parser.add_subparsers(help="sub-commands")
 
     # Sub parser for machine creation
-    parser_create = subparsers.add_parser("create", help="create systems")
+    parser_create = subparsers.add_parser("create", help="Create VM machines")
     parser_create.set_defaults(func=create_machines)
-    parser_create.add_argument("--configfile", default="experiment.yaml", help="Config file to create from")
+    parser_create.add_argument("--configfile", default="experiment.yaml", help="Config file to create VMs from")
 
-    parser_download_caldera_client = subparsers.add_parser("fetch_client", help="download the caldera client")
+    parser_download_caldera_client = subparsers.add_parser("fetch_client", help="Download the caldera client")
     parser_download_caldera_client.set_defaults(func=download_caldera_client)
-    parser_download_caldera_client.add_argument("--ip", default="192.168.178.189", help="Ip of Caldera to connect to")
+    parser_download_caldera_client.add_argument("--ip", default="192.168.178.189", help="IP of Caldera to connect to")
     parser_download_caldera_client.add_argument("--platform", default="windows", help="platform to download the client for")
     parser_download_caldera_client.add_argument("--file", default="sandcat.go", help="The agent to download")
     parser_download_caldera_client.add_argument("--target_dir", default=".", help="The target dir to download the file to")

plugin_manager.py

@@ -10,6 +10,10 @@ from app.pluginmanager import PluginManager
 from app.attack_log import AttackLog
 
 
+class CmdlineArgumentException(Exception):
+    """ An error in the user supplied command line """
+
+
 def list_plugins(arguments):
     """ List plugins """
 
@@ -37,6 +41,10 @@ def get_default_config(arguments):
 
     attack_logger = AttackLog(arguments.verbose)
     plugin_manager = PluginManager(attack_logger)
+    if arguments.subclass_name is None:
+        raise CmdlineArgumentException("Getting configuration requires a subclass_name")
+    if arguments.plugin_name is None:
+        raise CmdlineArgumentException("Getting configuration requires a plugin_name")
     plugin_manager.print_default_config(arguments.subclass_name, arguments.plugin_name)
 
 
@@ -44,7 +52,7 @@ def create_parser():
     """ Creates the parser for the command line arguments"""
 
     main_parser = argparse.ArgumentParser("Manage plugins")
-    main_parser.add_argument('--verbose', '-v', action='count', default=0)
+    main_parser.add_argument('--verbose', '-v', action='count', default=0, help="Verbosity level")
     subparsers = main_parser.add_subparsers(help="sub-commands")
 
     # Sub parser for plugin list
@@ -53,13 +61,13 @@ def create_parser():
     # parser_list.add_argument("--configfile", default="experiment.yaml", help="Config file to create from")
 
     # Sub parser for plugin check
-    parser_list = subparsers.add_parser("check", help="check plugin implementation")
+    parser_list = subparsers.add_parser("check", help="Check plugin implementation")
     parser_list.set_defaults(func=check_plugins)
 
-    parser_default_config = subparsers.add_parser("raw_config", help="print raw default config of the given plugin")
+    parser_default_config = subparsers.add_parser("raw_config", help="Print raw default config of the given plugin")
     parser_default_config.set_defaults(func=get_default_config)
-    parser_default_config.add_argument("subclass_name", help="name of the subclass")
-    parser_default_config.add_argument("plugin_name", help="name of the plugin")
+    parser_default_config.add_argument("subclass_name", help="Name of the subclass")
+    parser_default_config.add_argument("plugin_name", help="Name of the plugin")
 
     # TODO: Get default config
     return main_parser
@@ -71,5 +79,9 @@ if __name__ == "__main__":
     argcomplete.autocomplete(parser)
     args = parser.parse_args()
 
-    exval = args.func(args)
-    sys.exit(exval)
+    try:
+        exit_val = args.func(args)
+        sys.exit(exit_val)
+    except CmdlineArgumentException as ex:
+        parser.print_help()
+        print(f"\nCommandline error: {ex}")

pydantic_test.py

@@ -22,7 +22,7 @@ def create_parser():
     """ Creates the parser for the command line arguments"""
     lparser = argparse.ArgumentParser("Parse a config file and verifies it")
 
-    lparser.add_argument('--filename', default="experiment_ng.yaml")
+    lparser.add_argument('--filename', default="experiment_ng.yaml", help="Config file to verify")
 
     return lparser