Logging, filehandling, and packaging updates

README.md

@@ -18,14 +18,14 @@ Future versions will include fine-grained ability to control detailed aspects of
 ### ToDo
 - [ ] Output formats: HTML, SQL dump
 - [ ] Country relationship details
-- [ ] Detailed representation of organization networks (asset improvement)
+- [ ] Detailed representation of defender networks (asset improvement)
 - [ ] Visualization of relationships between data points
 - [ ] Improved world map generation
 - [ ] "web feeds" of intelligence/events (ex. news reports, dark web posts, etc.)
 
 ## Components
 
-- Agents: Threat actors, organizations (companies)
+- Agents: Threat actors, defenders (companies)
     - Friendly, enemy, and neutral players in the simulation
 - Assets: Cyber infrastructure
     - Networks, software, hardware, configurations, and vulnerabilities
@@ -46,10 +46,9 @@ CDAS installs the following packages and their dependencies upon setup:
 numpy
 reportlab
 drawSVG
+cyberdem
 ```
 
-You may also want to install ```libcairo2```. Optional, but you will receive errors when running CDAS without it.
-
 ### Installing
 
 1. Download CDAS and unzip the download folder
@@ -62,7 +61,7 @@ $ pip3 install .
 3. To test that CDAS is installed properly run
 
 ```
-$ python3 -m cdas -c sample_configs/randomize_all_small_pdf.json
+$ python3 -m cdas -c sample_configs/randomize_all_small_pdf.json -v
 Setting up directories...
 Creating fake countries...
 Creating fake threat actors...
@@ -77,13 +76,13 @@ Saving output...
 Done
 ```
 
-CDAS should finish with no errors (you will get warnings about libcairo2 and CairoSVG if you did not install that) and the results will be in a folder called cdas-output. Results will include
+CDAS should finish with no errors and the results will be in a folder called cdas-output. Results will include
 - SVG map of countries
 - A "pdf" folder containing
     - 'actors' folder containing PDF files with threat actor descriptions
     - 'countries' folder containing PDF files with country attributes
     - 'reports' folder containing PDF files with event reports
-    - 'organizations' folder containing PDF files with organization descriptions
+    - 'defenders' folder containing PDF files with organization descriptions
 
 ## Configuration
 
@@ -93,7 +92,7 @@ Additionally, there are three available command line flags: the required config-
 
 ```
 $ python3 -m cdas -h
-usage: __main__.py [-h] -c CONFIG_FILE [-i INPUT_DIRECTORY] [-o OUTPUT_DIRECTORY]
+usage: __main__.py [-h] -c CONFIG_FILE [-i INPUT_DIRECTORY] [-o OUTPUT_DIRECTORY] [--verbose]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -103,6 +102,7 @@ optional arguments:
                         directory for specifying custom data
   -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                         directory for storing results
+  --verbose, -v         v for basic status, vv for detailed status
 ```
 
 ## Data Customization

cdas/__init__.py

@@ -1,4 +1,4 @@
 # __init__.py
 
 # Version of the cdas package
-__version__ = "0.0.2"
+__version__ = "0.0.5"

cdas/__main__.py

@@ -39,15 +39,17 @@
 import argparse
 from datetime import datetime, timedelta
 import json
+import logging
 import numpy as np
 import os
 import pkg_resources
 import shutil
 import sys
+import cyberdem
+import uuid
 # Import custom modules
 from . import context, agents, simulator, filestore
 
-
 def main():
 
     parser = argparse.ArgumentParser()
@@ -60,8 +62,16 @@ def main():
     parser.add_argument(
         "-o", "--output-directory",
         help="directory for storing results")
+    parser.add_argument('--verbose', '-v', action='count', default=0,
+        help="v for basic status, vv for detailed status")
 
     args = parser.parse_args()
+    if args.verbose == 1:
+        logging.basicConfig(
+            format='%(message)s', level=logging.INFO)
+    elif args.verbose == 2:
+        logging.basicConfig(
+            format='%(message)s', level=logging.DEBUG)
 
     # Load the configuration file
     with open(args.config_file, 'r') as f:
@@ -73,12 +83,13 @@ def main():
         'defenders': '',
         'threat-actors': '',
         'malware': '',
+        'networks': '',
         'geoseed.json': '',
         'tools': '',
         'ttps': ''
         }
 
-    print("Setting up directories...")
+    logging.info("Setting up directories...")
     # Set up the Output directory
     if not args.output_directory:
         args.output_directory = config['output']['output_directory']
@@ -90,7 +101,7 @@ def main():
     # Set up the temp directory
     if config['output']['temp_directory'] == "":
         raise Exception(f'No temporary directory specified')
-    temp_dir = filestore.FileStore(
+    filestore.FileStore(
         config['output']['temp_directory'], "temp", write=True)
 
     # Check the input folder if provided
@@ -143,11 +154,11 @@ def main():
     # Load or create country data
     if datastore['countries'] != '':
         # Using custom data
-        print("Loading custom country data...")
+        logging.info("Loading custom country data...")
         countries_fs = filestore.FileStore(
             datastore['countries'], context.Country)
     elif config['countries']['randomize'] is True:
-        print("Creating fake countries...")
+        logging.info("Creating fake countries...")
         countries_fs = filestore.FileStore(
             os.path.join(config['output']['temp_directory'], 'countries'),
             context.Country, write=True)
@@ -175,7 +186,7 @@ def main():
             countries_fs.save(country, overwrite=True)
     else:
         # Using country data files instead of random generation
-        print("Loading default country data...")
+        logging.info("Loading default country data...")
         countries_fs = filestore.FileStore(
             pkg_resources.resource_filename(
                 __name__, 'data/cia_world_factbook/'), context.Country)
@@ -185,8 +196,8 @@ def main():
         path = args.output_directory + "/" + ot
         os.mkdir(path)
         os.mkdir(path + '/countries/')
-        for country in countries_fs.get(
-                [i[0] for i in countries_fs.query("SELECT id")]):
+        for i in countries_fs.query("SELECT id"):
+            country = countries_fs.get(i[0])
             output_dir.output(ot+'/countries', country, ot)
         if ot == "html":
             html_src = pkg_resources.resource_filename(
@@ -197,9 +208,10 @@ def main():
             f = open(path+'/COUNTRY.html', 'r')
             c_template = f.read()
             f.close()
+            countries = [c[0] for c in countries_fs.query("SELECT name")]
             for country in countries:
-                f = open(path + '/countries/' + country.name + '.html', 'w')
-                f.write(c_template.replace('COUNTRY', country.name))
+                f = open(path + '/countries/' + country + '.html', 'w')
+                f.write(c_template.replace('COUNTRY', country))
                 f.close()
             os.remove(path+'/COUNTRY.html')
 
@@ -209,17 +221,20 @@ def main():
             "assets/stix_vocab.json"), encoding='utf-8') as json_file:
         stix_vocab = json.load(json_file)
     json_file.close()
-    tools = tools_fs.get([name[0] for name in tools_fs.query("SELECT id")])
-    malwares = malware_fs.get([n[0] for n in malware_fs.query("SELECT id")])
-    ttps = ttp_fs.get([name[0] for name in ttp_fs.query("SELECT id")])
+    names = tools_fs.query("SELECT id")
+    tools = [tools_fs.get(name[0]) for name in names]
+    names = malware_fs.query("SELECT id")
+    malwares = [malware_fs.get(name[0]) for name in names]
+    names = ttp_fs.query("SELECT id")
+    ttps = [ttp_fs.get(name[0]) for name in names]
 
     if datastore['threat-actors'] != '':
-        print("Loading custom threat actor data...")
+        logging.info("Loading custom threat actor data...")
         # Using custom threat actors provided by the user in the input folder
         threat_actor_fs = filestore.FileStore(
             datastore['threat-actors'], agents.ThreatActor)
     elif config['agents']['randomize_threat_actors'] is True:
-        print("Creating fake threat actors...")
+        logging.info("Creating fake threat actors...")
         threat_actor_fs = filestore.FileStore(
             os.path.join(config['output']['temp_directory'], 'threat-actors'),
             agents.ThreatActor, write=True)
@@ -245,15 +260,15 @@ def main():
                 stix_vocab['threat-actor-sophistication'])
             actors += 1
     else:
-        print("Loading default threat actor data...")
+        logging.info("Loading default threat actor data...")
         threat_actor_fs = filestore.FileStore(
             pkg_resources.resource_filename(
                 __name__, 'assets/mitre_cti/threat-actors/'),
             agents.ThreatActor)
 
     # Output threat actor reports
-    actors = threat_actor_fs.get(
-        [name[0] for name in threat_actor_fs.query("SELECT id")])
+    names = threat_actor_fs.query("SELECT id")
+    actors = [threat_actor_fs.get(name[0]) for name in names]
     for ot in config['output']['output_types']:
         os.mkdir(args.output_directory + "/" + ot + '/actors/')
         for apt in actors:
@@ -262,12 +277,12 @@ def main():
 
     # Create or load defending organizations
     if datastore['defenders'] != '':
-        print("Loading custom defender data...")
+        logging.info("Loading custom defender data...")
         # Using custom defenders provided by the user in the input folder
         defender_fs = filestore.FileStore(
             datastore['defenders'], agents.Defender)
     else:
-        print("Creating random defending organizations...")
+        logging.info("Creating random defending organizations...")
         defender_fs = filestore.FileStore(
             os.path.join(config['output']['temp_directory'], 'defenders'),
             agents.Defender, write=True)
@@ -315,54 +330,72 @@ def main():
                 defs += 1
 
     # Output defender info
-    defenders = defender_fs.get(
-        [name[0] for name in defender_fs.query("SELECT id")])
+    names = defender_fs.query("SELECT id")
+    defenders = [defender_fs.get(name[0]) for name in names]
     for ot in config['output']['output_types']:
         os.mkdir(args.output_directory + "/" + ot + '/defenders/')
         for d in defenders:
             output_dir.output(ot+'/defenders', d, ot)
 
     # Create or load networks of defenders
-
-    # Output network information
-
+    if datastore['networks'] != '':
+        logging.info("Loading custom network data...")
+        # Using custom networks provided by the user in the input folder
+        network_fs = filestore.FileStore(
+            datastore['networks'], 'Networks')
+    else:
+        logging.info("Creating random networks of defenders...")
+        network_fs = filestore.FileStore(
+            os.path.join(config['output']['temp_directory'], 'networks'),
+            'Networks', write=True)
+        for d in defenders:
+            net_name = 'network--'+str(uuid.uuid4())
+            fs = cyberdem.filesystem.FileSystem(os.path.join(
+                config['output']['temp_directory'], 'networks', net_name))
+            context.random_network(fs, 100)
+            relationships.append((d.id, 'owns', net_name))
 
     # Run simulation
-    print('Running simulation...')
+    logging.info('Running simulation...')
     events_fs = filestore.FileStore(
         os.path.join(config['output']['temp_directory'], 'events'),
         context.Event, write=True)
-    # For now, we're assuming all of the attackers decide to attack the one
-    # defender, who is too dumb to make changes the network
     simulator.simulate(
-        actors, defenders, tools, malwares, events_fs, relationships, start)
-    # We don't know ahead of time how many moves will be made, so go back to 
-    # the events and set to the desired time frame
+        actors, defenders, config['defenders']['allow_defense'], events_fs,
+        relationships, stix_vocab['threat-actor-sophistication'])
+    # We don't know ahead of time how many moves will be made, so once the 
+    # simulation is done, go back to the events and set to the correct day/time
+    events = [events_fs.get(i[0]) for i in events_fs.query("SELECT id")]
     start = datetime.strptime(
         config["simulation"]['time_range'][0], '%Y-%m-%d')
     end = datetime.strptime(config["simulation"]['time_range'][1], '%Y-%m-%d')
-    time_increment = (end - start)/num_events
-    print("@TODO: Event date times...")
+    time_increment = (end - start)/len(events)
+    newlist = sorted(events, key=lambda x: x.date)
+    for e in newlist:
+        e.date = start
+        e.name = "Report_" + start.strftime("%Y%m%d_%H%M%S")
+        events_fs.save(e, overwrite=True)
+        start += time_increment
 
     # Create output files
-    print('Saving output...')
+    logging.info('Saving output...')
     # Map
     try:
         map_matrix.plot_map(args.output_directory, **country_names)
     except NameError:
         pass
 
     for ot in config['output']['output_types']:
-        print(f'    {ot}...')
+        logging.debug(f'    {ot}...')
         path = args.output_directory + "/" + ot
         os.mkdir(path + '/reports/')
-        print(f'\t  Events...')
-        for e in events_fs.get([i[0] for i in events_fs.query("SELECT id")]):
+        logging.debug(f'\t  Events...')
+        for e in [events_fs.get(i[0]) for i in events_fs.query("SELECT id")]:
             output_dir.output(ot+'/reports', e, ot)
 
     shutil.rmtree(config['output']['temp_directory'])
 
-    print('Done')
+    logging.info('Done')
 
 
 if __name__ == "__main__":

cdas/assets/mitre_cti/attack-patterns/attack-pattern--028ad431-84c5-4eb7-a364-2b797c234f88.json

@@ -1,7 +1,7 @@
 {
     "id": "attack-pattern--028ad431-84c5-4eb7-a364-2b797c234f88",
     "name": "Acquire OSINT data sets and information",
-    "description": "Data sets can be anything from Security Exchange Commission (SEC) filings to public phone numbers. Many datasets are now either publicly available for free or can be purchased from a variety of data vendors.  Open source intelligence (OSINT) is intelligence gathered from publicly available sources. This can include both information gathered on-line as well as in the physical world. (Citation: SANSThreatProfile) (Citation: Infosec-osint) (Citation: isight-osint)",
+    "description": "This object is deprecated as its content has been merged into the enterprise domain. Please see the [PRE](http://attack.mitre.org/matrices/enterprise/pre/) matrix for its replacement. The prior content of this page has been preserved [here](https://attack.mitre.org/versions/v7/techniques/T1277).\n\nData sets can be anything from Security Exchange Commission (SEC) filings to public phone numbers. Many datasets are now either publicly available for free or can be purchased from a variety of data vendors.  Open source intelligence (OSINT) is intelligence gathered from publicly available sources. This can include both information gathered on-line as well as in the physical world. (Citation: SANSThreatProfile) (Citation: Infosec-osint) (Citation: isight-osint)",
     "references": [
         {
             "source_name": "mitre-pre-attack",

cdas/assets/mitre_cti/attack-patterns/attack-pattern--03259939-0b57-482f-8eb5-87c0e0d54334.json

@@ -16,7 +16,8 @@
     ],
     "platforms": [
         "macOS",
-        "Windows"
+        "Windows",
+        "Linux"
     ],
     "kill_chain": [
         {

cdas/assets/mitre_cti/attack-patterns/attack-pattern--03da0598-ed46-4a73-bf43-0313b3522400.json

@@ -1,7 +1,7 @@
 {
     "id": "attack-pattern--03da0598-ed46-4a73-bf43-0313b3522400",
     "name": "Submit KITs, KIQs, and intelligence requirements",
-    "description": "Once they have been created, intelligence requirements, Key Intelligence Topics (KITs), and Key Intelligence Questions (KIQs) are submitted into a central management system. (Citation: ICD204) (Citation: KIT-Herring)",
+    "description": "This object is deprecated as its content has been merged into the enterprise domain. Please see the [PRE](http://attack.mitre.org/matrices/enterprise/pre/) matrix for its replacement. The prior content of this page has been preserved [here](https://attack.mitre.org/versions/v7/techniques/T1237).\n\nOnce they have been created, intelligence requirements, Key Intelligence Topics (KITs), and Key Intelligence Questions (KIQs) are submitted into a central management system. (Citation: ICD204) (Citation: KIT-Herring)",
     "references": [
         {
             "source_name": "mitre-pre-attack",

cdas/assets/mitre_cti/attack-patterns/attack-pattern--03f4a766-7a21-4b5e-9ccf-e0cf422ab983.json

@@ -1,16 +1,17 @@
 {
     "id": "attack-pattern--03f4a766-7a21-4b5e-9ccf-e0cf422ab983",
     "name": "Acquire or compromise 3rd party signing certificates",
-    "description": "Code signing is the process of digitally signing executables and scripts to confirm the software author and guarantee that the code has not been altered or corrupted. Users may trust a signed piece of code more than an unsigned piece of code even if they don't know who issued the certificate or who the author is. (Citation: DiginotarCompromise)",
+    "description": "This object is deprecated as its content has been merged into the enterprise domain. Please see the [PRE](http://attack.mitre.org/matrices/enterprise/pre/) matrix for its replacement. The prior content of this page has been preserved [here](https://attack.mitre.org/versions/v7/techniques/T1332).\n\nCode signing is the process of digitally signing executables and scripts to confirm the software author and guarantee that the code has not been altered or corrupted. Users may trust a signed piece of code more than an unsigned piece of code even if they don't know who issued the certificate or who the author is. (Citation: DiginotarCompromise)",
     "references": [
         {
             "source_name": "mitre-pre-attack",
             "url": "https://attack.mitre.org/techniques/T1332",
             "external_id": "T1332"
         },
         {
-            "description": "Dennis Fisher. (2012, October 31). FINAL REPORT ON DIGINOTAR HACK SHOWS TOTAL COMPROMISE OF CA SERVERS. Retrieved March 6, 2017.",
-            "source_name": "DiginotarCompromise"
+            "description": "Fisher, D. (2012, October 31). Final Report on DigiNotar Hack Shows Total Compromise of CA Servers. Retrieved March 6, 2017.",
+            "source_name": "DiginotarCompromise",
+            "url": "https://threatpost.com/final-report-diginotar-hack-shows-total-compromise-ca-servers-103112/77170/"
         }
     ],
     "kill_chain": [

cdas/assets/mitre_cti/attack-patterns/attack-pattern--0440f60f-9056-4791-a740-8eae96eb61fa.json

@@ -9,8 +9,9 @@
             "url": "https://attack.mitre.org/techniques/T1386"
         },
         {
-            "description": "PETER BRIGHT. (2011, February 15). Anonymous speaks: the inside story of the HBGary hack. Retrieved March 9, 2017.",
-            "source_name": "AnonHBGary"
+            "source_name": "AnonHBGary",
+            "description": "Bright, P. (2011, February 15). Anonymous speaks: the inside story of the HBGary hack. Retrieved March 9, 2017.",
+            "url": "https://arstechnica.com/tech-policy/2011/02/anonymous-speaks-the-inside-story-of-the-hbgary-hack/"
         }
     ],
     "kill_chain": [