Merge pull request #1 from life360/pk/plugnplay

readme, some bug fixes

.circleci/config.yml

@@ -0,0 +1,44 @@
+version: 2.1
+
+orbs:
+  python: circleci/[email protected]
+
+jobs:
+  test-and-code-analysis:
+    docker:
+      - image: circleci/python:3.8.3
+    steps:
+      - checkout
+      - run: sudo chown -R circleci:circleci /usr/local/bin
+      - run: sudo chown -R circleci:circleci /usr/local/lib/python3.8/site-packages
+      - restore_cache:
+          key: deps9-{{ .Branch }}-{{ checksum "itsybitsy/setup.py" }}
+      - run:
+          command: |
+            python3 -m venv .venv
+            source .venv/bin/activate
+            pip install -r requirements.txt
+      - save_cache:
+          key: deps9-{{ .Branch }}-{{ checksum "itsybitsy/setup.py" }}
+          paths:
+            - ".venv"
+            - "/usr/local/bin"
+            - "/usr/local/lib/python3.8/site-packages"
+      - run:
+          command: |
+            source .venv/bin/activate
+            make test
+      - run:
+          command: |
+            source .venv/bin/activate
+            make coverage
+      - run:
+          command: |
+            source .venv/bin/activate
+            make analyze
+
+
+workflows:
+  main:
+    jobs:
+      - test-and-code-analysis

.gitignore

@@ -139,3 +139,6 @@ cython_debug/
 
 # IDEs
 .idea
+
+# outputs
+outputs/

Makefile

@@ -0,0 +1,8 @@
+test:
+	@pytest
+
+coverage:
+	@pytest --cov --cov-fail-under=70 --cov-config .coveragerc
+
+analyze:
+	@prospector --profile .prospector.yaml

README.md

@@ -11,19 +11,30 @@ Configure charlotte, give it a seed node, and it crawls the graph/tree of your s
 * dot/graphviz binaries installed in system PATH (e.g. `brew install graphviz`)
 
 
-## Configure
-1. Configure Charlotte.  Charlotte is the configuration engine which allows specification of use defined crawl strategies.  Please see [charlotte.d/examples/ExampleSSHCrawlStrategy.yaml](charlotte.d/examples/ExampleSSHCrawlStrategy.yaml) for example/documentation.
-2. Configure Charlotte Web.   "Providers", "skips" (known services to skip crawling for whatever reason), and "Hints" (user defined links in the system) are all defined in [charlotte.d/web.yaml](charlotte.d/web.yaml)
-3. Run `itsybitsy --help` for all available commands and `itsybitsy spider --help` and `itsybitsy render --help` for command specific configuration.  Set any configurations which are known to be required for every run in a conf file such as [./examples/spider.conf.example](./examples/spider.conf.examle)
-    1. NOTE: unlike the `spider` command, `render` is written to stand alone and parse the default json file: it requires no arguments by default.
+## Configure itsybitsy in 8 easy steps!
+1. Clone itsybitsy
+    1. `git clone [email protected]/life360/itsybitsy`
+1. Review the example project in [examples/example-project(examples/example-project)]
+1. Start a new project / empty folder
+    1. `mkdir myitsybitsy && cd myitsybitsy`
+    1. `echo "-e /Users/patrick/repos/itsybitsy" > requirements.txt`
+    1. `pip install -r requirements.txt`
+1. Configure charlotte - the configuration engine with which you will describe your service graph to itsybitsy
+    1. `mkdir charlotte.d`
+    1. Create a/several `...CrawlStrategy.yaml` file(s).
+        1. Please see [examples/ExampleSSHCrawlStrategy.yaml](examples/ExampleSSHCrawlStrategy.yaml) for example/documentation.
+    2. Crate `web.yaml` file  
+        1. "Providers", "skips" , and "Hints" are all defined in [examples/web.yaml](examples/web.yaml). 
+1. Run `itsybitsy --help` for all available commands and `itsybitsy spider --help` and `itsybitsy render --help` for command specific configuration.
+1. Disable builtin provider with the argument `--disable-providers ssh aws k8s`
+1. Set any configurations which are known to be required for every run in `spider.conf` see [./examples/spider.conf.example](./examples/spider.conf.example)
+  1. Hint: `spider.conf` is always inherited, but you can create different profiles such as `spider.prod.conf` and reference them with the `--profile` arg
+1. Note: unlike the `spider` command, `render` is written to stand alone and parse the default json file in `outputs/.lastrun.json` it requires no arguments by default.
 
 
 ## Use
 #### 1 Run in `spider` mode:
 
-Either provide (a) seed(s) with the `-s` argument, or `--load-json` to load and render. 
-
-##### ascii live output
 ```
 $ itsybitsy spider -s ssh:$SEED_IP
 foo [seed] (10.1.0.26)
@@ -33,7 +44,7 @@ foo [seed] (10.1.0.26)
 ```
 
 
-#### 3 Run the script in `load-json` mode
+#### 3 Run in `render` mode
 It will by default render the "last run" automatically dumped to .lastrun.json.  Or you can pass in `-f` to load a specific file.  The default renderer is `ascii` unless a different render is passed in, as in `--output graphviz`
 
 ``` 

examples/example-project/.python-version

@@ -0,0 +1 @@
+3.8.2

examples/example-project/charlotte.d/Netstat.yaml

@@ -0,0 +1,32 @@
+---
+type: "CrawlStrategy"
+description: "Discovered TCP connections by reading netstat output"
+name: "Netstat"
+providers: ["ssh"]
+protocol: "TCP"
+providerArgs:
+    shell_command: |
+        # pre-requisties
+        which netstat >/dev/null || exit
+
+        # determine listening ports on this instance
+        listening_ports=$(netstat -lnt | awk '{print $4}' | awk -F: '{print $NF}' | grep '[0-9]' | sort | uniq | tr '\n' '|')
+
+        # get connections on ports we are interested in
+        netstat -ant | awk '$5 ~ /:(80|8080|11211|3306)$/ {print}' |
+
+        # filter TCP responses - we only want originating requests this filters out HTTP server response to
+        # clients on ephemeral ports which happen to be in our list of ports of interest
+        awk '$4 !~ /:('"$listening_ports"')$/ {print $5}' |
+
+        # exclude if self is the dest
+        grep -v $(hostname | sed 's/ip-//' | tr '-' '.') |
+
+        # only take 1 server per port
+        tr ':' ' ' | sort -k2 | uniq | awk '$2!=port{print $2,$1;port=$2}'
+childProvider:
+    type: "matchPort"
+    matches:
+        3306: "aws"
+        11211: "aws"
+    default: "ssh"

examples/example-project/charlotte.d/web.yaml

@@ -0,0 +1,7 @@
+protocols:
+  TCP:
+    name: "Netstat"
+    blocking: true
+    is_database: false
+#hints:
+#skips:

examples/example-project/requirements.txt

@@ -0,0 +1 @@
+-e "../.."

examples/example-project/spider.conf

@@ -0,0 +1,3 @@
+ssh-name-command = hostname
+disable-providers = [k8s]
+aws-service-name-tag = service_name

itsybitsy/charlotte.py

@@ -65,10 +65,12 @@ def determine_child_provider(self, protocol_mux: str, address: str = None) -> Op
             try:
                 if int(protocol_mux) in self.child_provider['matches']:
                     return self.child_provider['matches'][int(protocol_mux)]
+                return self.child_provider['default']
             except (ValueError, IndexError):
                 return self.child_provider['default']
 
-        raise CrawlStrategyException(f"child provider match type: {self.child_provider['type']} not supported")
+        logs.logger.fatal(f"child provider match type: {self.child_provider['type']} not supported")
+        raise CrawlStrategyException()
 
     def rewrite_service_name(self, service_name: str, node) -> str:
         """

itsybitsy/itsybitsy.py

@@ -66,7 +66,6 @@ def _format_action_invocation(self, action):
     formatter_class = lambda prog: ConciseHelpFormatter(prog, max_help_position=100, width=200)
     global argparser, spider_subparser
     argparser = configargparse.ArgumentParser(
-        default_config_files=['./spider.conf'],
         description=(
             "Give it (a) seed host(s).\n"
             "It will crawl them.\n"
@@ -81,7 +80,7 @@ def _format_action_invocation(self, action):
     subparsers.required = True
     subparsers.dest = 'command'
     spider_p = subparsers.add_parser(command_spider, help='Crawl a network of services - given a seed',
-                                     formatter_class=formatter_class)
+                                     formatter_class=formatter_class, default_config_files=['./spider.conf'])
     render_p = subparsers.add_parser(command_render, help='Render results of a previous crawl',
                                      formatter_class=formatter_class)
     spider_subparser = spider_p

itsybitsy/itsybitsy_plugins/provider_aws.py

@@ -39,7 +39,7 @@ def register_cli_args(argparser: ProviderArgParser):
                                                   'This will override the AWS_PROFILE environment variable.')
         argparser.add_argument('--service-name-tag', required=True, metavar='TAG',
                                help='AWS tag associated with service name')
-        argparser.add_argument('--tag-filters', nargs='*',  metavar='FILTER',
+        argparser.add_argument('--tag-filters', nargs='*',  metavar='FILTER', default = [],
                                help='Additional AWS tags to filter on or services.  Specified in format: '
                                     '"TAG_NAME=VALUE" pairs')
 

itsybitsy/providers.py

@@ -149,13 +149,16 @@ def init():
 
     :return:
     """
-    for provider_class in [cls for cls in ProviderInterface.__subclasses__()
-                           if cls.ref() not in constants.ARGS.disable_providers]:
+    for provider_class in _enabled_providers():
         if provider_class.ref() in provider_registry:
             raise ProviderClobberException(f"Provider {provider_class.ref()} already registered!")
         provider_registry[provider_class.ref()] = provider_class()
 
 
+def _enabled_providers() -> List[ProviderInterface]:
+    return [cls for cls in ProviderInterface.__subclasses__() if cls.ref() not in constants.ARGS.disable_providers]
+
+
 def get(provider_ref: str) -> ProviderInterface:
     """
     Take a provider string reference and return a singleton instance of the provider