From f1d289580bc6fa35d5be3e97f96523d315bb2178 Mon Sep 17 00:00:00 2001 From: Patrick Kennel Date: Wed, 1 Jul 2020 18:00:46 -0700 Subject: [PATCH 1/2] updated README and .gitignore; add example-project; bugfix: `spider.conf` was not being used as default conf file; bugfix: crawl stragegy provider edge case bug; bugfix: do not load args for disabled providers; --- .gitignore | 3 ++ README.md | 29 +++++++++++------ examples/example-project/.python-version | 1 + .../example-project/charlotte.d/Netstat.yaml | 32 +++++++++++++++++++ examples/example-project/charlotte.d/web.yaml | 7 ++++ examples/example-project/requirements.txt | 1 + examples/example-project/spider.conf | 3 ++ itsybitsy/charlotte.py | 4 ++- itsybitsy/itsybitsy.py | 3 +- itsybitsy/itsybitsy_plugins/provider_aws.py | 2 +- itsybitsy/providers.py | 7 ++-- 11 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 examples/example-project/.python-version create mode 100644 examples/example-project/charlotte.d/Netstat.yaml create mode 100644 examples/example-project/charlotte.d/web.yaml create mode 100644 examples/example-project/requirements.txt create mode 100644 examples/example-project/spider.conf diff --git a/.gitignore b/.gitignore index 7e19f1c..c7a203c 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,6 @@ cython_debug/ # IDEs .idea + +# outputs +outputs/ \ No newline at end of file diff --git a/README.md b/README.md index dcc3366..ea6f13a 100644 --- a/README.md +++ b/README.md @@ -11,19 +11,30 @@ Configure charlotte, give it a seed node, and it crawls the graph/tree of your s * dot/graphviz binaries installed in system PATH (e.g. `brew install graphviz`) -## Configure -1. Configure Charlotte. Charlotte is the configuration engine which allows specification of use defined crawl strategies. Please see [charlotte.d/examples/ExampleSSHCrawlStrategy.yaml](charlotte.d/examples/ExampleSSHCrawlStrategy.yaml) for example/documentation. -2. Configure Charlotte Web. "Providers", "skips" (known services to skip crawling for whatever reason), and "Hints" (user defined links in the system) are all defined in [charlotte.d/web.yaml](charlotte.d/web.yaml) -3. Run `itsybitsy --help` for all available commands and `itsybitsy spider --help` and `itsybitsy render --help` for command specific configuration. Set any configurations which are known to be required for every run in a conf file such as [./examples/spider.conf.example](./examples/spider.conf.examle) - 1. NOTE: unlike the `spider` command, `render` is written to stand alone and parse the default json file: it requires no arguments by default. +## Configure itsybitsy in 8 easy steps! +1. Clone itsybitsy + 1. `git clone git@github.com/life360/itsybitsy` +1. Review the example project in [examples/example-project(examples/example-project)] +1. Start a new project / empty folder + 1. `mkdir myitsybitsy && cd myitsybitsy` + 1. `echo "-e /Users/patrick/repos/itsybitsy" > requirements.txt` + 1. `pip install -r requirements.txt` +1. Configure charlotte - the configuration engine with which you will describe your service graph to itsybitsy + 1. `mkdir charlotte.d` + 1. Create a/several `...CrawlStrategy.yaml` file(s). + 1. Please see [examples/ExampleSSHCrawlStrategy.yaml](examples/ExampleSSHCrawlStrategy.yaml) for example/documentation. + 2. Crate `web.yaml` file + 1. "Providers", "skips" , and "Hints" are all defined in [examples/web.yaml](examples/web.yaml). +1. Run `itsybitsy --help` for all available commands and `itsybitsy spider --help` and `itsybitsy render --help` for command specific configuration. +1. Disable builtin provider with the argument `--disable-providers ssh aws k8s` +1. Set any configurations which are known to be required for every run in `spider.conf` see [./examples/spider.conf.example](./examples/spider.conf.example) + 1. Hint: `spider.conf` is always inherited, but you can create different profiles such as `spider.prod.conf` and reference them with the `--profile` arg +1. Note: unlike the `spider` command, `render` is written to stand alone and parse the default json file in `outputs/.lastrun.json` it requires no arguments by default. ## Use #### 1 Run in `spider` mode: -Either provide (a) seed(s) with the `-s` argument, or `--load-json` to load and render. - -##### ascii live output ``` $ itsybitsy spider -s ssh:$SEED_IP foo [seed] (10.1.0.26) @@ -33,7 +44,7 @@ foo [seed] (10.1.0.26) ``` -#### 3 Run the script in `load-json` mode +#### 3 Run in `render` mode It will by default render the "last run" automatically dumped to .lastrun.json. Or you can pass in `-f` to load a specific file. The default renderer is `ascii` unless a different render is passed in, as in `--output graphviz` ``` diff --git a/examples/example-project/.python-version b/examples/example-project/.python-version new file mode 100644 index 0000000..a08ffae --- /dev/null +++ b/examples/example-project/.python-version @@ -0,0 +1 @@ +3.8.2 diff --git a/examples/example-project/charlotte.d/Netstat.yaml b/examples/example-project/charlotte.d/Netstat.yaml new file mode 100644 index 0000000..1fa898f --- /dev/null +++ b/examples/example-project/charlotte.d/Netstat.yaml @@ -0,0 +1,32 @@ +--- +type: "CrawlStrategy" +description: "Discovered TCP connections by reading netstat output" +name: "Netstat" +providers: ["ssh"] +protocol: "TCP" +providerArgs: + shell_command: | + # pre-requisties + which netstat >/dev/null || exit + + # determine listening ports on this instance + listening_ports=$(netstat -lnt | awk '{print $4}' | awk -F: '{print $NF}' | grep '[0-9]' | sort | uniq | tr '\n' '|') + + # get connections on ports we are interested in + netstat -ant | awk '$5 ~ /:(80|8080|11211|3306)$/ {print}' | + + # filter TCP responses - we only want originating requests this filters out HTTP server response to + # clients on ephemeral ports which happen to be in our list of ports of interest + awk '$4 !~ /:('"$listening_ports"')$/ {print $5}' | + + # exclude if self is the dest + grep -v $(hostname | sed 's/ip-//' | tr '-' '.') | + + # only take 1 server per port + tr ':' ' ' | sort -k2 | uniq | awk '$2!=port{print $2,$1;port=$2}' +childProvider: + type: "matchPort" + matches: + 3306: "aws" + 11211: "aws" + default: "ssh" diff --git a/examples/example-project/charlotte.d/web.yaml b/examples/example-project/charlotte.d/web.yaml new file mode 100644 index 0000000..85a1c26 --- /dev/null +++ b/examples/example-project/charlotte.d/web.yaml @@ -0,0 +1,7 @@ +protocols: + TCP: + name: "Netstat" + blocking: true + is_database: false +#hints: +#skips: \ No newline at end of file diff --git a/examples/example-project/requirements.txt b/examples/example-project/requirements.txt new file mode 100644 index 0000000..f6d05a1 --- /dev/null +++ b/examples/example-project/requirements.txt @@ -0,0 +1 @@ +-e "../.." diff --git a/examples/example-project/spider.conf b/examples/example-project/spider.conf new file mode 100644 index 0000000..974a642 --- /dev/null +++ b/examples/example-project/spider.conf @@ -0,0 +1,3 @@ +ssh-name-command = hostname +disable-providers = [k8s] +aws-service-name-tag = service_name diff --git a/itsybitsy/charlotte.py b/itsybitsy/charlotte.py index 05bcb5c..d1ba284 100644 --- a/itsybitsy/charlotte.py +++ b/itsybitsy/charlotte.py @@ -65,10 +65,12 @@ def determine_child_provider(self, protocol_mux: str, address: str = None) -> Op try: if int(protocol_mux) in self.child_provider['matches']: return self.child_provider['matches'][int(protocol_mux)] + return self.child_provider['default'] except (ValueError, IndexError): return self.child_provider['default'] - raise CrawlStrategyException(f"child provider match type: {self.child_provider['type']} not supported") + logs.logger.fatal(f"child provider match type: {self.child_provider['type']} not supported") + raise CrawlStrategyException() def rewrite_service_name(self, service_name: str, node) -> str: """ diff --git a/itsybitsy/itsybitsy.py b/itsybitsy/itsybitsy.py index d8e48a9..9c342be 100644 --- a/itsybitsy/itsybitsy.py +++ b/itsybitsy/itsybitsy.py @@ -66,7 +66,6 @@ def _format_action_invocation(self, action): formatter_class = lambda prog: ConciseHelpFormatter(prog, max_help_position=100, width=200) global argparser, spider_subparser argparser = configargparse.ArgumentParser( - default_config_files=['./spider.conf'], description=( "Give it (a) seed host(s).\n" "It will crawl them.\n" @@ -81,7 +80,7 @@ def _format_action_invocation(self, action): subparsers.required = True subparsers.dest = 'command' spider_p = subparsers.add_parser(command_spider, help='Crawl a network of services - given a seed', - formatter_class=formatter_class) + formatter_class=formatter_class, default_config_files=['./spider.conf']) render_p = subparsers.add_parser(command_render, help='Render results of a previous crawl', formatter_class=formatter_class) spider_subparser = spider_p diff --git a/itsybitsy/itsybitsy_plugins/provider_aws.py b/itsybitsy/itsybitsy_plugins/provider_aws.py index dcc93b3..e130ba3 100644 --- a/itsybitsy/itsybitsy_plugins/provider_aws.py +++ b/itsybitsy/itsybitsy_plugins/provider_aws.py @@ -39,7 +39,7 @@ def register_cli_args(argparser: ProviderArgParser): 'This will override the AWS_PROFILE environment variable.') argparser.add_argument('--service-name-tag', required=True, metavar='TAG', help='AWS tag associated with service name') - argparser.add_argument('--tag-filters', nargs='*', metavar='FILTER', + argparser.add_argument('--tag-filters', nargs='*', metavar='FILTER', default = [], help='Additional AWS tags to filter on or services. Specified in format: ' '"TAG_NAME=VALUE" pairs') diff --git a/itsybitsy/providers.py b/itsybitsy/providers.py index 26fc5dc..293fa74 100644 --- a/itsybitsy/providers.py +++ b/itsybitsy/providers.py @@ -149,13 +149,16 @@ def init(): :return: """ - for provider_class in [cls for cls in ProviderInterface.__subclasses__() - if cls.ref() not in constants.ARGS.disable_providers]: + for provider_class in _enabled_providers(): if provider_class.ref() in provider_registry: raise ProviderClobberException(f"Provider {provider_class.ref()} already registered!") provider_registry[provider_class.ref()] = provider_class() +def _enabled_providers() -> List[ProviderInterface]: + return [cls for cls in ProviderInterface.__subclasses__() if cls.ref() not in constants.ARGS.disable_providers] + + def get(provider_ref: str) -> ProviderInterface: """ Take a provider string reference and return a singleton instance of the provider From 13a71b733964488b052a40922e07c46a715b8206 Mon Sep 17 00:00:00 2001 From: Patrick Kennel Date: Wed, 1 Jul 2020 18:13:34 -0700 Subject: [PATCH 2/2] add ci --- .circleci/config.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++ Makefile | 8 ++++++++ 2 files changed, 52 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 Makefile diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..08539c8 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,44 @@ +version: 2.1 + +orbs: + python: circleci/python@0.3.0 + +jobs: + test-and-code-analysis: + docker: + - image: circleci/python:3.8.3 + steps: + - checkout + - run: sudo chown -R circleci:circleci /usr/local/bin + - run: sudo chown -R circleci:circleci /usr/local/lib/python3.8/site-packages + - restore_cache: + key: deps9-{{ .Branch }}-{{ checksum "itsybitsy/setup.py" }} + - run: + command: | + python3 -m venv .venv + source .venv/bin/activate + pip install -r requirements.txt + - save_cache: + key: deps9-{{ .Branch }}-{{ checksum "itsybitsy/setup.py" }} + paths: + - ".venv" + - "/usr/local/bin" + - "/usr/local/lib/python3.8/site-packages" + - run: + command: | + source .venv/bin/activate + make test + - run: + command: | + source .venv/bin/activate + make coverage + - run: + command: | + source .venv/bin/activate + make analyze + + +workflows: + main: + jobs: + - test-and-code-analysis \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a76b36a --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +test: + @pytest + +coverage: + @pytest --cov --cov-fail-under=70 --cov-config .coveragerc + +analyze: + @prospector --profile .prospector.yaml \ No newline at end of file