From aab35468d0af63277358efb42f18127b1d665854 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Fri, 15 Dec 2017 14:16:47 -0800 Subject: [PATCH] Release v1.1 (#93) --- docs/source/adding-yara-rules.rst | 8 ++-- docs/source/conf.py | 4 +- docs/source/credits.rst | 46 ++++++++++++++++++++++ docs/source/index.rst | 1 + docs/source/troubleshooting-faq.rst | 2 + lambda_functions/analyzer/README.rst | 2 +- lambda_functions/analyzer/yara_analyzer.py | 17 ++++++-- manage.py | 2 +- terraform/lambda_iam.tf | 11 ++++-- terraform/terraform.tfvars | 4 +- 10 files changed, 79 insertions(+), 18 deletions(-) create mode 100644 docs/source/credits.rst diff --git a/docs/source/adding-yara-rules.rst b/docs/source/adding-yara-rules.rst index d18590e..a0496fd 100644 --- a/docs/source/adding-yara-rules.rst +++ b/docs/source/adding-yara-rules.rst @@ -8,6 +8,8 @@ Included Rules BinaryAlert includes a number of `custom YARA rules `_ written by Airbnb's analysts which detect a variety of hacktools, malware, and ransomware. All included rules have been tested against a corpus of more than 2 million binaries to ensure the highest fidelity. +.. _clone-yara-rules: + Clone Rules From Other Projects ------------------------------- BinaryAlert makes it easy to clone YARA rules from other open-source projects: @@ -16,11 +18,7 @@ BinaryAlert makes it easy to clone YARA rules from other open-source projects: $ ./manage.py clone_rules -This will copy a subset of YARA rules from each of the following repositories: - -* `Neo23x0/signature-base `_ -* `YARA-Rules/rules `_ - +This will copy a subset of YARA rules from several :ref:`open-source collections `. You can add more rule sources in `rules/clone_rules.py `_ diff --git a/docs/source/conf.py b/docs/source/conf.py index afc0196..4908aff 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -63,9 +63,9 @@ # built documents. # # The short X.Y version. -version = '1.0' +version = '1.1' # The full version, including alpha/beta/rc tags. -release = '1.0.0' +release = '1.1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/credits.rst b/docs/source/credits.rst new file mode 100644 index 0000000..7040292 --- /dev/null +++ b/docs/source/credits.rst @@ -0,0 +1,46 @@ +Credits +======= + +People +------ +BinaryAlert is brought to you by `Airbnb `_: + +- `Austin Byers `_ (Architect, Primary Engineer) +- `mime-frame `_ (Concept, Design Review, YARA Rules) +- `Daimon `_ (YARA Rules) +- And many others in the `full list of contributors `_ + + +.. _yara-credits: + +YARA Rules +---------- +When :ref:`cloning YARA rules from other projects `, subsets of the following +collections are included by default: + +- `Neo23x0/signature-base `_ +- `YARA-Rules/rules `_ + + +Open-Source Tools +----------------- +We are proud to contribute to the open-source community, without which BinaryAlert would not be +possible. BinaryAlert relies on several open-source tools and libraries: + +- `backoff `_: Function decoration for backoff and retry +- `boto3 `_: AWS SDK for Python +- `cbapi `_: Carbon Black API for Python +- `pyhcl `_: Python parser for HCL (e.g. Terraform configuration) +- `terraform `_: Infrastructure-as-Code +- `yara `_: Pattern matching for malware analysis +- `yara-python `_: The Python interface for YARA +- `yextend `_: YARA analysis of archive data + + +Bundled Software +................ +The following tools are pre-compiled for use in Lambda and included in the BinaryAlert repo: + +- `cbapi-python `_ | `LICENSE `__ +- `yara-python `_ | `LICENSE `__ +- `yextend `_ | `LICENSE `__ diff --git a/docs/source/index.rst b/docs/source/index.rst index 442b61e..958e7b3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -46,3 +46,4 @@ Table of Contents yara-matches metrics-and-monitoring troubleshooting-faq + credits diff --git a/docs/source/troubleshooting-faq.rst b/docs/source/troubleshooting-faq.rst index e77fa82..bb82979 100644 --- a/docs/source/troubleshooting-faq.rst +++ b/docs/source/troubleshooting-faq.rst @@ -33,6 +33,8 @@ Why did my live test fail? -------------------------- Check the :ref:`Lambda execution logs ` and the :ref:`BinaryAlert dashboard ` for abnormalities. A common problem is that the BinaryAlert analyzers don't understand the compiled YARA rules file. Make sure your `virtual environment `_ is set up correctly and that your YARA rules only use the :ref:`supported modules `. It is also possible that one or more AWS components might be down. +It may take 1-3 minutes after a deploy before the Lambda functions are ready to go. If a live test fails immediately after a deploy, wait a few minutes and try again. + How do I setup YARA match / metric alarm alerts? ------------------------------------------------ diff --git a/lambda_functions/analyzer/README.rst b/lambda_functions/analyzer/README.rst index c29f5fa..0e3a934 100644 --- a/lambda_functions/analyzer/README.rst +++ b/lambda_functions/analyzer/README.rst @@ -72,4 +72,4 @@ and install ``yara-python`` and ``yextend`` as follows: zip -r yara3.7.0_yextend1.6.zip * -Then ``scp`` the newzipfile to replace the one in the repo. +Then ``scp`` the new zipfile to replace the one in the repo. diff --git a/lambda_functions/analyzer/yara_analyzer.py b/lambda_functions/analyzer/yara_analyzer.py index 750ef49..79655a3 100644 --- a/lambda_functions/analyzer/yara_analyzer.py +++ b/lambda_functions/analyzer/yara_analyzer.py @@ -7,6 +7,11 @@ import yara +if __package__: + from lambda_functions.analyzer.common import LOGGER +else: + from common import LOGGER # type: ignore + # YARA matches from both yara-python and yextend are stored in this generic YaraMatch tuple. YaraMatch = collections.namedtuple( @@ -109,9 +114,13 @@ def analyze(self, target_file: str, original_target_path: str = '') -> List[Yara # Yextend matches os.environ['LD_LIBRARY_PATH'] = os.environ['LAMBDA_TASK_ROOT'] - yextend_output = subprocess.check_output( - ['./yextend', '-r', self._compiled_rules_file, '-t', target_file, '-j']) - yextend_list = json.loads(yextend_output.decode('utf-8')) - yextend_matches = _convert_yextend_to_yara_match(yextend_list[0]) + try: + yextend_output = subprocess.check_output( + ['./yextend', '-r', self._compiled_rules_file, '-t', target_file, '-j']) + yextend_list = json.loads(yextend_output.decode('utf-8')) + except (json.JSONDecodeError, subprocess.CalledProcessError): + LOGGER.exception('Fatal error when running yextend') + return yara_python_matches + yextend_matches = _convert_yextend_to_yara_match(yextend_list[0]) return yara_python_matches + yextend_matches diff --git a/manage.py b/manage.py index df3c11c..23b3545 100755 --- a/manage.py +++ b/manage.py @@ -21,7 +21,7 @@ from tests import live_test # BinaryAlert version. -VERSION = '1.1.0.beta' +VERSION = '1.1.0' # File locations. PROJECT_DIR = os.path.dirname(os.path.realpath(__file__)) # Directory containing this file. diff --git a/terraform/lambda_iam.tf b/terraform/lambda_iam.tf index ed3c9ea..9630833 100644 --- a/terraform/lambda_iam.tf +++ b/terraform/lambda_iam.tf @@ -117,9 +117,14 @@ data "aws_iam_policy_document" "binaryalert_analyzer_policy" { } statement { - sid = "GetFromBinaryAlertBucket" - effect = "Allow" - actions = ["s3:GetObject"] + sid = "GetFromBinaryAlertBucket" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:HeadObject", + ] + resources = ["${aws_s3_bucket.binaryalert_binaries.arn}/*"] } diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars index c8b8f63..f7c2ae1 100644 --- a/terraform/terraform.tfvars +++ b/terraform/terraform.tfvars @@ -58,7 +58,7 @@ sqs_retention_minutes = 60 // Number of S3 object keys to pack into a single SQS message. // Each downstream analyzer will process at most 10 SQS messages, each with this many objects. // Higher values allow for higher throughput, but are constrained by analyzer execution time limit. -lambda_batch_objects_per_message = 15 +lambda_batch_objects_per_message = 5 // Memory limit (MB) for the batching Lambda function. 128 is the minimum allowed by Lambda. lambda_batch_memory_mb = 128 @@ -68,7 +68,7 @@ lambda_batch_memory_mb = 128 lambda_dispatch_frequency_minutes = 2 // Maximum number of analyzers that can be asynchronously invoked during one dispatcher run. -// Higher values allow for more throughtput, but if too many analyzers are invoked too quickly, +// Higher values allow for more throughput, but if too many analyzers are invoked too quickly, // Lambda invocations may be throttled. lambda_dispatch_limit = 500