Merge pull request #65 from oasis-open/53-2.1support

Support multiple STIX versions

.gitignore

@@ -109,3 +109,6 @@ venv.bak/
 
 # PyCharm
 .idea/
+
+# Vim
+*.swp

.pre-commit-config.yaml

@@ -2,12 +2,12 @@
     sha: ea227f024bd89d638aea319c92806737e3375979
     hooks:
     -   id: trailing-whitespace
-        exclude: stix2patterns/grammars/*
+        exclude: grammars
     -   id: flake8
+        exclude: grammars
         args:
         - --ignore=F403,F405
         - --max-line-length=160
-        - --exclude=stix2patterns/grammars/*
     -   id: check-merge-conflict
 -   repo: https://github.com/FalconSocial/pre-commit-python-sorter
     sha: b57843b0b874df1d16eb0bef00b868792cb245c2

MANIFEST.in

@@ -5,7 +5,8 @@ include CONTRIBUTING.md
 include dev-requirements.txt
 include LICENSE
 include tox.ini
-include stix2patterns/test/spec_examples.txt
+include stix2patterns/test/v20/spec_examples.txt
+include stix2patterns/test/v21/spec_examples.txt
 
 recursive-include docs *
 prune docs/_build

docs/developers.rst

@@ -62,11 +62,11 @@ source files. (The .jar file is not needed for normal use of the validator).
 1. Download antlr-4.7.1-complete.jar from http://www.antlr.org/download/
 2. Clone the stix2-json-schemas repository or download the STIXPattern.g4 file.
 3. Change to the directory containing the STIXPattern.g4 file.
-4. Run the following command
+4. Run the following command (for STIX v2.1)
 
    .. prompt:: bash
 
-       java -jar "/path/to/antlr-4.7.1-complete.jar" -Dlanguage=Python2 STIXPattern.g4 -visitor -o /path/to/cti-pattern-validator/stix2patterns/grammars
+       java -jar "/path/to/antlr-4.7.1-complete.jar" -Dlanguage=Python2 STIXPattern.g4 -visitor -o /path/to/cti-pattern-validator/stix2patterns/v21/grammars
 
 5. Commit the resulting files to git.
 
@@ -97,13 +97,14 @@ You can run a specific test file by passing it on the command line:
 
 .. prompt:: bash
 
-    pytest stix2patterns/test/test_<xxx>.py
+    pytest stix2patterns/test/v21/test_<xxx>.py
 
 You can also test against the examples provided in the supplied example file.
+Note that you must specify which version to test.
 
 .. prompt:: bash
 
-    validate-patterns -f stix2patterns/test/spec_examples.txt
+    validate-patterns -v 2.1 -f stix2patterns/test/v21/spec_examples.txt
 
 To ensure that the test you wrote is running, you can deliberately add an
 ``assert False`` statement at the beginning of the test. This is another benefit

setup.py

@@ -20,7 +20,6 @@
     'bumpversion',
     'check-manifest',
     'pre-commit',
-    'readme_renderer',
     # test_requires are installed into every tox environemnt, so we don't
     # want to include tox there.
     'tox',
@@ -31,6 +30,7 @@
     version='1.1.0',
     description='Validate STIX 2 Patterns.',
     long_description=readme,
+    long_description_content_type='text/x-rst',
     url="https://github.com/oasis-open/cti-pattern-validator",
     author='OASIS Cyber Threat Intelligence Technical Committee',
     author_email='[email protected]',
@@ -44,7 +44,8 @@
         'typing ; python_version<"3.5" and python_version>="3"',
     ],
     package_data={
-        'stix2patterns.test': ['spec_examples.txt'],
+        'stix2patterns.test.v20': ['spec_examples.txt'],
+        'stix2patterns.test.v21': ['spec_examples.txt'],
     },
     entry_points={
         'console_scripts': [

stix2patterns/__init__.py

@@ -0,0 +1 @@
+DEFAULT_VERSION = '2.0'  # Default version should always be the latest STIX 2.X version

stix2patterns/exceptions.py

@@ -0,0 +1,28 @@
+from antlr4.error.ErrorListener import ErrorListener
+
+
+class STIXPatternErrorListener(ErrorListener):
+    """
+    Modifies ErrorListener to collect error message and set flag to False when
+    invalid pattern is encountered.
+    """
+    def __init__(self):
+        super(STIXPatternErrorListener, self).__init__()
+        self.err_strings = []
+
+    def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e):
+        self.err_strings.append("FAIL: Error found at line %d:%d. %s" %
+                                (line, column, msg))
+
+
+class ParserErrorListener(ErrorListener):
+    """
+    Simple error listener which just remembers the last error message received.
+    """
+    def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e):
+        self.error_message = u"{}:{}: {}".format(line, column, msg)
+
+
+class ParseException(Exception):
+    """Represents a parse error."""
+    pass

stix2patterns/inspector.py

@@ -1,7 +1,5 @@
 import collections
 
-from stix2patterns.grammars.STIXPatternListener import STIXPatternListener
-
 
 class InspectionException(Exception):
     """Represents a error that occurred during inspection."""
@@ -26,150 +24,3 @@ def _string_literal_to_string(string_literal_token):
     token_text = string_literal_token.getText()
     return token_text[1:-1].replace(u"\\'", u"'"). \
         replace(u"\\\\", u"\\")
-
-
-class InspectionListener(STIXPatternListener):
-    """This listener collects info about a pattern and puts it
-    in a python structure.  It is intended to assist apps which wish to
-    look "inside" a pattern and know what's in there.
-    """
-
-    def __init__(self):
-        self.__comparison_data = {}
-        self.__qualifiers = set()
-        self.__observation_ops = set()
-        self.__obj_type = None
-        self.__obj_path = None
-
-    def pattern_data(self):
-        return _PatternData(self.__comparison_data, self.__observation_ops,
-                            self.__qualifiers)
-
-    def __add_prop_tuple(self, obj_type, obj_path, op, value):
-        if obj_type not in self.__comparison_data:
-            self.__comparison_data[obj_type] = []
-
-        self.__comparison_data[obj_type].append((obj_path, op, value))
-
-    def exitObservationExpressions(self, ctx):
-        if ctx.FOLLOWEDBY():
-            self.__observation_ops.add(u"FOLLOWEDBY")
-
-    def exitObservationExpressionOr(self, ctx):
-        if ctx.OR():
-            self.__observation_ops.add(u"OR")
-
-    def exitObservationExpressionAnd(self, ctx):
-        if ctx.AND():
-            self.__observation_ops.add(u"AND")
-
-    def exitStartStopQualifier(self, ctx):
-        self.__qualifiers.add(
-            u"START {0} STOP {1}".format(
-                ctx.StringLiteral(0), ctx.StringLiteral(1)
-            )
-        )
-
-    def exitWithinQualifier(self, ctx):
-        self.__qualifiers.add(
-            u"WITHIN {0} SECONDS".format(
-                ctx.IntPosLiteral() or ctx.FloatPosLiteral()
-            )
-        )
-
-    def exitRepeatedQualifier(self, ctx):
-        self.__qualifiers.add(
-            u"REPEATS {0} TIMES".format(
-                ctx.IntPosLiteral()
-            )
-        )
-
-    def exitPropTestEqual(self, ctx):
-        op_tok = ctx.EQ() or ctx.NEQ()
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += op_tok.getText()
-
-        value = ctx.primitiveLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestOrder(self, ctx):
-        op_tok = ctx.GT() or ctx.LT() or ctx.GE() or ctx.LE()
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += op_tok.getText()
-
-        value = ctx.orderableLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestSet(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"IN"
-
-        value = ctx.setLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestLike(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"LIKE"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestRegex(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"MATCHES"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestIsSubset(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"ISSUBSET"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitPropTestIsSuperset(self, ctx):
-        op_str = u"NOT " if ctx.NOT() else u""
-        op_str += u"ISSUPERSET"
-
-        value = ctx.StringLiteral().getText()
-
-        self.__add_prop_tuple(self.__obj_type, self.__obj_path, op_str,
-                              value)
-
-    def exitObjectType(self, ctx):
-        self.__obj_type = ctx.getText()
-
-    def exitFirstPathComponent(self, ctx):
-        if ctx.StringLiteral():
-            path_component = _string_literal_to_string(ctx.StringLiteral())
-        else:
-            path_component = ctx.getText()
-
-        self.__obj_path = [path_component]
-
-    def exitKeyPathStep(self, ctx):
-        if ctx.IdentifierWithoutHyphen():
-            path_component = ctx.IdentifierWithoutHyphen().getText()
-        else:  # A StringLiteral
-            path_component = _string_literal_to_string(ctx.StringLiteral())
-
-        self.__obj_path.append(path_component)
-
-    def exitIndexPathStep(self, ctx):
-        if ctx.ASTERISK():
-            self.__obj_path.append(INDEX_STAR)
-        else:
-            self.__obj_path.append(int(ctx.IntPosLiteral().getText()))

stix2patterns/test/test_inspector.py → stix2patterns/test/v20/test_inspector.py

@@ -1,7 +1,7 @@
 import pytest
 
 from stix2patterns.inspector import INDEX_STAR
-from stix2patterns.pattern import Pattern
+from stix2patterns.v20.pattern import Pattern
 
 
 @pytest.mark.parametrize(u"pattern,expected_qualifiers", [

stix2patterns/test/test_validator.py → stix2patterns/test/v20/test_validator.py

@@ -17,7 +17,7 @@ def test_spec_patterns(test_input):
     """
     Validate patterns from STIX 2.0 Patterning spec.
     """
-    pass_test = validate(test_input, print_errs=True)
+    pass_test = validate(test_input, stix_version='2.0', print_errs=True)
     assert pass_test is True
 
 
@@ -59,7 +59,7 @@ def test_fail_patterns(test_input, test_output):
     """
     Validate that patterns fail as expected.
     """
-    pass_test, errors = validate(test_input, ret_errs=True, print_errs=True)
+    pass_test, errors = validate(test_input, stix_version='2.0', ret_errs=True, print_errs=True)
     assert errors[0].startswith(test_output)
     assert pass_test is False
 
@@ -95,5 +95,5 @@ def test_pass_patterns(test_input):
     """
     Validate that patterns pass as expected.
     """
-    pass_test = validate(test_input, print_errs=True)
+    pass_test = validate(test_input, stix_version='2.0', print_errs=True)
     assert pass_test is True

stix2patterns/test/v21/spec_examples.txt

@@ -0,0 +1,23 @@
+[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']
+[email-message:from_ref.value MATCHES '.+\\@example\\.com$' AND email-message:body_multipart[*].body_raw_ref.name MATCHES '^Final Report.+\\.exe$']
+[file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f' AND file:mime_type = 'application/x-pdf']
+[file:hashes.'SHA-256' = 'bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c' OR file:hashes.MD5 = 'cead3f77f6cda6ec00f57d76c9a6879f'] AND [file:hashes.'SHA-256' = 'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f']
+([file:hashes.MD5 = '79054025255fb1a26e4bc422aef54eb4'] FOLLOWEDBY [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\foo\\bar']) WITHIN 300 SECONDS
+[user-account:account_type = 'unix' AND user-account:user_id = '1007' AND user-account:account_login = 'Peter'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1008' AND user-account:account_login = 'Paul'] AND [user-account:account_type = 'unix' AND user-account:user_id = '1009' AND user-account:account_login = 'Mary']
+[artifact:mime_type = 'application/vnd.tcpdump.pcap' AND artifact:payload_bin MATCHES '\\xd4\\xc3\\xb2\\xa1\\x02\\x00\\x04\\x00']
+[file:name = 'foo.dll' AND file:parent_directory_ref.path = 'C:\\Windows\\System32']
+[file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.0]
+[file:mime_type = 'image/bmp' AND file:magic_number_hex = h'ffd8']
+[network-traffic:dst_ref.type = 'ipv4-addr' AND network-traffic:dst_ref.value = '203.0.113.33/32']
+[network-traffic:dst_ref.type = 'domain-name' AND network-traffic:dst_ref.value = 'example.com'] REPEATS 5 TIMES WITHIN 1800 SECONDS
+[domain-name:value = 'www.5z8.info' AND domain-name:resolves_to_refs[*].value = '198.51.100.1/32']
+[url:value = 'http://example.com/foo' OR url:value = 'http://example.com/bar']
+[x509-certificate:issuer = 'CN=WEBMAIL' AND x509-certificate:serial_number = '4c:0b:1d:19:74:86:a7:66:b4:1a:bf:40:27:21:76:28']
+[windows-registry-key:key = 'HKEY_CURRENT_USER\\Software\\CryptoLocker\\Files' OR windows-registry-key:key = 'HKEY_CURRENT_USER\\Software\\Microsoft\\CurrentVersion\\Run\\CryptoLocker_0388']
+[(file:name = 'pdf.exe' OR file:size = '371712') AND file:created = t'2014-01-13T07:03:17Z']
+[email-message:sender_ref.value = '[email protected]' AND email-message:subject = 'Conference Info']
+[x-usb-device:usbdrive.serial_number = '575833314133343231313937']
+[process:command_line MATCHES '^.+>-add GlobalSign.cer -c -s -r localMachine Root$'] FOLLOWEDBY [process:command_line MATCHES'^.+>-add GlobalSign.cer -c -s -r localMachineTrustedPublisher$'] WITHIN 300 SECONDS
+[network-traffic:dst_ref.value ISSUBSET '2001:0db8:dead:beef:0000:0000:0000:0000/64']
+([file:name = 'foo.dll'] AND [win-registry-key:key = 'HKEY_LOCAL_MACHINE\\foo\\bar']) OR [process:name = 'fooproc' OR process:name = 'procfoo']
+[file:hashes.MD5 = 'cead3f77f6cda6ec00f57d76c9a69faa']