diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml new file mode 100644 index 0000000..9d95926 --- /dev/null +++ b/.github/workflows/python-ci-tests.yml @@ -0,0 +1,29 @@ +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: stix-validator test harness +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, '3.10', '3.11', '3.12'] + + name: Python ${{ matrix.python-version }} Build + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install and update essential dependencies + run: | + pip install -U pip setuptools + pip install tox-gh-actions + - name: Test with Tox + run: | + tox diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0a109af..0000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -os: linux -language: python -cache: pip -dist: xenial -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" -jobs: - include: - # https://blog.travis-ci.com/2019-04-15-xenial-default-build-environment - # https://travis-ci.community/t/issue-with-python-2-6-on-linux/3861/2 - - python: 2.6 - dist: trusty -before_install: - - sudo apt-get install -qq libxml2 -install: - - pip install -U tox-travis -script: tox -branches: - only: - - master -notifications: - email: - - stix-commits-list@groups.mitre.org diff --git a/sdv/scripts/__init__.py b/sdv/scripts/__init__.py index 29e94dc..6a8be0e 100755 --- a/sdv/scripts/__init__.py +++ b/sdv/scripts/__init__.py @@ -8,7 +8,6 @@ # external import lxml.etree -from mixbox.vendor.six import iteritems, itervalues # internal import sdv @@ -290,7 +289,7 @@ def print_profile_results(results, level): for e in results.errors: errors_[e.message].append(e.line) - for msg, lines in iteritems(errors_): + for msg, lines in errors_.items(): lines = ', '.join(str(x) for x in lines) print_level("[!] %s [%s]", level+1, msg, lines) @@ -303,7 +302,7 @@ def print_json_results(results): results to print. """ json_results = {} - for fn, result in iteritems(results): + for fn, result in results.items(): d = {} if result.schema_results is not None: d['schema validation'] = result.schema_results.as_dict() @@ -335,7 +334,7 @@ def print_results(results, options): return level = 0 - for fn, result in sorted(iteritems(results)): + for fn, result in sorted(results.items()): print("=" * 80) print_level("[-] Results: %s", level, fn) @@ -511,7 +510,7 @@ def status_code(results): """ status = codes.EXIT_SUCCESS - for result in itervalues(results): + for result in results.values(): schema = result.schema_results best_practice = result.best_practice_results profile = result.profile_results diff --git a/sdv/test/cybox_schema_tests.py b/sdv/test/cybox_schema_test.py similarity index 98% rename from sdv/test/cybox_schema_tests.py rename to sdv/test/cybox_schema_test.py index 811eea8..08baebd 100644 --- a/sdv/test/cybox_schema_tests.py +++ b/sdv/test/cybox_schema_test.py @@ -1,8 +1,8 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. +from io import StringIO import unittest -from mixbox.vendor.six import StringIO import sdv import sdv.errors as errors diff --git a/sdv/test/stix_best_practices_tests.py b/sdv/test/stix_best_practices_test.py similarity index 99% rename from sdv/test/stix_best_practices_tests.py rename to sdv/test/stix_best_practices_test.py index d4d241e..de43b84 100644 --- a/sdv/test/stix_best_practices_tests.py +++ b/sdv/test/stix_best_practices_test.py @@ -1,8 +1,9 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. + +from io import StringIO import json import unittest -from mixbox.vendor.six import StringIO from lxml import etree diff --git a/sdv/test/stix_profile_tests.py b/sdv/test/stix_profile_test.py similarity index 97% rename from sdv/test/stix_profile_tests.py rename to sdv/test/stix_profile_test.py index 8c58372..fcc48da 100644 --- a/sdv/test/stix_profile_tests.py +++ b/sdv/test/stix_profile_test.py @@ -1,8 +1,8 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. +from io import StringIO import unittest -from mixbox.vendor.six import StringIO import sdv import sdv.errors as errors diff --git a/sdv/test/stix_schema_tests.py b/sdv/test/stix_schema_test.py similarity index 99% rename from sdv/test/stix_schema_tests.py rename to sdv/test/stix_schema_test.py index ca094c4..71cacd5 100644 --- a/sdv/test/stix_schema_tests.py +++ b/sdv/test/stix_schema_test.py @@ -1,8 +1,8 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. +from io import StringIO import unittest -from mixbox.vendor.six import StringIO import sdv import sdv.errors as errors diff --git a/sdv/test/utils_tests.py b/sdv/test/utils_test.py similarity index 97% rename from sdv/test/utils_tests.py rename to sdv/test/utils_test.py index 12dba9d..f39c548 100644 --- a/sdv/test/utils_tests.py +++ b/sdv/test/utils_test.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, The MITRE Corporation. All rights reserved. # See LICENSE.txt for complete terms. +from io import StringIO import unittest import datetime -from mixbox.vendor.six import StringIO from lxml import etree import dateutil.parser @@ -146,4 +146,4 @@ def test_is_qname(self): self.assertTrue(utils.is_qname(s)) for s in invalid: - self.assertEqual(False, utils.is_qname(s), msg=s) \ No newline at end of file + self.assertEqual(False, utils.is_qname(s), msg=s) diff --git a/sdv/utils.py b/sdv/utils.py index f2832d5..dbe2d8f 100644 --- a/sdv/utils.py +++ b/sdv/utils.py @@ -6,11 +6,11 @@ import contextlib import datetime from distutils.version import StrictVersion +from io import StringIO, BytesIO # external import dateutil.parser from lxml import etree -from mixbox.vendor.six import StringIO, BytesIO # relative from . import errors, xmlconst @@ -500,4 +500,4 @@ def remove_version_prefix(version): """ if version.startswith('stix-'): version = version.partition('stix-')[2] - return version \ No newline at end of file + return version diff --git a/sdv/validators/base.py b/sdv/validators/base.py index bf23924..3c6bbea 100644 --- a/sdv/validators/base.py +++ b/sdv/validators/base.py @@ -5,9 +5,6 @@ import abc import json -# external -from mixbox.vendor.six import iteritems - # internal from .. import utils @@ -93,7 +90,7 @@ def _get_validators(self, schema_dir=None): self._KEY_USER_DEFINED: self._get_validator_impl(schema_dir) } else: - for version, location in iteritems(self._SCHEMAS): + for version, location in self._SCHEMAS.items(): validator = self._get_validator_impl(location) validators[version] = validator @@ -148,4 +145,4 @@ def _validate(self, doc, version=None, schemaloc=False): 'ValidationError', 'ValidationResults', 'BaseSchemaValidator' -] \ No newline at end of file +] diff --git a/sdv/validators/stix/best_practice.py b/sdv/validators/stix/best_practice.py index 5506bb4..3a17a1c 100644 --- a/sdv/validators/stix/best_practice.py +++ b/sdv/validators/stix/best_practice.py @@ -5,12 +5,10 @@ import re import itertools import collections -import distutils.version +from packaging.version import parse as parse_version # external from lxml import etree -from mixbox.vendor.six import iteritems, itervalues, with_metaclass -from mixbox import compat # internal from sdv import utils, xmlconst @@ -21,13 +19,6 @@ from ...utils import remove_version_prefix -# Python 2.6 doesn't have collections.OrderedDict :( -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict - - # STIX ID Format: [ns prefix]:[construct type]-[GUID] # Note: This will validate invalid QNames, so this should be used with a # QName format check. @@ -62,7 +53,7 @@ def __new__(metacls, name, bases, dict_): ruledict = collections.defaultdict(list) # Find all @rule marked functions in the class dict_ - rulefuncs = (x for x in itervalues(dict_) if hasattr(x, 'is_rule')) + rulefuncs = (x for x in dict_.values() if hasattr(x, 'is_rule')) # Build the rule function dict. for rule in rulefuncs: @@ -74,7 +65,7 @@ def __new__(metacls, name, bases, dict_): return obj -class BestPracticeWarning(compat.MutableMapping, base.ValidationError): +class BestPracticeWarning(collections.abc.MutableMapping, base.ValidationError): """Represents a best practice warning. These are built within best practice rule checking methods and attached to :class:`BestPracticeWarningCollection` instances. @@ -102,7 +93,7 @@ class BestPracticeWarning(compat.MutableMapping, base.ValidationError): def __init__(self, node, message=None): base.ValidationError.__init__(self) - self._inner = OrderedDict() + self._inner = collections.OrderedDict() self._node = node self['line'] = node.sourceline @@ -185,10 +176,10 @@ def as_dict(self): necessary. """ - return dict(iteritems(self)) + return dict(self.items()) -class BestPracticeWarningCollection(compat.MutableSequence): +class BestPracticeWarningCollection(collections.abc.MutableSequence): """A collection of :class:`BestPracticeWarning` instances for a given type of STIX Best Practice. @@ -257,7 +248,7 @@ def as_dict(self): return {self.name: [x.as_dict() for x in self]} -class BestPracticeValidationResults(base.ValidationResults, compat.MutableSequence): +class BestPracticeValidationResults(base.ValidationResults, collections.abc.MutableSequence): """Represents STIX best practice validation results. This class behaves like a ``list`` and accepts instances of :class:`BestPracticeWarningCollection`. @@ -338,7 +329,7 @@ def as_dict(self): return d -class STIXBestPracticeValidator(with_metaclass(BestPracticeMeta, object)): +class STIXBestPracticeValidator(object, metaclass=BestPracticeMeta): """Performs STIX Best Practice validation.""" @rule('1.0') @@ -449,7 +440,7 @@ def _check_1_2_duplicate_ids(self, root, namespaces, version): # noqa idnodes[node.attrib.get('id')].append(node) # Find all nodes that have duplicate IDs - dups = [x for x in itervalues(idnodes) if len(x) > 1] + dups = [x for x in idnodes.values() if len(x) > 1] # Build warnings for all nodes that have conflicting id/timestamp pairs. for nodeset in dups: @@ -469,7 +460,7 @@ def _check_1_0_duplicate_ids(self, root, namespaces, version): # noqa id_nodes[node.attrib['id']].append(node) results = BestPracticeWarningCollection('Duplicate IDs') - for nodes in itervalues(id_nodes): + for nodes in id_nodes.values(): if len(nodes) > 1: results.extend(BestPracticeWarning(node=x) for x in nodes) @@ -611,7 +602,7 @@ def _is_expected(node, expected): return True return node.attrib['version'] == expected - for selector, expected in iteritems(to_check): + for selector, expected in to_check.items(): xpath = "//%s" % selector for node in root.xpath(xpath, namespaces=namespaces): @@ -1124,7 +1115,7 @@ def can_inspect(node): warns = [] seen = set() - + for node in filtered: o = node.attrib.get('ordinality') @@ -1192,17 +1183,17 @@ def can_run(stix_version, rule_min, rule_max): if not rule_min: return True - doc_ver = StrictVersion(remove_version_prefix(stix_version)) - min_ver = StrictVersion(remove_version_prefix(rule_min)) + doc_ver = parse_version(remove_version_prefix(stix_version)) + min_ver = parse_version(remove_version_prefix(rule_min)) if rule_max: - max_ver = StrictVersion(remove_version_prefix(rule_max)) + max_ver = parse_version(remove_version_prefix(rule_max)) return (min_ver <= doc_ver <= max_ver) return min_ver <= doc_ver - StrictVersion = distutils.version.StrictVersion - all_rules = iteritems(self._rules) # noqa + #StrictVersion = distutils.version.StrictVersion + all_rules = self._rules.items() # noqa # Get a generator which yields all best practice methods that are # assigned a version number <= the input STIX document version number. diff --git a/sdv/validators/stix/common.py b/sdv/validators/stix/common.py index 2295e54..bf7a380 100644 --- a/sdv/validators/stix/common.py +++ b/sdv/validators/stix/common.py @@ -4,7 +4,8 @@ # builtin import re import functools -from distutils.version import StrictVersion +# from distutils.version import StrictVersion +from packaging.version import parse as parse_version # external from lxml import etree @@ -284,7 +285,7 @@ def is_idref_content_exception(node): def _get_cybox_vocab_version(name, version): versions = CYBOX_VOCAB_VERSIONS - descending = sorted(versions, key=StrictVersion, reverse=True) + descending = sorted(versions, key=parse_version, reverse=True) idx = descending.index for key in descending[idx(version):]: @@ -301,7 +302,7 @@ def _get_cybox_vocab_version(name, version): def _get_stix_vocab_version(name, version): versions = STIX_VOCAB_VERSIONS descending = sorted(versions, key=lambda v: - StrictVersion(utils.remove_version_prefix(v)), reverse=True) + parse_version(utils.remove_version_prefix(v)), reverse=True) idx = descending.index for key in descending[idx(version):]: @@ -451,7 +452,7 @@ def get_stix_namespaces(version): found=version ) - if StrictVersion(utils.remove_version_prefix(version)) < '1.2.1': + if parse_version(utils.remove_version_prefix(version)) < parse_version('1.2.1'): nsmap = { PREFIX_XSI: xmlconst.NS_XSI, PREFIX_STIX_CORE: 'http://stix.mitre.org/stix-1', diff --git a/sdv/validators/stix/profile.py b/sdv/validators/stix/profile.py index fdf379e..80dd760 100644 --- a/sdv/validators/stix/profile.py +++ b/sdv/validators/stix/profile.py @@ -6,9 +6,7 @@ import itertools import collections import functools -from mixbox.vendor.six import StringIO, string_types, iteritems -from mixbox.vendor.six.moves import range -from mixbox import compat +from io import StringIO # external import xlrd @@ -103,7 +101,7 @@ def selectors(self, value): """ if not value: self._selectors = [] - elif isinstance(value, string_types): + elif isinstance(value, str): self._selectors = [x.strip().replace('"', "'") for x in value.split(",")] elif hasattr(value, "__iter__"): self._selectors = [str(x) for x in value] @@ -159,7 +157,7 @@ def validate(self): raise errors.ProfileParseError(err.format(label=self.label)) -class Profile(compat.MutableSequence): +class Profile(collections.abc.MutableSequence): def __init__(self, namespaces): self.id = "STIX_Schematron_Profile" self._rules = [RootRule(namespaces)] @@ -230,7 +228,7 @@ def rules(self): rules = [notype, typed] collected = self._collect_rules() - for ctx, profile_rules in iteritems(collected): + for ctx, profile_rules in collected.items(): rule = schematron.make_rule(ctx) rule.extend(x.as_etree() for x in profile_rules) @@ -249,7 +247,7 @@ def namespaces(self): """ namespaces = [] - for ns, prefix in iteritems(self._namespaces): + for ns, prefix in self._namespaces.items(): ns = schematron.make_ns(prefix, ns) namespaces.append(ns) @@ -456,7 +454,7 @@ def values(self, value): """ if not value: self._values = [] - elif isinstance(value, string_types): + elif isinstance(value, str): self._values = [x.strip() for x in value.split(',')] elif hasattr(value, "__getitem__"): self._values = [str(x) for x in value] @@ -522,7 +520,7 @@ def impls(self, value): """ if not value: self._impls = [] - elif isinstance(value, string_types): + elif isinstance(value, str): self._impls = [x.strip() for x in value.split(',')] elif hasattr(value, "__iter__"): self._impls = [str(x) for x in value] diff --git a/sdv/validators/xml_schema.py b/sdv/validators/xml_schema.py index 0a1ed26..691cf36 100644 --- a/sdv/validators/xml_schema.py +++ b/sdv/validators/xml_schema.py @@ -7,7 +7,6 @@ # external from lxml import etree -from mixbox.vendor.six import iteritems, itervalues, python_2_unicode_compatible # internal from sdv import errors, utils, xmlconst @@ -16,7 +15,7 @@ from . import base -@python_2_unicode_compatible +#@python_2_unicode_compatible class XmlSchemaError(base.ValidationError): """Represents an XML Schema validation error. @@ -199,7 +198,7 @@ def _is_included(self, graph, fp): schemas in `graph`. """ - return any(fp in includes for includes in itervalues(graph)) + return any(fp in includes for includes in graph.values()) def _get_include_root(self, ns, list_schemas): """Attempts to determine the "root" schema for a targetNamespace. @@ -225,7 +224,7 @@ def _get_include_root(self, ns, list_schemas): """ graph = self._build_include_graph(list_schemas) - if all(not(x) for x in itervalues(graph)): + if all(not(x) for x in graph.values()): return list_schemas[0] for fp in graph: @@ -267,7 +266,7 @@ def _process_includes(self, imports): """ processed = {} - for ns, schemas in iteritems(imports): + for ns, schemas in imports.items(): if len(schemas) > 1: base_schema = self._get_include_root(ns, schemas) processed[ns] = base_schema @@ -312,7 +311,7 @@ def _walk_schemas(self, schema_dir): schemalocs[target_ns].append(fp) seen.append((target_ns, fn)) - for ns, loc in iteritems(self.OVERRIDE_SCHEMALOC): + for ns, loc in self.OVERRIDE_SCHEMALOC.items(): schemalocs[ns] = [loc] return schemalocs @@ -374,7 +373,7 @@ def _get_required_schemas(self, root): def _get_schemalocs(node): schemalocs = {} - for ns in itervalues(node.nsmap): + for ns in node.nsmap.values(): if ns not in self._schemalocs: continue @@ -432,7 +431,7 @@ def _build_uber_schema(self, doc, schemaloc=False): """ ) - for ns, loc in iteritems(imports): + for ns, loc in imports.items(): loc = loc.replace("\\", "/") attrib = {'namespace': ns, 'schemaLocation':loc} import_ = etree.Element(xmlconst.TAG_XS_IMPORT, attrib=attrib) diff --git a/setup.py b/setup.py index fd860f9..6b927e4 100644 --- a/setup.py +++ b/setup.py @@ -21,11 +21,6 @@ def get_version(): raise AttributeError("Package does not have a __version__") -py_maj, py_minor = sys.version_info[:2] - -if (py_maj, py_minor) < (2, 6) or (py_maj == 3 and py_minor < 3): - raise Exception('stix-validator requires Python 2.6, 2.7 or 3.3+') - fn_readme = join(BASE_DIR, "README.rst") with open(fn_readme) as f: readme = f.read() @@ -33,25 +28,12 @@ def get_version(): install_requires = [ 'xlrd>=0.9.2', 'ordereddict', - 'mixbox>=1.0.5', - 'python-dateutil' + 'python-dateutil', + 'packaging', + 'lxml>=3.3.5', + 'setuptools', ] -# lxml has dropped support for Python 2.6, 3.3 after version 4.2.6 -if (py_maj, py_minor) == (2, 6) or (py_maj, py_minor) == (3, 3): - install_requires.append('lxml>=3.3.5,<4.3.0') -# lxml has dropped support for Python 2.6, 3.3, 3.4 after version 4.4.0 -elif (py_maj, py_minor) == (2, 6) or (py_maj, py_minor) == (3, 4): - install_requires.append('lxml>=3.3.5,<4.4.0') -else: - install_requires.append('lxml>=3.3.5') - - -# Python 2.6 does not come with argparse -try: - import argparse -except ImportError: - install_requires.append('argparse') extras_require = { 'docs': [ @@ -60,8 +42,8 @@ def get_version(): ], 'test': [ "bumpversion", - "nose==1.3.7", - "tox==2.3.1" + "pytest", + "tox" ], } @@ -91,15 +73,10 @@ def get_version(): 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ] ) diff --git a/tox.ini b/tox.ini index 490c5a6..136c1b3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,27 +1,24 @@ [tox] -envlist = py26, py27, py34, py35, py36, py37, py38 +envlist = py38, py39, py310, py311, py312 [testenv] commands = - nosetests sdv + pytest sdv/test stix-validator examples/stix/all_valid.xml stix-validator examples/stix/all_valid.xml --json - stix-validator --profile examples/stix/Example_STIX_Profile.xlsx examples/stix/all_valid.xml - stix-validator --profile examples/stix/Example_STIX_Profile.xlsx examples/stix/all_valid.xml --json + #stix-validator --profile examples/stix/Example_STIX_Profile.xlsx examples/stix/all_valid.xml + #stix-validator --profile examples/stix/Example_STIX_Profile.xlsx examples/stix/all_valid.xml --json stix-validator --best-practices examples/stix/all_valid.xml stix-validator --best-practices examples/stix/all_valid.xml --json cybox-validator examples/cybox/schema_valid.xml cybox-validator examples/cybox/schema_valid.xml --json - deps = -rrequirements.txt [travis] python = - 2.6: py26 - 2.7: py27 - 3.4: py34 - 3.5: py35 - 3.6: py36 - 3.7: py37 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312