diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index b7ae0155a..039bdd963 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -29,14 +29,16 @@ jobs: toxenv: py310,style,coverage-ci - python-version: 3.11 toxenv: py311,style,coverage-ci + - python-version: 3.12 + toxenv: py312,style,coverage-ci steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: submodules: recursive fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Setup python - uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -50,7 +52,7 @@ jobs: - name: Override Coverage Source Path for Sonar run: sed -i "s/\/home\/runner\/work\/caldera\/caldera/\/github\/workspace/g" /home/runner/work/caldera/caldera/coverage.xml - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@5875562561d22a34be0c657405578705a169af6c + uses: SonarSource/sonarcloud-github-action@49e6cd3b187936a73b8280d59ffd9da69df63ec9 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index afabcb7e5..6e1a9d344 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.0.0 hooks: - id: flake8 additional_dependencies: [flake8-bugbear] - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 + rev: 1.7.7 hooks: - id: bandit entry: bandit -ll --exclude=tests/ --skip=B303 diff --git a/app/api/packs/advanced.py b/app/api/packs/advanced.py index 70e0c195c..9c83a1e91 100644 --- a/app/api/packs/advanced.py +++ b/app/api/packs/advanced.py @@ -14,6 +14,7 @@ def __init__(self, services): self.rest_svc = services.get('rest_svc') async def enable(self): + self.app_svc.application.router._frozen = False self.app_svc.application.router.add_route('GET', '/advanced/sources', self._section_sources) self.app_svc.application.router.add_route('GET', '/advanced/objectives', self._section_objectives) self.app_svc.application.router.add_route('GET', '/advanced/planners', self._section_planners) diff --git a/app/api/packs/campaign.py b/app/api/packs/campaign.py index d6f37e9df..5fb5601c4 100644 --- a/app/api/packs/campaign.py +++ b/app/api/packs/campaign.py @@ -16,6 +16,7 @@ def __init__(self, services): self.rest_svc = services.get('rest_svc') async def enable(self): + self.app_svc.application.router._frozen = False self.app_svc.application.router.add_route('GET', '/campaign/agents', self._section_agent) self.app_svc.application.router.add_route('GET', '/campaign/abilities', self._section_abilities) self.app_svc.application.router.add_route('GET', '/campaign/adversaries', self._section_profiles) diff --git a/app/api/v2/managers/schedule_api_manager.py b/app/api/v2/managers/schedule_api_manager.py index 4d2608ec4..d7a581249 100644 --- a/app/api/v2/managers/schedule_api_manager.py +++ b/app/api/v2/managers/schedule_api_manager.py @@ -30,6 +30,6 @@ def _merge_dictionaries(self, dict1, dict2): for key in dict1.keys(): if key not in dict2: dict2[key] = dict1[key] - elif type(dict1[key]) == dict: + elif isinstance(dict1[key], dict): self._merge_dictionaries(dict1[key], dict2[key]) return dict2 diff --git a/app/contacts/contact_dns.py b/app/contacts/contact_dns.py index aa49d04fa..d1b66369c 100644 --- a/app/contacts/contact_dns.py +++ b/app/contacts/contact_dns.py @@ -18,12 +18,17 @@ def __init__(self, services): self.contact_svc = services.get('contact_svc') self.domain = self.get_config('app.contact.dns.domain') self.handler = Handler(self.domain, services, self.name) + self.transport = None async def start(self): loop = asyncio.get_event_loop() dns = self.get_config('app.contact.dns.socket') addr, port = dns.split(':') - await loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port)) + self.transport, _ = await loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port)) + + async def stop(self): + if self.transport: + self.transport.close() class DnsPacket: diff --git a/app/contacts/contact_udp.py b/app/contacts/contact_udp.py index 3b3fa0fc4..73d10766a 100644 --- a/app/contacts/contact_udp.py +++ b/app/contacts/contact_udp.py @@ -13,12 +13,17 @@ def __init__(self, services): self.log = self.create_logger('contact_udp') self.contact_svc = services.get('contact_svc') self.handler = Handler(services) + self.transport = None async def start(self): loop = asyncio.get_event_loop() udp = self.get_config('app.contact.udp') addr, port = udp.split(':') - loop.create_task(loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port))) + self.transport, _ = await loop.create_task(loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port))) + + async def stop(self): + if self.transport: + self.transport.close() class Handler(asyncio.DatagramProtocol): diff --git a/app/learning/p_ip.py b/app/learning/p_ip.py index f80b21584..1a07baee3 100644 --- a/app/learning/p_ip.py +++ b/app/learning/p_ip.py @@ -22,6 +22,6 @@ def _is_valid_ip(raw_ip): if raw_ip in ['0.0.0.0', '127.0.0.1']: # nosec return False ip_address(raw_ip) - except BaseException: + except ValueError: return False return True diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index a26da0dad..188c133c0 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -12,6 +12,10 @@ class AbilitySchema(ma.Schema): + + class Meta: + unknown = ma.EXCLUDE + ability_id = ma.fields.String() tactic = ma.fields.String(load_default=None) technique_name = ma.fields.String(load_default=None) diff --git a/app/objects/c_adversary.py b/app/objects/c_adversary.py index e2c4ccd6f..bcc85fd42 100644 --- a/app/objects/c_adversary.py +++ b/app/objects/c_adversary.py @@ -11,6 +11,9 @@ class AdversarySchema(ma.Schema): + class Meta: + unknown = ma.EXCLUDE + adversary_id = ma.fields.String() name = ma.fields.String() description = ma.fields.String() diff --git a/app/objects/c_objective.py b/app/objects/c_objective.py index 7b87a5b47..e00ef0a0b 100644 --- a/app/objects/c_objective.py +++ b/app/objects/c_objective.py @@ -9,6 +9,9 @@ class ObjectiveSchema(ma.Schema): + class Meta: + unknown = ma.EXCLUDE + id = ma.fields.String() name = ma.fields.String() description = ma.fields.String() diff --git a/app/objects/c_operation.py b/app/objects/c_operation.py index 600cf6e1c..9648b86cb 100644 --- a/app/objects/c_operation.py +++ b/app/objects/c_operation.py @@ -31,10 +31,14 @@ class InvalidOperationStateError(Exception): class OperationOutputRequestSchema(ma.Schema): - enable_agent_output = ma.fields.Boolean(default=False) + enable_agent_output = ma.fields.Boolean(dump_default=False) class OperationSchema(ma.Schema): + + class Meta: + unknown = ma.EXCLUDE + id = ma.fields.String() name = ma.fields.String(required=True) host_group = ma.fields.List(ma.fields.Nested(AgentSchema()), attribute='agents', dump_only=True) diff --git a/app/objects/c_schedule.py b/app/objects/c_schedule.py index f52720530..17ea93160 100644 --- a/app/objects/c_schedule.py +++ b/app/objects/c_schedule.py @@ -9,6 +9,9 @@ class ScheduleSchema(ma.Schema): + class Meta: + unknown = ma.EXCLUDE + id = ma.fields.String() schedule = ma.fields.Time(required=True) task = ma.fields.Nested(OperationSchema()) diff --git a/app/objects/c_source.py b/app/objects/c_source.py index bcbcb451d..d7852d20a 100644 --- a/app/objects/c_source.py +++ b/app/objects/c_source.py @@ -39,7 +39,7 @@ class SourceSchema(ma.Schema): def fix_adjustments(self, in_data, **_): x = [] raw_adjustments = in_data.pop('adjustments', {}) - if raw_adjustments and type(raw_adjustments) == dict: + if raw_adjustments and isinstance(raw_adjustments, dict): for ability_id, adjustments in raw_adjustments.items(): for trait, block in adjustments.items(): for change in block: diff --git a/app/objects/secondclass/c_link.py b/app/objects/secondclass/c_link.py index 793dc34a0..7afd4dc7e 100644 --- a/app/objects/secondclass/c_link.py +++ b/app/objects/secondclass/c_link.py @@ -39,7 +39,7 @@ class Meta: relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema())) used = ma.fields.List(ma.fields.Nested(FactSchema())) unique = ma.fields.String() - collect = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, default='') + collect = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, dump_default='') finish = ma.fields.String() ability = ma.fields.Nested(AbilitySchema()) executor = ma.fields.Nested(ExecutorSchema()) diff --git a/app/utility/base_world.py b/app/utility/base_world.py index 291b01e95..96e460c59 100644 --- a/app/utility/base_world.py +++ b/app/utility/base_world.py @@ -4,7 +4,7 @@ import yaml import logging import subprocess -import distutils.version +import packaging.version from base64 import b64encode, b64decode from datetime import datetime, timezone from importlib import import_module @@ -126,7 +126,7 @@ def check_program_version(command, version, **kwargs): def compare_versions(version_string, minimum_version): version = parse_version(version_string) - return distutils.version.StrictVersion(version) >= distutils.version.StrictVersion(str(minimum_version)) + return packaging.version.parse(version) >= packaging.version.parse(str(minimum_version)) def parse_version(version_string, pattern=r'([0-9]+(?:\.[0-9]+)+)'): groups = re.search(pattern, version_string) diff --git a/requirements.txt b/requirements.txt index 557eac315..eea409f15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ aiohttp-jinja2==1.5.1 -aiohttp==3.8.6 +aiohttp==3.9.3 aiohttp_session==2.12.0 aiohttp-security==0.4.0 -aiohttp-apispec==2.2.3 -jinja2==3.1.2 +aiohttp-apispec==3.0.0b2 +jinja2==3.1.3 pyyaml==6.0.1 -cryptography==41.0.4 +cryptography==42.0.2 websockets==11.0.3 Sphinx==7.1.2 sphinx_rtd_theme==1.3.0 @@ -22,3 +22,4 @@ Markdown==3.4.4 # training dnspython==2.4.2 asyncssh==2.14.1 aioftp~=0.20.0 +packaging==23.2 diff --git a/tests/api/v2/handlers/test_schedules_api.py b/tests/api/v2/handlers/test_schedules_api.py index 369bde476..446a6f2dd 100644 --- a/tests/api/v2/handlers/test_schedules_api.py +++ b/tests/api/v2/handlers/test_schedules_api.py @@ -27,7 +27,7 @@ def expected_updated_schedule_dump(test_schedule, updated_schedule_payload): def _merge_dictionaries(dict1, dict2): for key in dict1.keys(): - if type(dict1[key]) == dict: + if isinstance(dict1[key], dict): _merge_dictionaries(dict1[key], dict2[key]) else: dict2[key] = dict1[key] diff --git a/tests/planners/test_atomic.py b/tests/planners/test_atomic.py index 11f3a67e8..bd481fa69 100644 --- a/tests/planners/test_atomic.py +++ b/tests/planners/test_atomic.py @@ -35,6 +35,9 @@ class LinkStub(): def __init__(self, ability_id): self.ability = AbilityStub(ability_id) + def __eq__(self, other): + return self.ability.ability_id == other.ability.ability_id + @pytest.fixture def atomic_planner(): @@ -61,8 +64,8 @@ def test_atomic_with_links_in_order(self, event_loop, atomic_planner): assert atomic_planner.operation.apply.call_count == 1 assert atomic_planner.operation.wait_for_links_completion.call_count == 1 - assert atomic_planner.operation.apply.called_with(LinkStub('ability_b')) - assert atomic_planner.operation.wait_for_links_completion.called_with([LinkStub('ability_b')]) + atomic_planner.operation.apply.assert_called_with(LinkStub('ability_b')) + atomic_planner.operation.wait_for_links_completion.assert_called_with([LinkStub('ability_b')]) def test_atomic_with_links_out_of_order(self, event_loop, atomic_planner): @@ -77,8 +80,8 @@ def test_atomic_with_links_out_of_order(self, event_loop, atomic_planner): assert atomic_planner.operation.apply.call_count == 1 assert atomic_planner.operation.wait_for_links_completion.call_count == 1 - assert atomic_planner.operation.apply.called_with(LinkStub('ability_b')) - assert atomic_planner.operation.wait_for_links_completion.called_with([LinkStub('ability_b')]) + atomic_planner.operation.apply.assert_called_with(LinkStub('ability_b')) + atomic_planner.operation.wait_for_links_completion.assert_called_with([LinkStub('ability_b')]) def test_atomic_no_links(self, event_loop, atomic_planner): diff --git a/tox.ini b/tox.ini index 1a3de5584..5c2d14aae 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ [tox] skipsdist = True envlist = - py{38,39,310,311} + py{38,39,310,311,312} style coverage safety