diff --git a/src/integration-tests/README.md b/src/integration-tests/README.md index 31cec2a21b..ad96efd1ff 100644 --- a/src/integration-tests/README.md +++ b/src/integration-tests/README.md @@ -1,46 +1,14 @@ -## Integration Tests +# BlazingMQ Integration Tests -This directory contains integration tests based on the `ito` and 'bmqit' -frameworks. It contains the following test suites: +[WIP] +To run the tests: -### `00breathing_test.py` - -Provides a basic integration test suite. - - -### `20cluster_node_shutdown_test.py` - -Integration test that shuts down a cluster node and confirms that the system -recovers and keeps working as expected. - - -### `20node_status_change_test.py` - -Integration test that suspends a node and confirms that the system recovers -and performs as expected. - -### `30leader_node_delay.py` - -Integration test that temporarily suspends the leader node, resulting in -followers becoming leaderless. When leader is unpaused, they will re-discover -the leader but in PASSIVE state, and then in ACTIVE state once healing logic -kicks in. - -### `50appids_test.py` - -Integration test suite exercising AppIDs. - - -### `50broadcast_test.py` - -Integration test suite exercising broadcast functionality. - - -### `50list_messages_test.py` - -Integration test suite exercising list messages functionality. - -### `50maxqueues_test.py` - -Integration test suite exercising Max Queues functionality. +* (create and) activate a Python 3.8 (or above) `venv` + * `python -m venv /path/to/venv` + * `source /path/to/venv/bin/activate` +* install required modules + * `pip install -r src/python/requirements.txt` +* run the tests + * `cd src/integration-tests` + * `./run-tests [extra pytest options]` diff --git a/src/integration-tests/conftest.py b/src/integration-tests/conftest.py index f51f05209c..384d3d664f 100644 --- a/src/integration-tests/conftest.py +++ b/src/integration-tests/conftest.py @@ -1,9 +1,10 @@ import contextlib import logging +import pytest -import bmq.dev.it.logging -import bmq.util.logging as bul -from bmq.dev.pytest import PYTEST_LOG_SPEC_VAR +import blazingmq.dev.it.logging +import blazingmq.util.logging as bul +from blazingmq.dev.pytest import PYTEST_LOG_SPEC_VAR def pytest_addoption(parser): @@ -89,9 +90,18 @@ def pytest_addoption(parser): ) parser.addini(PYTEST_LOG_SPEC_VAR, help_, type=None, default=None) + help_ = "run only with the specified order" + parser.addoption( + "--bmq-wave", + type=int, + action="store", + metavar="WAVE", + help=help_, + ) + def pytest_configure(config): - logging.setLoggerClass(bmq.dev.it.logging.BMQLogger) + logging.setLoggerClass(blazingmq.dev.it.logging.BMQLogger) level_spec = config.getoption(PYTEST_LOG_SPEC_VAR) or config.getini( PYTEST_LOG_SPEC_VAR @@ -103,3 +113,28 @@ def pytest_configure(config): top_level = levels[0] logging.getLogger("proc").setLevel(top_level) logging.getLogger("test").setLevel(top_level) + + +def pytest_collection_modifyitems(config, items): + active_wave = config.getoption("bmq_wave") + if active_wave is None: + return + + for item in items: + mark = None + for mark in item.iter_markers(name="order"): + pass + + if mark is None: + order = 0 + else: + order = int(mark.args[0]) + + if order == active_wave: + continue + + item.add_marker( + pytest.mark.skip( + reason=f"order = {order}, running {active_wave} only" + ) + ) diff --git a/src/integration-tests/pytest.ini b/src/integration-tests/pytest.ini index 6b246791d0..451bd4c417 100644 --- a/src/integration-tests/pytest.ini +++ b/src/integration-tests/pytest.ini @@ -1,8 +1,8 @@ [pytest] -log_cli_format = %(bmqContext)-16s %(filename)s:%(lineno)d %(message)s +log_cli_format = %(bmqprocess)-16s %(filename)s:%(lineno)d %(message)s log_level = INFO -log_format = %(bmqContext16)s %(asctime)s.%(msecs)03d (%(thread)15d) %(levelname)-8s %(name24)s %(filename)10s:%(lineno)-03d %(message)s -log_file_format = %(bmqContext16)s %(asctime)s.%(msecs)03d (%(thread)15d) %(levelname)-8s %(name24)s %(filename)10s:%(lineno)-03d %(message)s +log_format = %(bmqprocess16)s %(asctime)s.%(msecs)03d (%(thread)15d) %(levelname)-8s %(name24)s %(filename)10s:%(lineno)-03d %(message)s +log_file_format = %(bmqprocess16)s %(asctime)s.%(msecs)03d (%(thread)15d) %(levelname)-8s %(name24)s %(filename)10s:%(lineno)-03d %(message)s log_file_level = INFO log_file_date_format = %d%b%Y_%H:%M:%S addopts = --strict-markers -p no:cacheprovider diff --git a/src/integration-tests/run-tests b/src/integration-tests/run-tests new file mode 100755 index 0000000000..c2f8ef5183 --- /dev/null +++ b/src/integration-tests/run-tests @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +set -e + +repo_dir=$(realpath "$0") +repo_dir=${repo_dir%/src/*} + +export PYTHONPATH=$repo_dir/src/python:$PYTHONPATH +cd "$repo_dir/src/integration-tests" + +python -m pytest -m "not csl_mode and not fsm_mode" "$@" diff --git a/src/integration-tests/test_admin_client.py b/src/integration-tests/test_admin_client.py index 6a8d001ab2..6733b3ef8b 100644 --- a/src/integration-tests/test_admin_client.py +++ b/src/integration-tests/test_admin_client.py @@ -6,12 +6,12 @@ import json import re -from bmq.dev.it.fixtures import Cluster, local_cluster # pylint: disable=unused-import -from bmq.dev.it.process.admin import AdminClient +from blazingmq.dev.it.fixtures import Cluster, single_node, order # pylint: disable=unused-import +from blazingmq.dev.it.process.admin import AdminClient -def test_admin(local_cluster: Cluster): - cluster: Cluster = local_cluster +def test_admin(single_node: Cluster): + cluster: Cluster = single_node endpoint: str = cluster.config.definition.nodes[0].transport.tcp.endpoint # type: ignore # Extract the (host, port) pair from the config diff --git a/src/integration-tests/test_alarms.py b/src/integration-tests/test_alarms.py index df94606ace..b662435de3 100644 --- a/src/integration-tests/test_alarms.py +++ b/src/integration-tests/test_alarms.py @@ -4,8 +4,8 @@ import time -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster, cluster, tweak # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import Cluster, cluster, order, tweak # pylint: disable=unused-import @tweak.cluster.queue_operations.consumption_monitor_period_ms(500) diff --git a/src/integration-tests/test_appids.py b/src/integration-tests/test_appids.py index c2ff1608ee..66938dd2fa 100644 --- a/src/integration-tests/test_appids.py +++ b/src/integration-tests/test_appids.py @@ -1,16 +1,19 @@ import time from typing import List -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, cluster, - logger, - standard_cluster, + test_logger, + order, + multi_node, tweak, ) -from bmq.dev.it.process.client import Client -from bmq.dev.it.util import attempt, wait_until +from blazingmq.dev.it.process.client import Client +from blazingmq.dev.it.util import attempt, wait_until + +pytestmark = order(3) authorized_app_ids = ["foo", "bar", "baz"] timeout = 60 @@ -23,10 +26,10 @@ def set_app_ids(cluster: Cluster, app_ids: List[str]): # noqa: F811 cluster.config.domains[ tc.DOMAIN_FANOUT ].definition.parameters.mode.fanout.app_ids = app_ids # type: ignore - cluster.reconfigure_domain_values(tc.DOMAIN_FANOUT, {}, succeed=True) + cluster.reconfigure_domain(tc.DOMAIN_FANOUT, succeed=True) -def test_open_alarm_authorize_post(cluster: Cluster, logger): +def test_open_alarm_authorize_post(cluster: Cluster): leader = cluster.last_known_leader proxies = cluster.proxy_cycle() @@ -62,16 +65,16 @@ def test_open_alarm_authorize_post(cluster: Cluster, logger): leader.dump_queue_internals(tc.DOMAIN_FANOUT, tc.TEST_QUEUE) - barStatus, bazStatus, fooStatus, quuxStatus = sorted( + bar_status, baz_status, foo_status, quuxStatus = sorted( [ leader.capture(r"(\w+).*: status=(\w+)(?:, StorageIter.atEnd=(\w+))?", 60) for i in all_app_ids ], - key=lambda m: m[1], + key=lambda match: match[1], ) - assert barStatus[2] == "alive" - assert bazStatus[2] == "alive" - assert fooStatus[2] == "alive" + assert bar_status[2] == "alive" + assert baz_status[2] == "alive" + assert foo_status[2] == "alive" assert quuxStatus.group(2, 3) == ("unauthorized", None) assert ( @@ -87,7 +90,7 @@ def test_open_alarm_authorize_post(cluster: Cluster, logger): # --------------------------------------------------------------------- # Check that 'quux' (unauthorized) client did not receive it. - logger.info('Check that "quux" has not seen any messages') + test_logger.info('Check that "quux" has not seen any messages') assert not quux.wait_push_event(timeout=2, quiet=True) assert len(quux.list(f"{tc.URI_FANOUT}?id=quux", block=True)) == 0 @@ -116,7 +119,7 @@ def test_open_alarm_authorize_post(cluster: Cluster, logger): leader.dump_queue_internals(tc.DOMAIN_FANOUT, tc.TEST_QUEUE) # pylint: disable=cell-var-from-loop; passing lambda to 'wait_until' is safe for app_id in authorized_app_ids: - logger.info(f"Check if {app_id} has seen 2 messages") + test_logger.info(f"Check if {app_id} has seen 2 messages") assert wait_until( lambda: len( consumers[app_id].list(f"{tc.URI_FANOUT}?id={app_id}", block=True) @@ -125,7 +128,7 @@ def test_open_alarm_authorize_post(cluster: Cluster, logger): 3, ) - logger.info("Check if quux has seen 1 message") + test_logger.info("Check if quux has seen 1 message") assert wait_until( lambda: len(quux.list(f"{tc.URI_FANOUT}?id=quux", block=True)) == 1, 3 ) @@ -236,7 +239,7 @@ def _test_command_errors(cluster): set_app_ids(cluster, authorized_app_ids) -def test_unregister_in_presence_of_queues(cluster: Cluster, logger): +def test_unregister_in_presence_of_queues(cluster: Cluster): leader = cluster.last_known_leader proxies = cluster.proxy_cycle() @@ -266,7 +269,7 @@ def _(): assert leader.outputs_substr("Num virtual storages: 2") assert leader.outputs_substr("foo: status=unauthorized") - logger.info("confirm msg 1 for bar, expecting 1 msg in storage") + test_logger.info("confirm msg 1 for bar, expecting 1 msg in storage") time.sleep(1) # Let the message reach the proxy bar.confirm(tc.URI_FANOUT_BAR, "+1", succeed=True) @@ -275,7 +278,7 @@ def _(): leader.dump_queue_internals(tc.DOMAIN_FANOUT, tc.TEST_QUEUE) assert leader.outputs_regex("Storage.*: 1 messages") - logger.info("confirm msg 1 for baz, expecting 0 msg in storage") + test_logger.info("confirm msg 1 for baz, expecting 0 msg in storage") time.sleep(1) # Let the message reach the proxy baz.confirm(tc.URI_FANOUT_BAZ, "+1", succeed=True) @@ -489,18 +492,18 @@ def test_unauthorization(cluster: Cluster): consumer.open(appid_uri, flags=["read"], succeed=True) -def test_two_consumers_of_unauthorized_app(standard_cluster: Cluster): +def test_two_consumers_of_unauthorized_app(multi_node: Cluster): """DRQS 167201621: First client open authorized and unauthorized apps; second client opens unauthorized app. Then, primary shuts down causing replica to issue wildcard close requests to primary. """ - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader - replica1 = standard_cluster.nodes()[0] + replica1 = multi_node.nodes()[0] if replica1 == leader: - replica1 = standard_cluster.nodes()[1] + replica1 = multi_node.nodes()[1] # --------------------------------------------------------------------- # Two "foo" and "unauthorized" consumers @@ -508,9 +511,9 @@ def test_two_consumers_of_unauthorized_app(standard_cluster: Cluster): consumer1.open(tc.URI_FANOUT_FOO, flags=["read"], succeed=True) consumer1.open(f"{tc.URI_FANOUT}?id=unauthorized", flags=["read"], succeed=True) - replica2 = standard_cluster.nodes()[2] + replica2 = multi_node.nodes()[2] if replica2 == leader: - replica2 = standard_cluster.nodes()[3] + replica2 = multi_node.nodes()[3] consumer2 = replica2.create_client("consumer2") consumer2.open(f"{tc.URI_FANOUT}?id=unauthorized", flags=["read"], succeed=True) diff --git a/src/integration-tests/test_breathing.py b/src/integration-tests/test_breathing.py index d32a0b6216..ffba36f33b 100644 --- a/src/integration-tests/test_breathing.py +++ b/src/integration-tests/test_breathing.py @@ -5,20 +5,21 @@ from collections import namedtuple -import bmq.dev.it.testconstants as tc +import blazingmq.dev.it.testconstants as tc import pytest -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, cartesian_product_cluster, cluster, - standard_cluster, + order, + multi_node, start_cluster, tweak, ) -from bmq.dev.it.process.client import Client -from bmq.dev.it.util import wait_until +from blazingmq.dev.it.process.client import Client +from blazingmq.dev.it.util import wait_until -pytestmark = pytest.mark.order(0) +pytestmark = order(1) BmqClient = namedtuple("BmqClient", "handle, uri") @@ -611,12 +612,12 @@ def test_verify_priority_queue_redelivery(cluster: Cluster): _stop_clients([producer, consumer]) -def test_verify_partial_close(standard_cluster: Cluster): +def test_verify_partial_close(multi_node: Cluster): """Drop one of two producers both having unacked message (primary is suspended. Make sure the remaining producer does not get NACK but gets ACK when primary resumes. """ - proxies = standard_cluster.proxy_cycle() + proxies = multi_node.proxy_cycle() proxy = next(proxies) proxy = next(proxies) @@ -627,7 +628,7 @@ def test_verify_partial_close(standard_cluster: Cluster): producer2 = proxy.create_client("producer2") producer2.open(tc.URI_FANOUT, flags=["write", "ack"], succeed=True) - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader leader.suspend() producer1.post(tc.URI_FANOUT, payload=["1"], succeed=True, wait_ack=False) @@ -644,16 +645,16 @@ def test_verify_partial_close(standard_cluster: Cluster): @start_cluster(True, True, True) @tweak.cluster.queue_operations.open_timeout_ms(2) -def test_command_timeout(standard_cluster: Cluster): +def test_command_timeout(multi_node: Cluster): """Simple test to execute onOpenQueueResponse timeout.""" # make sure the cluster is healthy and the queue is assigned # Cannot use proxies as they do not read cluster config - leader = standard_cluster.last_known_leader - host = standard_cluster.nodes()[0] + leader = multi_node.last_known_leader + host = multi_node.nodes()[0] if host == leader: - host = standard_cluster.nodes()[1] + host = multi_node.nodes()[1] client = host.create_client("client") # this may fail due to the short timeout; we just need queue assigned @@ -667,19 +668,19 @@ def test_command_timeout(standard_cluster: Cluster): assert result == Client.e_TIMEOUT -def test_queue_purge_command(standard_cluster: Cluster): +def test_queue_purge_command(multi_node: Cluster): """Ensure that 'queue purge' command is working as expected. Post a message to the queue, then purge the queue, then bring up a consumer. Ensure that consumer does not receive any message. """ - proxy = next(standard_cluster.proxy_cycle()) + proxy = next(multi_node.proxy_cycle()) # Start a producer and post a message producer = proxy.create_client("producer") producer.open(tc.URI_FANOUT, flags=["write", "ack"], succeed=True) producer.post(tc.URI_FANOUT, ["msg1"], succeed=True, wait_ack=True) - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader # Purge queue, but *only* for 'foo' appId leader.command(f"DOMAINS DOMAIN {tc.DOMAIN_FANOUT} QUEUE {tc.TEST_QUEUE} PURGE foo") diff --git a/src/integration-tests/test_broadcast.py b/src/integration-tests/test_broadcast.py index c9e054aadf..af143e331a 100644 --- a/src/integration-tests/test_broadcast.py +++ b/src/integration-tests/test_broadcast.py @@ -1,8 +1,10 @@ from itertools import islice -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster, cluster # pylint: disable=unused-import -from bmq.dev.it.process.client import Client +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import Cluster, cluster, order # pylint: disable=unused-import +from blazingmq.dev.it.process.client import Client + +pytestmark = order(3) def test_breathing(cluster: Cluster): diff --git a/src/integration-tests/test_cluster_node_shutdown.py b/src/integration-tests/test_cluster_node_shutdown.py index daff63e237..f1b3a094e7 100644 --- a/src/integration-tests/test_cluster_node_shutdown.py +++ b/src/integration-tests/test_cluster_node_shutdown.py @@ -7,12 +7,15 @@ from threading import Semaphore from time import sleep -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, - standard_cluster, + order, + multi_node, ) -from bmq.dev.it.process.client import Client +from blazingmq.dev.it.process.client import Client + +pytestmark = order(6) class TestClusterNodeShutdown: @@ -98,14 +101,14 @@ def setup_cluster(self, cluster: Cluster): == Client.e_SUCCESS ) - def test_primary_shutdown_with_proxy(self, standard_cluster: Cluster): - cluster = standard_cluster + def test_primary_shutdown_with_proxy(self, multi_node: Cluster): + cluster = multi_node primary = cluster.last_known_leader self._post_kill_recover_post(cluster, primary) - def test_replica_shutdown_with_proxy(self, standard_cluster: Cluster): - cluster = standard_cluster + def test_replica_shutdown_with_proxy(self, multi_node: Cluster): + cluster = multi_node replica = cluster.process(self.proxy2.get_active_node()) self._post_kill_recover_post(cluster, replica) diff --git a/src/integration-tests/test_compression.py b/src/integration-tests/test_compression.py index c79660226b..39f673ca41 100644 --- a/src/integration-tests/test_compression.py +++ b/src/integration-tests/test_compression.py @@ -4,10 +4,11 @@ - cluster gets restarted after sending ack to producer. """ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster, cluster # pylint: disable=unused-import -from bmq.dev.it.util import random_string +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import Cluster, cluster, order # pylint: disable=unused-import +from blazingmq.dev.it.util import random_string +pytestmark = order(10) def test_compression_restart(cluster: Cluster): @@ -37,7 +38,7 @@ def test_compression_restart(cluster: Cluster): cluster.restart_nodes() # For a standard cluster, states have already been restored as part of # leader re-election. - if cluster.is_local: + if cluster.is_single_node: producer.wait_state_restored() consumer = next(proxies).create_client("consumer") diff --git a/src/integration-tests/test_confirm_after_killing_primary.py b/src/integration-tests/test_confirm_after_killing_primary.py index 90b5fc47b7..6899d90d63 100644 --- a/src/integration-tests/test_confirm_after_killing_primary.py +++ b/src/integration-tests/test_confirm_after_killing_primary.py @@ -4,13 +4,14 @@ local. """ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster -from bmq.dev.it.fixtures import ( - standard_cluster as cluster, # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( + Cluster, + multi_node as cluster, + order, # pylint: disable=unused-import ) -from bmq.dev.it.process.client import Client -from bmq.dev.it.util import wait_until +from blazingmq.dev.it.process.client import Client +from blazingmq.dev.it.util import wait_until def test_confirm_after_killing_primary(cluster: Cluster): diff --git a/src/integration-tests/test_confirms_buffering.py b/src/integration-tests/test_confirms_buffering.py index d1ca080783..eb09a95fa9 100644 --- a/src/integration-tests/test_confirms_buffering.py +++ b/src/integration-tests/test_confirms_buffering.py @@ -1,9 +1,10 @@ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, - standard_cluster, + order, + multi_node, ) -from bmq.dev.it.util import wait_until +from blazingmq.dev.it.util import wait_until class TestConfirmsBuffering: @@ -39,7 +40,7 @@ def setup_cluster(self, cluster: Cluster): self.candidate = node def test_kill_primary_confirm_puts_close_app_start_primary( - self, standard_cluster: Cluster # pylint: disable=unused-argument + self, multi_node: Cluster # pylint: disable=unused-argument ): # add one more tc.URI_FANOUT_BAZ consumer baz = self.replica_proxy.create_client("baz") diff --git a/src/integration-tests/test_fanout_priorities.py b/src/integration-tests/test_fanout_priorities.py index c40302df46..e3310b66ef 100644 --- a/src/integration-tests/test_fanout_priorities.py +++ b/src/integration-tests/test_fanout_priorities.py @@ -4,10 +4,11 @@ """ -from bmq.dev.it.fixtures import Cluster, cluster # pylint: disable=unused-import -from bmq.dev.it.process.client import Client -from bmq.dev.it.util import wait_until +from blazingmq.dev.it.fixtures import Cluster, cluster, order # pylint: disable=unused-import +from blazingmq.dev.it.process.client import Client +from blazingmq.dev.it.util import wait_until +pytestmark = order(4) def test_fanout_priorities(cluster: Cluster): # create foo, bar, and baz clients on every node. @@ -26,13 +27,13 @@ def test_fanout_priorities(cluster: Cluster): nodes.append(next(proxies)) for node in nodes: - client = node.create_client(f"{node.name}Consumer2") + client = node.create_client("consumer2") [queues] = client.open_fanout_queues( 1, flags=["read"], consumer_priority=2, block=True, appids=apps ) highPriorityQueues += queues - client = node.create_client(f"{node.name}Consumer1") + client = node.create_client("consumer1") [queues] = client.open_fanout_queues( 1, flags=["read"], consumer_priority=1, block=True, appids=apps ) diff --git a/src/integration-tests/test_graceful_shutdown.py b/src/integration-tests/test_graceful_shutdown.py index b9e6a453a9..03b9609436 100644 --- a/src/integration-tests/test_graceful_shutdown.py +++ b/src/integration-tests/test_graceful_shutdown.py @@ -2,20 +2,21 @@ import re from typing import Iterator -import bmq.dev.it.testconstants as tc +import blazingmq.dev.it.testconstants as tc import pytest -from bmq.dev.it import fixtures -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +from blazingmq.dev.it import fixtures +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, Mode, - logger, - standard_cluster, + test_logger, + order, + multi_node, tweak, virtual_cluster_config, ) -from bmq.dev.it.process.client import Client -from bmq.dev.it.util import wait_until -from bmq.dev.workspace import Workspace +from blazingmq.dev.it.process.client import Client +from blazingmq.dev.it.util import wait_until +from blazingmq.dev.workspace import Workspace OTHER_DOMAIN = f"{tc.DOMAIN_PRIORITY}.other" @@ -60,9 +61,9 @@ def multi_cluster(request): class TestGracefulShutdown: - def post_kill_confirm(self, node, peer, logger): + def post_kill_confirm(self, node, peer): - logger.info("posting...") + test_logger.info("posting...") # post 3 PUTs for i in range(1, 4): @@ -149,8 +150,8 @@ def num_broker_messages(): node.wait() # assert node.return_code == 0 - def kill_wait_unconfirmed(self, peer, logger): - logger.info("posting...") + def kill_wait_unconfirmed(self, peer): + test_logger.info("posting...") uriWrite = tc.URI_FANOUT uriRead = tc.URI_FANOUT_FOO @@ -214,46 +215,46 @@ def setup_cluster(self, cluster): self.producer.open(tc.URI_FANOUT, flags=["write,ack"], succeed=True) @tweak.cluster.queue_operations.stop_timeout_ms(1000) - def test_shutting_down_primary(self, standard_cluster: Cluster, logger): - cluster = standard_cluster + def test_shutting_down_primary(self, multi_node: Cluster): + cluster = multi_node leader = cluster.last_known_leader active_node = cluster.process(self.replica_proxy.get_active_node()) - self.post_kill_confirm(leader, active_node, logger) + self.post_kill_confirm(leader, active_node) @tweak.cluster.queue_operations.stop_timeout_ms(1000) - def test_shutting_down_replica(self, standard_cluster: Cluster, logger): - cluster = standard_cluster + def test_shutting_down_replica(self, multi_node: Cluster): + cluster = multi_node leader = cluster.last_known_leader active_node = cluster.process(self.replica_proxy.get_active_node()) - self.post_kill_confirm(active_node, leader, logger) + self.post_kill_confirm(active_node, leader) @tweak.cluster.queue_operations.stop_timeout_ms(1000) @tweak.cluster.queue_operations.shutdown_timeout_ms(5000) def test_wait_unconfirmed_proxy( - self, standard_cluster, logger # pylint: disable=unused-argument + self, multi_node # pylint: disable=unused-argument ): proxy = self.replica_proxy - self.kill_wait_unconfirmed(proxy, logger) + self.kill_wait_unconfirmed(proxy) @tweak.cluster.queue_operations.stop_timeout_ms(1000) @tweak.cluster.queue_operations.shutdown_timeout_ms(5000) def test_wait_unconfirmed_replica( - self, standard_cluster, logger # pylint: disable=unused-argument + self, multi_node # pylint: disable=unused-argument ): - cluster = standard_cluster + cluster = multi_node replica = cluster.process(self.replica_proxy.get_active_node()) - self.kill_wait_unconfirmed(replica, logger) + self.kill_wait_unconfirmed(replica) @tweak.cluster.queue_operations.stop_timeout_ms(3000) @tweak.cluster.queue_operations.shutdown_timeout_ms(2000) def test_cancel_unconfirmed_timer( - self, standard_cluster # pylint: disable=unused-argument + self, multi_node # pylint: disable=unused-argument ): uriWrite = tc.URI_FANOUT uriRead = tc.URI_FANOUT_FOO - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader # post 2 PUTs self.producer.post(uriWrite, payload=["msg1"], succeed=True) @@ -264,7 +265,7 @@ def test_cancel_unconfirmed_timer( consumer.open(uriRead, flags=["read"], succeed=True) - replica = standard_cluster.process(self.replica_proxy.get_active_node()) + replica = multi_node.process(self.replica_proxy.get_active_node()) # receive messages consumer.wait_push_event() diff --git a/src/integration-tests/test_leader_node_delay.py b/src/integration-tests/test_leader_node_delay.py index 050e5d55ee..9ec5dda0d1 100644 --- a/src/integration-tests/test_leader_node_delay.py +++ b/src/integration-tests/test_leader_node_delay.py @@ -5,12 +5,15 @@ transitioning from PASSIVE to ACTIVE. """ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster -from bmq.dev.it.fixtures import ( - standard_cluster as cluster, # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import Cluster +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import + multi_node as cluster, + order, ) +pytestmark = order(6) + def test_leader_node_delay(cluster: Cluster): leader = cluster.last_known_leader diff --git a/src/integration-tests/test_list_messages.py b/src/integration-tests/test_list_messages.py index adc27140e9..1312475894 100644 --- a/src/integration-tests/test_list_messages.py +++ b/src/integration-tests/test_list_messages.py @@ -1,11 +1,12 @@ import time -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster, cluster # pylint: disable=unused-import -from bmq.dev.it.util import wait_until +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import Cluster, cluster, order # pylint: disable=unused-import +from blazingmq.dev.it.util import wait_until TIMEOUT = 30 +pytestmark = order(3) def expected_header(start, count, total, size): return ( diff --git a/src/integration-tests/test_maxqueues.py b/src/integration-tests/test_maxqueues.py index e847957c8f..85596212f0 100644 --- a/src/integration-tests/test_maxqueues.py +++ b/src/integration-tests/test_maxqueues.py @@ -1,12 +1,15 @@ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, cluster, - standard_cluster, + order, + multi_node, start_cluster, tweak, ) -from bmq.dev.it.process.client import Client +from blazingmq.dev.it.process.client import Client + +pytestmark = order(4) timeout = 60 @@ -65,7 +68,7 @@ def test_max_queue(self, cluster: Cluster): active_replica = ( leader - if cluster.is_local + if cluster.is_single_node else cluster.process(replica_proxy.get_active_node()) ) check_num_assigned_queues(5, leader, active_replica) @@ -101,7 +104,7 @@ def test_max_queue(self, cluster: Cluster): # --------------------------------------------------------------------- # force a change of leadership - if not cluster.is_local: + if not cluster.is_single_node: for node in cluster.nodes(): # avoid picking up old leader node.drain() @@ -127,8 +130,8 @@ def test_max_queue(self, cluster: Cluster): # open queues before there is a leader (restore, in flight contexts) @start_cluster(False) - def test_max_queue_restore(self, standard_cluster: Cluster): - cluster = standard_cluster + def test_max_queue_restore(self, multi_node: Cluster): + cluster = multi_node cluster.start_node("west1") cluster.start_node("east1") diff --git a/src/integration-tests/test_maxunconfirmed.py b/src/integration-tests/test_maxunconfirmed.py index ef8437af4e..47b5fdc348 100644 --- a/src/integration-tests/test_maxunconfirmed.py +++ b/src/integration-tests/test_maxunconfirmed.py @@ -1,15 +1,17 @@ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, - standard_cluster, + order, + multi_node, tweak, ) -from bmq.dev.it.process.client import Client +from blazingmq.dev.it.process.client import Client +pytestmark = order(4) class TestMaxunconfirmed: - def setup_cluster(self, standard_cluster): - proxies = standard_cluster.proxy_cycle() + def setup_cluster(self, multi_node): + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) self.proxy = next(proxies) @@ -31,7 +33,7 @@ def post_n_msgs(self, uri, n): return all(res == Client.e_SUCCESS for res in results) @tweak.cluster.queue_operations.stop_timeout_ms(1000) - def test_maxunconfirmed(self, standard_cluster: Cluster): + def test_maxunconfirmed(self, multi_node: Cluster): # Post 100 messages assert self.post_n_msgs(tc.URI_PRIORITY, 100) @@ -47,11 +49,11 @@ def test_maxunconfirmed(self, standard_cluster: Cluster): assert len(self.consumer.list(tc.URI_PRIORITY, block=True)) == 1 # Shutdown the primary - leader = standard_cluster.last_known_leader - active_node = standard_cluster.process(self.proxy.get_active_node()) + leader = multi_node.last_known_leader + active_node = multi_node.process(self.proxy.get_active_node()) active_node.set_quorum(1) - nodes = standard_cluster.nodes(exclude=[active_node, leader]) + nodes = multi_node.nodes(exclude=[active_node, leader]) for node in nodes: node.set_quorum(4) @@ -59,7 +61,7 @@ def test_maxunconfirmed(self, standard_cluster: Cluster): # Make sure the active node is new primary leader = active_node - assert leader == standard_cluster.wait_leader() + assert leader == multi_node.wait_leader() # Confirm 1 message self.consumer.confirm(tc.URI_PRIORITY, "*", succeed=True) diff --git a/src/integration-tests/test_poison_messages.py b/src/integration-tests/test_poison_messages.py index 4d9ba018a5..2320ebefd9 100644 --- a/src/integration-tests/test_poison_messages.py +++ b/src/integration-tests/test_poison_messages.py @@ -2,15 +2,17 @@ Testing poison message detection and handling. """ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, - standard_cluster, + order, + multi_node, start_cluster, tweak, ) -from bmq.dev.workspace import Workspace +from blazingmq.dev.workspace import Workspace +pytestmark = order(5) def message_throttling(high: int, low: int): def tweaker(workspace: Workspace): @@ -33,7 +35,7 @@ def _list_messages(self, broker, uri, messages): broker.list_messages(uri, tc.TEST_QUEUE, 0, len(messages)) assert broker.outputs_substr(f"Printing {len(messages)} message(s)", 10) - def _post_crash_consumers(self, standard_cluster, proxy, domain, suffixes): + def _post_crash_consumers(self, multi_node, proxy, domain, suffixes): # We want to make sure a messages aren't redelivered when the rda # count reaches zero after a consumer crash. In the case of fanout, # the 'suffixes' list will be populated with an app id for each @@ -67,8 +69,8 @@ def _post_crash_consumers(self, standard_cluster, proxy, domain, suffixes): consumer.open(f"{uri}{suffix}", flags=["read"], succeed=True) consumers.append(consumer) - replica = standard_cluster.process(proxy.get_active_node()) - leader = standard_cluster.last_known_leader + replica = multi_node.process(proxy.get_active_node()) + leader = multi_node.last_known_leader self._list_messages(proxy, domain, ["1"]) self._list_messages(replica, domain, ["1"]) @@ -109,12 +111,12 @@ def _post_crash_consumers(self, standard_cluster, proxy, domain, suffixes): # change the leader and check if the original message ('1') is still # gone replica.set_quorum(1) - nodes = standard_cluster.nodes(exclude=[replica, leader]) + nodes = multi_node.nodes(exclude=[replica, leader]) for node in nodes: node.set_quorum(4) leader.stop() leader = replica - assert leader == standard_cluster.wait_leader() + assert leader == multi_node.wait_leader() # Wait for new leader to become active primary by opening a queue from # a new producer for synchronization. @@ -131,7 +133,7 @@ def _post_crash_consumers(self, standard_cluster, proxy, domain, suffixes): consumer.confirm(f"{uri}{suffixes[count]}", "*", True) def _crash_consumer_restart_leader( - self, standard_cluster, proxy, domain, make_active_node_leader + self, multi_node, proxy, domain, make_active_node_leader ): # We want to make sure the rda counter resets to the value in the # configuration after losing a leader. Since the rda counter is set to @@ -155,8 +157,8 @@ def _crash_consumer_restart_leader( consumer = proxy.create_client("consumer_0") consumer.open(f"{uri}", flags=["read"], succeed=True) - replica = standard_cluster.process(proxy.get_active_node()) - leader = standard_cluster.last_known_leader + replica = multi_node.process(proxy.get_active_node()) + leader = multi_node.last_known_leader self._list_messages(proxy, domain, ["1"]) self._list_messages(replica, domain, ["1"]) @@ -178,14 +180,14 @@ def _crash_consumer_restart_leader( if make_active_node_leader: # make the active replica the new leader replica.set_quorum(1) - nodes = standard_cluster.nodes(exclude=replica) + nodes = multi_node.nodes(exclude=replica) for node in nodes: node.set_quorum(4) leader.stop() leader = replica else: # make a replica that isn't the active node the new leader - nodes = standard_cluster.nodes(exclude=[replica, leader]) + nodes = multi_node.nodes(exclude=[replica, leader]) assert nodes leader_candidate = nodes.pop() leader_candidate.set_quorum(1) @@ -194,7 +196,7 @@ def _crash_consumer_restart_leader( node.set_quorum(4) leader.stop() leader = leader_candidate - assert leader == standard_cluster.wait_leader() + assert leader == multi_node.wait_leader() consumer.check_exit_code = False consumer.kill() @@ -210,7 +212,7 @@ def _crash_consumer_restart_leader( producer.exit_gracefully() consumer.confirm(f"{uri}", "*", True) - def _crash_one_consumer(self, standard_cluster, proxy, domain, suffixes): + def _crash_one_consumer(self, multi_node, proxy, domain, suffixes): # We want to make sure when the rda counter reaches 0 for app #1 while # the other apps (#2 and #3) haven't been confirmed, the message # doesn't get redelivered for app #1. In this method, we will: @@ -273,7 +275,7 @@ def _crash_one_consumer(self, standard_cluster, proxy, domain, suffixes): # make sure after the confirms, the first message is gone from # everywhere - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader self._list_messages(proxy, domain, ["2"]) self._list_messages(leader, domain, ["2"]) @@ -281,7 +283,7 @@ def _crash_one_consumer(self, standard_cluster, proxy, domain, suffixes): for count, consumer in enumerate(consumers): consumer.confirm(f"{uri}{suffixes[count]}", "*", True) - def _stop_consumer_gracefully(self, standard_cluster, proxy, domain): + def _stop_consumer_gracefully(self, multi_node, proxy, domain): # We want to make sure the rda counter isn't decremented when a # consumer is shut down gracefully. To test this, we set the rda # counter to 1 and: @@ -305,13 +307,13 @@ def _stop_consumer_gracefully(self, standard_cluster, proxy, domain): consumer.open(f"{uri}", flags=["read"], succeed=True) consumer.wait_push_event() - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader self._list_messages(proxy, domain, ["1"]) self._list_messages(leader, domain, ["1"]) consumer.confirm(f"{uri}", "*", True) - def _crash_consumer_connected_to_replica(self, standard_cluster, proxy, domain): + def _crash_consumer_connected_to_replica(self, multi_node, proxy, domain): # We want to make sure when a consumer on a replica node crashes and the # reject message propagates to the primary, when a new consumer appears # on the same replica, the updated rda bubbles down from the primary to @@ -329,8 +331,8 @@ def _crash_consumer_connected_to_replica(self, standard_cluster, proxy, domain): # synchronize so we can test that the first message won't be # redelivered. uri = f"bmq://{domain}/{tc.TEST_QUEUE}" - leader = standard_cluster.last_known_leader - potential_replicas = standard_cluster.nodes(exclude=leader) + leader = multi_node.last_known_leader + potential_replicas = multi_node.nodes(exclude=leader) assert potential_replicas @@ -360,7 +362,7 @@ def _crash_consumer_connected_to_replica(self, standard_cluster, proxy, domain): assert msgs[0].payload == "2" consumer_1.confirm(f"{uri}", "*", True) - def _stop_proxy(self, standard_cluster, proxy, domain, should_kill): + def _stop_proxy(self, multi_node, proxy, domain, should_kill): # We want to make sure when a broker either crashes or exits # gracefully, outstanding messages from that broker's downstream aren't # rejected. To test this, we will set the rda to 1 and: @@ -380,7 +382,7 @@ def _stop_proxy(self, standard_cluster, proxy, domain, should_kill): consumer_0 = proxy.create_client("consumer_0") consumer_0.open(f"{uri}", flags=["read"], succeed=True) - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader msgs = consumer_0.list(block=True) assert len(msgs) == 1 @@ -407,32 +409,32 @@ def _stop_proxy(self, standard_cluster, proxy, domain, should_kill): @max_delivery_attempts(1) @message_throttling(high=0, low=0) - def test_poison_proxy_and_replica_priority(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_proxy_and_replica_priority(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy = next(proxies) - self._post_crash_consumers(standard_cluster, proxy, tc.DOMAIN_PRIORITY, [""]) + self._post_crash_consumers(multi_node, proxy, tc.DOMAIN_PRIORITY, [""]) @max_delivery_attempts(1) @message_throttling(high=0, low=0) - def test_poison_proxy_and_replica_fanout(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_proxy_and_replica_fanout(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy = next(proxies) self._post_crash_consumers( - standard_cluster, proxy, tc.DOMAIN_FANOUT, ["?id=foo", "?id=bar", "?id=baz"] + multi_node, proxy, tc.DOMAIN_FANOUT, ["?id=foo", "?id=bar", "?id=baz"] ) @max_delivery_attempts(2) - def test_poison_rda_reset_priority_active(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_rda_reset_priority_active(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy = next(proxies) self._crash_consumer_restart_leader( - standard_cluster, proxy, tc.DOMAIN_PRIORITY, True + multi_node, proxy, tc.DOMAIN_PRIORITY, True ) # when set to true, make the # active node of the proxy # the new leader @@ -440,57 +442,57 @@ def test_poison_rda_reset_priority_active(self, standard_cluster: Cluster): @max_delivery_attempts(2) @message_throttling(high=1, low=0) @start_cluster(True, True, True) - def test_poison_rda_reset_priority_non_active(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_rda_reset_priority_non_active(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy = next(proxies) self._crash_consumer_restart_leader( - standard_cluster, proxy, tc.DOMAIN_PRIORITY, False + multi_node, proxy, tc.DOMAIN_PRIORITY, False ) @max_delivery_attempts(2) @message_throttling(high=1, low=0) - def test_poison_fanout_crash_one_app(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_fanout_crash_one_app(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy = next(proxies) self._crash_one_consumer( - standard_cluster, proxy, tc.DOMAIN_FANOUT, ["?id=foo", "?id=bar", "?id=baz"] + multi_node, proxy, tc.DOMAIN_FANOUT, ["?id=foo", "?id=bar", "?id=baz"] ) @max_delivery_attempts(1) @message_throttling(high=0, low=0) - def test_poison_consumer_graceful_shutdown(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_consumer_graceful_shutdown(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy = next(proxies) - self._stop_consumer_gracefully(standard_cluster, proxy, tc.DOMAIN_PRIORITY) + self._stop_consumer_gracefully(multi_node, proxy, tc.DOMAIN_PRIORITY) @max_delivery_attempts(2) @message_throttling(high=1, low=0) - def test_poison_replica_receives_updated_rda(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_replica_receives_updated_rda(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() next(proxies) proxy = next(proxies) self._crash_consumer_connected_to_replica( - standard_cluster, proxy, tc.DOMAIN_PRIORITY + multi_node, proxy, tc.DOMAIN_PRIORITY ) @max_delivery_attempts(1) @message_throttling(high=0, low=0) - def test_poison_no_reject_broker_crash(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_no_reject_broker_crash(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() next(proxies) proxy = next(proxies) - self._stop_proxy(standard_cluster, proxy, tc.DOMAIN_PRIORITY, True) + self._stop_proxy(multi_node, proxy, tc.DOMAIN_PRIORITY, True) @max_delivery_attempts(1) @message_throttling(high=0, low=0) - def test_poison_no_reject_broker_graceful_shutdown(self, standard_cluster: Cluster): - proxies = standard_cluster.proxy_cycle() + def test_poison_no_reject_broker_graceful_shutdown(self, multi_node: Cluster): + proxies = multi_node.proxy_cycle() next(proxies) proxy = next(proxies) - self._stop_proxy(standard_cluster, proxy, tc.DOMAIN_PRIORITY, False) + self._stop_proxy(multi_node, proxy, tc.DOMAIN_PRIORITY, False) diff --git a/src/integration-tests/test_puts_retransmission.py b/src/integration-tests/test_puts_retransmission.py index 495152d415..4da49409ad 100644 --- a/src/integration-tests/test_puts_retransmission.py +++ b/src/integration-tests/test_puts_retransmission.py @@ -1,20 +1,19 @@ # pylint: disable=protected-access; TODO: fix -import contextlib -import os +from pathlib import Path import re -import signal from collections import namedtuple -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, cartesian_product_cluster, - logger, - standard_cluster, + multi_node, + order, + test_logger, tweak, ) -from bmq.dev.it.util import wait_until +from blazingmq.dev.it.util import wait_until BACKLOG_MESSAGES = 400 # minimum to consume before killing / shutting down NUM_MESSAGES = 2000 @@ -52,7 +51,9 @@ class TestPutsRetransmission: consitency. """ - def inspect_results(self, logger, allow_duplicates=False): + work_dir: Path + + def inspect_results(self, allow_duplicates=False): if self.active_node in self.cluster.virtual_nodes(): self.active_node.wait_status(wait_leader=True, wait_ready=False) @@ -65,7 +66,7 @@ def inspect_results(self, logger, allow_duplicates=False): ) for uri, consumer in zip(self.uris, self.consumers): - logger.info(f"{uri[0]}: received {consumer[1]} messages") + test_logger.info(f"{uri[0]}: received {consumer[1]} messages") # assert consumer[1] == NUM_MESSAGES leader.command(f"DOMAINS DOMAIN {self.domain} QUEUE {tc.TEST_QUEUE} INTERNALS") @@ -73,14 +74,14 @@ def inspect_results(self, logger, allow_duplicates=False): leader.list_messages(self.domain, tc.TEST_QUEUE, 0, NUM_MESSAGES) assert leader.outputs_substr("Printing 0 message(s)", 10) - os.kill(self.producer._process.pid, signal.SIGTERM) + self.producer.force_stop() for consumer in self.consumers: - os.kill(consumer[0]._process.pid, signal.SIGTERM) + consumer[0].force_stop() - self.parse_message_logs(logger, allow_duplicates) + self.parse_message_logs(allow_duplicates=allow_duplicates) - def parse_message_logs(self, logger, allow_duplicates=False): + def parse_message_logs(self, allow_duplicates=False): Put = namedtuple("Put", ["message_index", "guid"]) Ack = namedtuple("Ack", ["message_index", "guid", "status"]) @@ -98,17 +99,16 @@ def parse_message_logs(self, logger, allow_duplicates=False): num_duplicates = 0 num_errors = 0 - def warning(logger, prefix, p1, p2=None): - logger.warning(f'{prefix}: {p1}{f" vs existing {p2}" if p2 else ""}') + def warning(test_logger, prefix, p1, p2=None): + test_logger.warning(f'{prefix}: {p1}{f" vs existing {p2}" if p2 else ""}') - def error(logger, prefix, p1, p2=None): + def error(test_logger, prefix, p1, p2=None): nonlocal num_errors num_errors += 1 - warning(logger, prefix, p1, p2) - - filePath = os.path.join(self.work_dir, "producer.log") - with open(filePath, encoding="ascii") as f: + warning(test_logger, prefix, p1, p2) + consumer_log = self.work_dir / "producer.log" + with open(consumer_log, encoding="ascii") as f: re_put = re.compile( r"(?i)" # case insensitive + r" PUT " # PUT @@ -160,11 +160,13 @@ def error(logger, prefix, p1, p2=None): ) if message_index is None: - error(logger, "unexpected ACK guid", ack) + error(test_logger, "unexpected ACK guid", ack) elif puts[message_index] is None: - error(logger, "unexpected ACK payload", ack) + error(test_logger, "unexpected ACK payload", ack) elif acks[message_index]: - error(logger, "duplicate ACK", ack, acks[message_index]) + error( + test_logger, "duplicate ACK", ack, acks[message_index] + ) else: acks[message_index] = ack @@ -173,13 +175,13 @@ def error(logger, prefix, p1, p2=None): else: num_nacks += 1 warning( - logger, + test_logger, f"unsuccessful ({status}) ACK", ack, puts[message_index], ) - logger.info(f"{num_puts} PUTs, {num_acks} acks, {num_nacks} nacks") + test_logger.info(f"{num_puts} PUTs, {num_acks} acks, {num_nacks} nacks") for uri in self.uris: re_push = re.compile( @@ -199,12 +201,12 @@ def error(logger, prefix, p1, p2=None): ) # |GUID| app = uri[1] # client Id - filePath = os.path.join(self.work_dir, f"{app}.log") + consumer_log = self.work_dir / f"{app}.log" consumed = 0 pushes = [None] * NUM_MESSAGES # Push num_confirms = 0 - with open(filePath, encoding="ascii") as f: + with open(consumer_log, encoding="ascii") as f: for line in f: match = re_push.search(line) if match: @@ -218,7 +220,7 @@ def error(logger, prefix, p1, p2=None): if pushes[message_index]: # duplicate PUSH num_duplicates += 1 warning( - logger, + test_logger, f"{app}: duplicate PUSH payload", push, pushes[message_index], @@ -228,12 +230,12 @@ def error(logger, prefix, p1, p2=None): consumed += 1 if puts[message_index] is None: # no corresponding PUT - error(logger, f"{app}: unexpected PUSH", push) + error(test_logger, f"{app}: unexpected PUSH", push) elif acks[message_index] is None: # no corresponding ACK: - error(logger, f"{app}: unexpected PUSH", push) + error(test_logger, f"{app}: unexpected PUSH", push) elif acks[message_index].guid != guid: # ACK GUID mismatch error( - logger, + test_logger, f"{app}: GUID mismatch", push, acks[message_index], @@ -248,14 +250,18 @@ def error(logger, prefix, p1, p2=None): if message_index is None: error( - logger, f"{app}: unexpected CONFIRM guid", confirm + test_logger, + f"{app}: unexpected CONFIRM guid", + confirm, ) elif pushes[message_index] is None: - error(logger, f"{app}: unexpected CONFIRM", confirm) + error( + test_logger, f"{app}: unexpected CONFIRM", confirm + ) else: num_confirms += 1 - logger.info( + test_logger.info( f"{app}: {num_puts} PUTs" f", {num_acks} acks" f", {num_nacks} nacks" @@ -269,10 +275,14 @@ def error(logger, prefix, p1, p2=None): if pushes[message_index] is None: # never received 'message_index' if acks[message_index]: - error(logger, f"{app}: missing message", acks[message_index]) + error( + test_logger, f"{app}: missing message", acks[message_index] + ) num_lost += 1 elif pushes[message_index].index != (message_index - num_lost): - error(logger, f"{app}: out of order PUSH", pushes[message_index]) + error( + test_logger, f"{app}: out of order PUSH", pushes[message_index] + ) assert num_puts == num_acks assert consumed == num_puts @@ -331,7 +341,7 @@ def setup_cluster_fanout(self, cluster): self.set_proxy(cluster) - self.mps = '\'[{"name": "p1", "value": "s1", "type": "E_STRING"}]\'' + self.mps = '[{"name": "p1", "value": "s1", "type": "E_STRING"}]' self.start_producer(0) self.start_consumers(cluster, BACKLOG_MESSAGES) @@ -347,14 +357,14 @@ def setup_cluster_broadcast(self, cluster): ] self.set_proxy(cluster) - self.mps = '\'[{"name": "p1", "value": "s1", "type": "E_STRING"}]\'' + self.mps = '[{"name": "p1", "value": "s1", "type": "E_STRING"}]' self.start_consumers(cluster, 0) self.start_producer(BACKLOG_MESSAGES) def set_proxy(self, cluster): self.cluster = cluster - self.work_dir = str(cluster.work_dir) + self.work_dir = cluster.work_dir proxies = cluster.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) @@ -367,16 +377,30 @@ def capture_number_of_produced_messages(self, producer, timeout=20): return produced def start_producer(self, after=BACKLOG_MESSAGES): - filePath = os.path.join(self.work_dir, "producer.log") - - with contextlib.suppress(FileNotFoundError): - os.unlink(filePath) + producer_log = self.work_dir / "producer.log" + producer_log.unlink(missing_ok=True) self.producer = self.replica_proxy.create_client( "producer", start=False, dump_messages=False, - options=f'--mode auto --queueflags "write,ack" --eventscount {NUM_MESSAGES} --postinterval {POST_INTERVAL_MS} --queueuri {self.uri} --messagepattern "msg%10d|" --log {filePath} --messageProperties={self.mps}', + options=[ + "--mode", + "auto", + "--queueflags", + "write,ack", + "--eventscount", + NUM_MESSAGES, + "--postinterval", + POST_INTERVAL_MS, + "--queueuri", + self.uri, + "--messagepattern", + "msg%10d|", + f"--messageProperties={self.mps}", + "--log", + producer_log, + ], ) # generate BACKLOG_MESSAGES if after > 0: @@ -395,19 +419,33 @@ def start_consumers(self, cluster, after=BACKLOG_MESSAGES): 20, ) - def create_consumer(proxy, uri, work_dir): + def create_consumer(proxy, uri, work_dir: Path): app = uri[1] # uri[1] - client id assert app - filePath = os.path.join(work_dir, f"{app}.log") - with contextlib.suppress(FileNotFoundError): - os.unlink(filePath) + consumer_log = work_dir / f"{app}.log" + consumer_log.unlink(missing_ok=True) return [ proxy.create_client( f"consumer{app}", start=False, dump_messages=False, - options=f'--mode auto --queueflags "read" --eventscount {NUM_MESSAGES} --queueuri {uri[0]} -c --log {filePath} --messageProperties={self.mps} --maxunconfirmed {NUM_MESSAGES}:{NUM_MESSAGES*1024}', + options=[ + "--mode", + "auto", + "--queueflags", + "read", + "--eventscount", + NUM_MESSAGES, + "--queueuri", + uri[0], + "-c", + "--log", + consumer_log, + f"--messageProperties={self.mps}", + "--maxunconfirmed", + f"{NUM_MESSAGES}:{NUM_MESSAGES*1024}", + ], ), 0, # total received 0, # last received (for tracking progress) @@ -424,11 +462,11 @@ def create_consumer(proxy, uri, work_dir): self.leader = cluster.last_known_leader self.active_node = cluster.process(self.replica_proxy.get_active_node()) - def test_shutdown_primary_convert_replica(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_shutdown_primary_convert_replica(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) # make the 'active_node' new primary - for node in standard_cluster.nodes(exclude=self.active_node): + for node in multi_node.nodes(exclude=self.active_node): node.set_quorum(4) self.active_node.drain() @@ -439,12 +477,11 @@ def test_shutdown_primary_convert_replica(self, standard_cluster: Cluster, logge # If shutting down primary, the replica needs to wait for new primary. self.active_node.wait_status(wait_leader=True, wait_ready=False) - self.active_node.capture("is back to healthy state") - self.inspect_results(logger, allow_duplicates=False) + self.inspect_results(allow_duplicates=False) - def test_shutdown_primary_keep_replica(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_shutdown_primary_keep_replica(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) # prevent 'active_node' from becoming new primary self.active_node.set_quorum(4) @@ -457,12 +494,11 @@ def test_shutdown_primary_keep_replica(self, standard_cluster: Cluster, logger): # If shutting down primary, the replica needs to wait for new primary. self.active_node.wait_status(wait_leader=True, wait_ready=False) - self.active_node.capture("is back to healthy state") - self.inspect_results(logger, allow_duplicates=False) + self.inspect_results(allow_duplicates=False) - def test_shutdown_replica(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_shutdown_replica(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) # Start graceful shutdown self.active_node.exit_gracefully() @@ -472,55 +508,54 @@ def test_shutdown_replica(self, standard_cluster: Cluster, logger): # Because the quorum is 3, cluster is still healthy after shutting down # replica. - self.inspect_results(logger, allow_duplicates=False) + self.inspect_results(allow_duplicates=False) - def test_kill_primary_convert_replica(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_kill_primary_convert_replica(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) # make the 'active_node' new primary - for node in standard_cluster.nodes(exclude=self.active_node): + nodes = multi_node.nodes(exclude=self.active_node) + for node in nodes: node.set_quorum(4) self.active_node.drain() - # Start graceful shutdown + # Kill leader. self.leader.force_stop() # If shutting down primary, the replica needs to wait for new primary. self.active_node.wait_status(wait_leader=True, wait_ready=False) - self.active_node.capture("is back to healthy state") - self.inspect_results(logger, allow_duplicates=True) + self.inspect_results(allow_duplicates=True) - def test_kill_primary_keep_replica(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_kill_primary_keep_replica(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) # prevent 'active_node' from becoming new primary self.active_node.set_quorum(4) self.active_node.drain() - # Start graceful shutdown + # Kill leader. self.leader.force_stop() # If shutting down primary, the replica needs to wait for new primary. self.active_node.wait_status(wait_leader=True, wait_ready=False) - self.active_node.capture("is back to healthy state") - self.inspect_results(logger, allow_duplicates=True) + self.inspect_results(allow_duplicates=True) - def test_kill_replica(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_kill_replica(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) # Start graceful shutdown self.active_node.force_stop() # Because the quorum is 3, cluster is still healthy after shutting down # replica. - self.inspect_results(logger, allow_duplicates=True) + self.inspect_results(allow_duplicates=True) @tweak.broker.app_config.network_interfaces.tcp_interface.low_watermark(512) @tweak.broker.app_config.network_interfaces.tcp_interface.high_watermark(1024) - def test_watermarks(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_watermarks(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) producer1 = self.replica_proxy.create_client("producer1") producer1.open(tc.URI_PRIORITY, flags=["write", "ack"], succeed=True) @@ -537,10 +572,10 @@ def test_watermarks(self, standard_cluster: Cluster, logger): assert msgs[0].payload == "msg" assert wait_until(self.has_consumed_all, 120) - self.inspect_results(logger, allow_duplicates=False) + self.inspect_results(allow_duplicates=False) - def test_kill_proxy(self, standard_cluster: Cluster, logger): - self.setup_cluster_fanout(standard_cluster) + def test_kill_proxy(self, multi_node: Cluster): + self.setup_cluster_fanout(multi_node) self.replica_proxy.force_stop() @@ -549,9 +584,9 @@ def test_kill_proxy(self, standard_cluster: Cluster, logger): self.replica_proxy.start() self.replica_proxy.wait_until_started() - self.inspect_results(logger, allow_duplicates=True) + self.inspect_results(allow_duplicates=True) - def test_shutdown_upstream_proxy(self, cartesian_product_cluster: Cluster, logger): + def test_shutdown_upstream_proxy(self, cartesian_product_cluster: Cluster): if not cartesian_product_cluster.virtual_nodes(): # Skip cluster without virtual nodes return @@ -561,4 +596,4 @@ def test_shutdown_upstream_proxy(self, cartesian_product_cluster: Cluster, logge # Shutdown upstream proxy (VR) self.active_node.exit_gracefully() - self.inspect_results(logger, allow_duplicates=True) + self.inspect_results(allow_duplicates=True) diff --git a/src/integration-tests/test_queue_close.py b/src/integration-tests/test_queue_close.py index 14eaba8622..e40acc8cfc 100644 --- a/src/integration-tests/test_queue_close.py +++ b/src/integration-tests/test_queue_close.py @@ -2,27 +2,27 @@ Integration test that tests closing a queue when the broker is down. """ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, - local_cluster, - standard_cluster, + single_node, order, + multi_node, start_cluster, tweak, ) -from bmq.dev.it.process.client import Client +from blazingmq.dev.it.process.client import Client -def test_close_queue(local_cluster: Cluster): - assert local_cluster.is_local +def test_close_queue(single_node: Cluster): + assert single_node.is_single_node # Start a consumer and open a queue - proxies = local_cluster.proxy_cycle() + proxies = single_node.proxy_cycle() consumer = next(proxies).create_client("consumer") consumer.open(tc.URI_PRIORITY, flags=["read"], succeed=True) # Shutdown the broker - leader = local_cluster.last_known_leader + leader = single_node.last_known_leader leader.stop() # Try to close the queue @@ -34,13 +34,13 @@ def test_close_queue(local_cluster: Cluster): @tweak.domain.max_consumers(1) @start_cluster(False) -def test_close_while_reopening(standard_cluster: Cluster): +def test_close_while_reopening(multi_node: Cluster): """ DRQS 169125974. Closing queue while reopen response is pending should not result in a dangling handle. """ - cluster = standard_cluster + cluster = multi_node west1 = cluster.start_node("west1") # make it primary @@ -98,11 +98,11 @@ def test_close_while_reopening(standard_cluster: Cluster): consumer3.open(tc.URI_PRIORITY, flags=["read"], succeed=False) -def test_close_open(standard_cluster: Cluster): +def test_close_open(multi_node: Cluster): """ DRQS 169326671. Close, followed by Open with a different subId. """ - proxies = standard_cluster.proxy_cycle() + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy = next(proxies) @@ -112,7 +112,7 @@ def test_close_open(standard_cluster: Cluster): consumer2 = proxy.create_client("consumer2") consumer2.open(tc.URI_FANOUT_BAR, flags=["read"], succeed=True) - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader consumer3 = leader.create_client("consumer3") consumer3.open(tc.URI_FANOUT_FOO, flags=["read"], succeed=True) @@ -122,14 +122,14 @@ def test_close_open(standard_cluster: Cluster): @tweak.domain.max_consumers(1) @tweak.cluster.queue_operations.reopen_retry_interval_ms(1234) -def test_close_while_retrying_reopen(standard_cluster: Cluster): +def test_close_while_retrying_reopen(multi_node: Cluster): """ DRQS 170043950. Trigger reopen failure causing proxy to retry on timeout. While waiting, close the queue and make sure, the retry accounts for that close. """ - proxies = standard_cluster.proxy_cycle() + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) proxy1 = next(proxies) @@ -142,7 +142,7 @@ def test_close_while_retrying_reopen(standard_cluster: Cluster): producer.open(tc.URI_PRIORITY, flags=["write,ack"], succeed=True) consumer1.open(tc.URI_PRIORITY, flags=["read"], succeed=True) - active_node = standard_cluster.process(proxy1.get_active_node()) + active_node = multi_node.process(proxy1.get_active_node()) proxy1.suspend() # this is to trigger reopen when proxy1 resumes diff --git a/src/integration-tests/test_queue_reopen.py b/src/integration-tests/test_queue_reopen.py index 02404bd341..c1207c3e1b 100644 --- a/src/integration-tests/test_queue_reopen.py +++ b/src/integration-tests/test_queue_reopen.py @@ -2,20 +2,20 @@ Integration tests for queue re-open scenarios. """ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import - Cluster, - standard_cluster, +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import + Cluster, order, + multi_node, ) -from bmq.dev.it.process.client import Client +from blazingmq.dev.it.process.client import Client -def test_reopen_empty_queue(standard_cluster: Cluster): +def test_reopen_empty_queue(multi_node: Cluster): """ If queue has no handles by the time cluster state restores, it should still be notified in order to update its state. """ - proxies = standard_cluster.proxy_cycle() + proxies = multi_node.proxy_cycle() # pick proxy in datacenter opposite to the primary's next(proxies) replica_proxy = next(proxies) @@ -25,7 +25,7 @@ def test_reopen_empty_queue(standard_cluster: Cluster): producer.open(tc.URI_PRIORITY, flags=["write,ack"], succeed=True) # If queue open has succeeded, then active_node is known - active_node = standard_cluster.process(replica_proxy.get_active_node()) + active_node = multi_node.process(replica_proxy.get_active_node()) # Close the queue. The replica keeps (stale) RemoteQueue producer.exit_gracefully() @@ -34,7 +34,7 @@ def test_reopen_empty_queue(standard_cluster: Cluster): active_node.set_quorum(4) # Shutdown the primary - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader leader.stop() # Start a producer and open a queue @@ -48,13 +48,13 @@ def test_reopen_empty_queue(standard_cluster: Cluster): ) -def test_reopen_substream(standard_cluster: Cluster): +def test_reopen_substream(multi_node: Cluster): """ DRQS 169527537. Make a primary's client reopen the same appId with a different subId. """ - leader = standard_cluster.last_known_leader + leader = multi_node.last_known_leader consumer1 = leader.create_client("consumer1") consumer1.open(tc.URI_FANOUT_FOO, flags=["read"], succeed=True) diff --git a/src/integration-tests/test_reconfigure_domains.py b/src/integration-tests/test_reconfigure_domains.py index 69b4ebfc1e..5ee1ea6ed7 100644 --- a/src/integration-tests/test_reconfigure_domains.py +++ b/src/integration-tests/test_reconfigure_domains.py @@ -3,13 +3,16 @@ """ import time -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, - standard_cluster, + order, + multi_node, tweak, ) -from bmq.dev.it.process.client import Client +from blazingmq.dev.it.process.client import Client + +pytestmark = order(6) INITIAL_MSG_QUOTA = 10 @@ -18,7 +21,7 @@ class TestReconfigureDomains: - def setup_cluster(self, cluster): + def setup_cluster(self, cluster: Cluster): proxy = next(cluster.proxy_cycle()) self.writer = proxy.create_client("writers") @@ -38,16 +41,16 @@ def post_n_msgs(self, uri, n): return all(res == Client.e_SUCCESS for res in results) # Helper method which tells 'leader' to reload the domain config. - def reconfigure_to_n_msgs(self, cluster, num_msgs, leader_only=True): - return cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, - {"storage.domain_limits.messages": num_msgs}, - leader_only=leader_only, - succeed=True, + def reconfigure_to_n_msgs(self, cluster: Cluster, num_msgs, leader_only=True): + cluster.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.storage.domain_limits.messages = num_msgs + return cluster.reconfigure_domain( + tc.DOMAIN_PRIORITY, leader_only=leader_only, succeed=True ) @tweak.domain.storage.domain_limits.messages(INITIAL_MSG_QUOTA) - def test_reconfigure_domain_message_limits(self, standard_cluster: Cluster): + def test_reconfigure_domain_message_limits(self, multi_node: Cluster): assert self.post_n_msgs(URI_PRIORITY_1, INITIAL_MSG_QUOTA) # Resource monitor allows exceeding message quota exactly once before @@ -63,7 +66,7 @@ def test_reconfigure_domain_message_limits(self, standard_cluster: Cluster): assert not self.post_n_msgs(URI_PRIORITY_2, 1) # Modify the domain configuration to hold 2 more messages. - self.reconfigure_to_n_msgs(standard_cluster, INITIAL_MSG_QUOTA + 10) + self.reconfigure_to_n_msgs(multi_node, INITIAL_MSG_QUOTA + 10) # Observe that posting two more messages succeeds. assert self.post_n_msgs(URI_PRIORITY_1, 5) @@ -75,7 +78,7 @@ def test_reconfigure_domain_message_limits(self, standard_cluster: Cluster): assert not self.post_n_msgs(URI_PRIORITY_2, 1) # Reconfigure limit back down to the initial value. - self.reconfigure_to_n_msgs(standard_cluster, INITIAL_MSG_QUOTA) + self.reconfigure_to_n_msgs(multi_node, INITIAL_MSG_QUOTA) # Observe that posting continues to fail. assert not self.post_n_msgs(URI_PRIORITY_1, 1) @@ -99,7 +102,7 @@ def test_reconfigure_domain_message_limits(self, standard_cluster: Cluster): assert self.post_n_msgs(URI_PRIORITY_2, 1) @tweak.domain.storage.queue_limits.messages(INITIAL_MSG_QUOTA) - def test_reconfigure_queue_message_limits(self, standard_cluster: Cluster): + def test_reconfigure_queue_message_limits(self, multi_node: Cluster): # Resource monitor allows exceeding message quota exactly once before # beginning to fail ACKs. So we expect 'INITIAL_MSG_QUOTA+1' messages # to succeed without error. @@ -115,11 +118,10 @@ def test_reconfigure_queue_message_limits(self, standard_cluster: Cluster): assert not self.post_n_msgs(URI_PRIORITY_2, 1) # Modify the domain configuration to hold 2 more messages per queue. - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, - {"storage.queue_limits.messages": INITIAL_MSG_QUOTA + 1}, - succeed=True, - ) + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.storage.queue_limits.messages = (INITIAL_MSG_QUOTA + 1) + multi_node.reconfigure_domain(tc.DOMAIN_PRIORITY, succeed=True) # Observe that posting one more message now succeeds for each queue. assert self.post_n_msgs(URI_PRIORITY_1, 1) @@ -130,15 +132,15 @@ def test_reconfigure_queue_message_limits(self, standard_cluster: Cluster): assert not self.post_n_msgs(URI_PRIORITY_2, 1) @tweak.domain.storage.domain_limits.messages(1) - def test_reconfigure_with_leader_change(self, standard_cluster: Cluster): - leader = standard_cluster.last_known_leader + def test_reconfigure_with_leader_change(self, multi_node: Cluster): + leader = multi_node.last_known_leader # Exhaust the message capacity of the domain. assert self.post_n_msgs(URI_PRIORITY_1, 2) assert not self.post_n_msgs(URI_PRIORITY_1, 1) # Reconfigure every node to accept an additional message. - self.reconfigure_to_n_msgs(standard_cluster, 3, leader_only=False) + self.reconfigure_to_n_msgs(multi_node, 3, leader_only=False) # Ensure the capacity increased as expected, then confirm one message. assert self.post_n_msgs(URI_PRIORITY_1, 2) @@ -151,9 +153,9 @@ def test_reconfigure_with_leader_change(self, standard_cluster: Cluster): leader.wait() # Wait for a new leader to be elected. - standard_cluster.wait_leader() - assert leader != standard_cluster.last_known_leader - leader = standard_cluster.last_known_leader + multi_node.wait_leader() + assert leader != multi_node.last_known_leader + leader = multi_node.last_known_leader # Verify that new leader accepts one more message and reaches capacity. assert self.post_n_msgs(URI_PRIORITY_1, 1) @@ -161,8 +163,8 @@ def test_reconfigure_with_leader_change(self, standard_cluster: Cluster): @tweak.domain.max_consumers(1) @tweak.domain.max_producers(1) - def test_reconfigure_max_clients(self, standard_cluster: Cluster): - proxy = next(standard_cluster.proxy_cycle()) + def test_reconfigure_max_clients(self, multi_node: Cluster): + proxy = next(multi_node.proxy_cycle()) # Create another client. ad_client = proxy.create_client("another-client") @@ -172,11 +174,11 @@ def test_reconfigure_max_clients(self, standard_cluster: Cluster): assert ad_client.open(URI_PRIORITY_2, flags=["read"], block=True) != 0 # Reconfigure the domain to allow for one more producer to connect. - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, - {"max_producers": 2}, - leader_only=True, - succeed=True, + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.max_producers = 2 + multi_node.reconfigure_domain( + tc.DOMAIN_PRIORITY, leader_only=True, succeed=True ) # Confirm that the queue can be opened for writing, but not reading. @@ -184,11 +186,11 @@ def test_reconfigure_max_clients(self, standard_cluster: Cluster): assert ad_client.open(URI_PRIORITY_2, flags=["read"], block=True) != 0 # Reconfigure the domain to allow for one more consumer to connect. - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, - {"max_consumers": 2}, - leader_only=True, - succeed=True, + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.max_consumers = 2 + multi_node.reconfigure_domain( + tc.DOMAIN_PRIORITY, leader_only=True, succeed=True ) # Confirm that the queue can be opened for reading. @@ -200,7 +202,7 @@ def test_reconfigure_max_clients(self, standard_cluster: Cluster): assert ad_client.open(URI_PRIORITY_2, flags=["read"], block=True) != 0 @tweak.domain.max_queues(2) - def test_reconfigure_max_queues(self, standard_cluster: Cluster): + def test_reconfigure_max_queues(self, multi_node: Cluster): ad_url_1 = f"{URI_PRIORITY_1}-third-queue" ad_url_2 = f"{URI_PRIORITY_1}-fourth-queue" @@ -208,11 +210,11 @@ def test_reconfigure_max_queues(self, standard_cluster: Cluster): assert self.reader.open(ad_url_1, flags=["read"], block=True) != 0 # Reconfigure the domain to allow for one more producer to connect. - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, - {"max_queues": 3}, - leader_only=True, - succeed=True, + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.max_queues = 3 + multi_node.reconfigure_domain( + tc.DOMAIN_PRIORITY, leader_only=True, succeed=True ) # Confirm that one more queue can be opened. @@ -223,8 +225,8 @@ def test_reconfigure_max_queues(self, standard_cluster: Cluster): @tweak.cluster.queue_operations.consumption_monitor_period_ms(500) @tweak.domain.max_idle_time(1) - def test_reconfigure_max_idle_time(self, standard_cluster: Cluster): - leader = standard_cluster.last_known_leader + def test_reconfigure_max_idle_time(self, multi_node: Cluster): + leader = multi_node.last_known_leader # Configure reader to have at most one outstanding unconfirmed message. self.reader.configure(URI_PRIORITY_1, block=True, maxUnconfirmedMessages=1) @@ -240,11 +242,11 @@ def test_reconfigure_max_idle_time(self, standard_cluster: Cluster): self.reader.confirm(URI_PRIORITY_1, "+2", succeed=True) # Reconfigure domain to tolerate as much as two seconds of idleness. - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, - {"max_idle_time": 2}, - leader_only=True, - succeed=True, + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.max_idle_time = 2 + multi_node.reconfigure_domain( + tc.DOMAIN_PRIORITY, leader_only=True, succeed=True ) # Write two further messages to the queue. @@ -260,8 +262,8 @@ def test_reconfigure_max_idle_time(self, standard_cluster: Cluster): assert not leader.alarms("QUEUE_CONSUMER_MONITOR", 1) @tweak.domain.message_ttl(1) - def test_reconfigure_message_ttl(self, standard_cluster: Cluster): - leader = standard_cluster.last_known_leader + def test_reconfigure_message_ttl(self, multi_node: Cluster): + leader = multi_node.last_known_leader # Write two messages to the queue (only one can be sent to reader). assert self.post_n_msgs(URI_PRIORITY_1, 2) @@ -273,11 +275,11 @@ def test_reconfigure_message_ttl(self, standard_cluster: Cluster): assert leader.erases_messages(URI_PRIORITY_1, msgs=2, timeout=1) # Reconfigure the domain to wait 3 seconds before GC'ing messages. - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, - {"message_ttl": 10}, - leader_only=True, - succeed=True, + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.message_ttl = 10 + multi_node.reconfigure_domain( + tc.DOMAIN_PRIORITY, leader_only=True, succeed=True ) # Write two further messages to the queue. @@ -293,9 +295,9 @@ def test_reconfigure_message_ttl(self, standard_cluster: Cluster): self.reader.confirm(URI_PRIORITY_1, "+2", succeed=True) @tweak.domain.max_delivery_attempts(0) - def test_reconfigure_max_delivery_attempts(self, standard_cluster: Cluster): + def test_reconfigure_max_delivery_attempts(self, multi_node: Cluster): URI = f"bmq://{tc.DOMAIN_PRIORITY}/my-queue" - proxy = next(standard_cluster.proxy_cycle()) + proxy = next(multi_node.proxy_cycle()) # Open the queue through the writer. self.writer.open(URI, flags=["write,ack"], succeed=True) @@ -327,9 +329,10 @@ def do_test(expect_success): do_test(True) # Reconfigure messages to expire after 5 delivery attempts. - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, {"max_delivery_attempts": 5}, succeed=True - ) + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.max_delivery_attempts = 5 + multi_node.reconfigure_domain(tc.DOMAIN_PRIORITY, succeed=True) # Expect that message will expire after failed deliveries. do_test(False) diff --git a/src/integration-tests/test_reopen_queue_failure.py b/src/integration-tests/test_reopen_queue_failure.py index 28f12f5901..8817a87968 100644 --- a/src/integration-tests/test_reopen_queue_failure.py +++ b/src/integration-tests/test_reopen_queue_failure.py @@ -5,12 +5,15 @@ the fix. """ -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster -from bmq.dev.it.fixtures import ( - standard_cluster as cluster, # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import + Cluster, + multi_node as cluster, + order, ) +pytestmark = order(6) + def test_reopen_queue_failure(cluster: Cluster): proxies = cluster.proxy_cycle() diff --git a/src/integration-tests/test_restart.py b/src/integration-tests/test_restart.py index c78347a9f5..6613d79041 100644 --- a/src/integration-tests/test_restart.py +++ b/src/integration-tests/test_restart.py @@ -7,10 +7,12 @@ import re import time -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import Cluster, cluster # pylint: disable=unused-import -from bmq.dev.it.process.client import Client -from bmq.dev.it.util import attempt, wait_until +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import Cluster, cluster, order # pylint: disable=unused-import +from blazingmq.dev.it.process.client import Client +from blazingmq.dev.it.util import attempt, wait_until + +pytestmark = order(2) def test_basic(cluster: Cluster): @@ -44,7 +46,7 @@ def test_basic(cluster: Cluster): cluster.restart_nodes() # For a standard cluster, states have already been restored as part of # leader re-election. - if cluster.is_local: + if cluster.is_single_node: producer.wait_state_restored() producer.post(tc.URI_PRIORITY, payload=["msg2"], wait_ack=True, succeed=True) diff --git a/src/integration-tests/test_strong_consistency.py b/src/integration-tests/test_strong_consistency.py index 144f4bfa17..71198cba34 100644 --- a/src/integration-tests/test_strong_consistency.py +++ b/src/integration-tests/test_strong_consistency.py @@ -1,14 +1,17 @@ import contextlib -import bmq.dev.it.testconstants as tc -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +import blazingmq.dev.it.testconstants as tc +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, - local_cluster, - standard_cluster, + single_node, + order, + multi_node, tweak, ) -from bmq.dev.it.util import wait_until -from bmq.schemas import mqbconf +from blazingmq.dev.it.util import wait_until +from blazingmq.schemas import mqbconf + +pytestmark = order(5) class Suspender: @@ -64,8 +67,8 @@ def setup_cluster(self, cluster): for uri in [tc.URI_FANOUT_SC_FOO, tc.URI_FANOUT_SC_BAR, tc.URI_FANOUT_SC_BAZ]: self.consumer.open(uri, flags=["read"], succeed=True) - def _break_post_unbreak(self, standard_cluster, breaker, has_timeout): - cluster = standard_cluster + def _break_post_unbreak(self, multi_node, breaker, has_timeout): + cluster = multi_node leader = cluster.last_known_leader active_node = cluster.process(self.replica_proxy.get_active_node()) @@ -120,14 +123,14 @@ def _break_post_unbreak(self, standard_cluster, breaker, has_timeout): lambda: len(self.consumer.list(uri, block=True)) == 1, 2 ) - def test_suspend_post_resume(self, standard_cluster: Cluster): - self._break_post_unbreak(standard_cluster, Suspender, False) + def test_suspend_post_resume(self, multi_node: Cluster): + self._break_post_unbreak(multi_node, Suspender, False) - def test_kill_post_start(self, standard_cluster: Cluster): - self._break_post_unbreak(standard_cluster, Killer, False) + def test_kill_post_start(self, multi_node: Cluster): + self._break_post_unbreak(multi_node, Killer, False) def test_strong_consistency_local( - self, local_cluster # pylint: disable=unused-argument + self, single_node # pylint: disable=unused-argument ): # post SC self.producer.post( @@ -141,19 +144,19 @@ def test_strong_consistency_local( assert wait_until(lambda: len(self.consumer.list(uri, block=True)) == 1, 2) @tweak.domain.deduplication_time_ms(1000) - def test_timeout_receipt(self, standard_cluster: Cluster): - self._break_post_unbreak(standard_cluster, Suspender, True) + def test_timeout_receipt(self, multi_node: Cluster): + self._break_post_unbreak(multi_node, Suspender, True) - def test_dynamic_replication_factor(self, standard_cluster: Cluster): - leader = standard_cluster.last_known_leader + def test_dynamic_replication_factor(self, multi_node: Cluster): + leader = multi_node.last_known_leader # Exercise the LIST_TUNABLES command to ensure it executes safely. leader.command("CLUSTERS CLUSTER itCluster STORAGE REPLICATION LIST_TUNABLES") with contextlib.ExitStack() as stack: # Suspend all nodes that are not the leader or active. - active_node = standard_cluster.process(self.replica_proxy.get_active_node()) - for node in standard_cluster.nodes(exclude=[leader, active_node]): + active_node = multi_node.process(self.replica_proxy.get_active_node()) + for node in multi_node.nodes(exclude=[leader, active_node]): stack.enter_context(Suspender(node)) # Default replication-factor is 3. Since only two nodes are active, no @@ -185,8 +188,8 @@ def test_dynamic_replication_factor(self, standard_cluster: Cluster): lambda: len(self.consumer.list(uri, block=True)) == 2, 2 ) - def test_change_consistency(self, standard_cluster: Cluster): - leader = standard_cluster.last_known_leader + def test_change_consistency(self, multi_node: Cluster): + leader = multi_node.last_known_leader assert leader # Open priority queue. @@ -194,10 +197,10 @@ def test_change_consistency(self, standard_cluster: Cluster): self.producer.open(tc.URI_PRIORITY, flags=["write,ack"], succeed=True) # Build list of nodes to be suspended. - suspended_nodes = standard_cluster.nodes( + suspended_nodes = multi_node.nodes( exclude=[ leader, - standard_cluster.process(self.replica_proxy.get_active_node()), + multi_node.process(self.replica_proxy.get_active_node()), ] ) @@ -216,19 +219,17 @@ def test_change_consistency(self, standard_cluster: Cluster): ) assert self.consumer.wait_push_event() - for broker in standard_cluster.workspace.brokers.values(): - broker.domains[ - tc.DOMAIN_PRIORITY - ].definition.parameters.consistency = mqbconf.Consistency( # type: ignore - strong=mqbconf.QueueConsistencyStrong() - ) - - standard_cluster.reconfigure_domain_values( - tc.DOMAIN_PRIORITY, {}, write_only=True + multi_node.config.domains[ + tc.DOMAIN_PRIORITY + ].definition.parameters.consistency = mqbconf.Consistency( # type: ignore + strong=mqbconf.QueueConsistencyStrong() + ) + multi_node.reconfigure_domain( + tc.DOMAIN_PRIORITY, write_only=True ) # Reconfigure domain to be strongly consistent. - for node in standard_cluster.nodes(exclude=suspended_nodes): + for node in multi_node.nodes(exclude=suspended_nodes): node.reconfigure_domain(tc.DOMAIN_PRIORITY, succeed=True) # Require messages to be written to three machines before push. diff --git a/src/integration-tests/test_subscriptions.py b/src/integration-tests/test_subscriptions.py index 97fa6b8595..a678bfd377 100644 --- a/src/integration-tests/test_subscriptions.py +++ b/src/integration-tests/test_subscriptions.py @@ -51,19 +51,22 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Tuple, Union -import bmq.dev.it.testconstants as tc +import blazingmq.dev.it.testconstants as tc import pytest -from bmq.dev.it.fixtures import ( # pylint: disable=unused-import +from blazingmq.dev.it.fixtures import ( # pylint: disable=unused-import Cluster, cluster, - logger, - standard_cluster, + test_logger, + order, + multi_node, tweak, ) -from bmq.dev.it.process.broker import Broker -from bmq.dev.it.process.client import Client, ITError, Message -from bmq.dev.it.util import wait_until -from bmq.schemas import mqbcfg +from blazingmq.dev.it.process.broker import Broker +from blazingmq.dev.it.process.client import Client, ITError, Message +from blazingmq.dev.it.util import wait_until +from blazingmq.schemas import mqbcfg + +pytestmark = order(7) EMPTY_SUBSCRIPTION = [] EPS = 1e-6 @@ -444,1404 +447,1426 @@ def _build_subscriptions( return res -class TestSubscriptions: - def test_second_configure(self, cluster: Cluster): - """ - Test: a simple scenario where a crash might occur. - - Create 1 producer / 1 consumer. - - Open 1 priority queue. - - Configure the only consumer. - - Produce 1 simple message to the queue. - - Receive the message and confirm it by consumer. - - Configure the only consumer again. - All these steps do not include any specified subscription or message - properties. - - Concerns: - - This scenario failed on the early development stage. - """ - proxy = next(cluster.proxy_cycle()) - producer = proxy.create_client("producer") - consumer = proxy.create_client("consumer") - - uri = tc.URI_PRIORITY - producer.open(uri, flags=["write,ack"], succeed=True) - consumer.open(uri, flags=["read"], succeed=True) - - consumer.configure(uri, block=True) - - assert ( - producer.post(uri, payload=["payload"], wait_ack=True) == Client.e_SUCCESS - ) - consumer.wait_push_event() - assert consumer.confirm(uri, "*", block=True) == Client.e_SUCCESS - - consumer.configure(uri, block=True) - - def test_open(self, cluster: Cluster): - """ - Test: open queue with the specified subscription. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, with subscription parameters for the consumer. - - Producer: produce messages M1-M3 with properties. - - Consumer: expect and confirm only M1-M2 in correct order. - - Concerns: - - Subscription expression must be passed with 'open' correctly. - - Only the messages expected by subscription must be received. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer = Consumer(cluster, uri, ["x <= 1"]) - - producer.post(x=0) - producer.post(x=1) - producer.post(x=2) - - consumer.expect_messages(["x: 0", "x: 1"], confirm=True) - - def test_non_blocking(self, cluster: Cluster): - """ - Test: messages that must not be received by the given consumer do not - block it to receive the following messages. - - Create 1 producer / 1 consumer: C1. - - Open 1 priority queue, with subscription parameters for the consumer C1. - - Producer: produce 3 messages M1-M3 with properties. - - Consumer C1: expect and confirm only messages M2-M3 (M1 is not - suitable for subscription). - - Create another consumer C2 and open the same priority queue, with - subscription expression that is always true. - - Consumer C2: expect and confirm the only remaining message M1. - - Producer -> M1, M2, M3 - Consumer C1 <- M2, M3 - M1 is not routed to C1 because it is not suitable for its subscription - Consumer C2 <- M1 - the remaining M1 is routed to C2 - - Concerns: - - The first message M1 must not block consumer C1 to receive messages - M2, M3. - - Consumer C2 must receive the skipped message M1. - - Consumer C2 must not receive messages M2, M3 that were confirmed. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer1 = Consumer(cluster, uri, ["x >= 1"]) - - producer.post(x=0) - producer.post(x=1) - producer.post(x=2) - - consumer1.expect_messages(["x: 1", "x: 2"], confirm=True) - - consumer2 = Consumer( - cluster, - uri, - ["x >= 0 || x < 0"], - ) - consumer2.expect_messages(["x: 0"], confirm=True) - - def test_configure_subscription(self, cluster: Cluster): - """ - Test: configure queue with the specified subscription. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, without subscription parameters. - - Configure queue for consumer with subscription parameters. - - Producer: produce messages M1-M3 with properties. - - Consumer: expect and confirm only M1-M2 in correct order. - - Concerns: - - Subscription expression must be passed with 'configure' correctly. - - Only the messages expected by subscription must be received. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer = Consumer(cluster, uri) - - consumer.configure(["x <= 1"]) - - producer.post(x=0) - producer.post(x=1) - producer.post(x=2) - - consumer.expect_messages(["x: 0", "x: 1"], confirm=True) - - def test_reconfigure_subscription(self, cluster: Cluster): - """ - Test: reconfigure queue with the specified subscription. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, with subscription parameters for the consumer. - - Produce 6 messages with properties to the queue: M1-M6. - - Consumer: expect and confirm only the last 2 messages M5-M6. - - - Consumer: configure queue with another subscription parameters. - - Consumer: expect and confirm the current last 2 messages M3-M4. - - - Consumer: configure queue with another subscription parameters. - - Consumer: expect and confirm the remaining last 2 messages M1-M2. - - Producer -> [M1, M2, M3, M4, M5, M6] - Consumer - open <- [M5, M6] - configure <- [M3, M4] - configure <- [M1, M2] - - Concerns: - - Subscription expressions are replaced correctly each time with - 'configure'. - - Only the messages expected by each subscription must be received. - - Confirmed messages are not being received again. - - Valid substreams: the order of messages is the same as it was during - the 'post' for each subscription. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer = Consumer(cluster, uri, ["x >= 4"]) - - producer.post(x=0) - producer.post(x=1) - - producer.post(x=2) - m3 = producer.post(x=3) # pylint: disable=unused-variable - - producer.post(x=4) - producer.post(x=5) - - consumer.expect_messages(["x: 4", "x: 5"], confirm=True) - - consumer.configure(["x >= 2"]) - consumer.expect_messages(["x: 2", "x: 3"], confirm=True) - - consumer.configure(["x >= 0"]) - consumer.expect_messages(["x: 0", "x: 1"], confirm=True) - - def test_non_overlapping(self, cluster: Cluster): - """ - Test: several non-overlapping subscriptions to the same queue. - - Create 1 producer / 3 consumers. - - Open 1 priority queue, each consumer has its own subscription - expression, these expressions are strictly non-overlapping. - - Produce N messages with properties to the queue. - - Expect and confirm only the messages that are good for the given - subscription expression for each consumer. - - Concerns: - - Routing of messages works correctly for multiple consumers with - subscriptions. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer1 = Consumer(cluster, uri, ["x > 0"]) - consumer2 = Consumer(cluster, uri, ["x < 0"]) - consumer3 = Consumer(cluster, uri, ["x == 0"]) - - values = [-1, 9, 0, 5, 678, 54, 17, -9, 0, -4, 3, -2, 10, 2, 11, -5] - expected = [producer.post(x=num) for num in values] - - expected1 = [payload for num, payload in zip(values, expected) if num > 0] - expected2 = [payload for num, payload in zip(values, expected) if num < 0] - expected3 = [payload for num, payload in zip(values, expected) if num == 0] - - consumer1.expect_messages(expected1, confirm=True) - consumer2.expect_messages(expected2, confirm=True) - consumer3.expect_messages(expected3, confirm=True) - - def test_priorities(self, cluster: Cluster): - """ - Test: several subscriptions to the same queue with different priorities. - - Create 1 producer / 2 consumers: C1, C2. - - Open 1 priority queue, the subscription expression is the same for - each consumer but priorities are different. - - Produce N messages with properties to the queue. - - - Expect and confirm only the messages that are suitable for the given - subscription expression for each consumer. - - Concerns: - - Routing of messages works correctly for multiple consumers with - subscriptions. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer1 = Consumer(cluster, uri) - consumer2 = Consumer(cluster, uri) - - consumer1.configure( - [("x >= 0", 10)], - ) - consumer2.configure( - [("x >= 0", 5)], - ) - - expected = producer.post_diff(num=5, offset=0) - consumer1.expect_messages(expected, confirm=True) - - consumer1.configure( - [("x >= 0", 1)], - ) - - expected = producer.post_diff(num=5, offset=100) - consumer2.expect_messages(expected, confirm=True) - - def test_fanout(self, cluster: Cluster): - """ - Test: subscriptions work in fanout mode. - - Create 1 producer / 3 consumers. - - Open 1 fanout queue, the subscription expression is the same for - each consumer but app uris are different. - - Produce N messages with properties to the queue. - - Expect all N messages for each consumer and confirm these. - - Concerns: - - Fanout works for subscriptions: every message delivered for each - consumer. - """ - producer_uri = tc.URI_FANOUT - app_uris = [tc.URI_FANOUT_FOO, tc.URI_FANOUT_BAR, tc.URI_FANOUT_BAZ] - - producer = Producer(cluster, producer_uri) - consumers = [ - Consumer( - cluster, - uri, - ["x >= 0"], - ) - for uri in app_uris - ] +def test_second_configure(cluster: Cluster): + """ + Test: a simple scenario where a crash might occur. + - Create 1 producer / 1 consumer. + - Open 1 priority queue. + - Configure the only consumer. + - Produce 1 simple message to the queue. + - Receive the message and confirm it by consumer. + - Configure the only consumer again. + All these steps do not include any specified subscription or message + properties. + + Concerns: + - This scenario failed on the early development stage. + """ + proxy = next(cluster.proxy_cycle()) + producer = proxy.create_client("producer") + consumer = proxy.create_client("consumer") + + uri = tc.URI_PRIORITY + producer.open(uri, flags=["write,ack"], succeed=True) + consumer.open(uri, flags=["read"], succeed=True) + + consumer.configure(uri, block=True) + + assert producer.post(uri, payload=["payload"], wait_ack=True) == Client.e_SUCCESS + consumer.wait_push_event() + assert consumer.confirm(uri, "*", block=True) == Client.e_SUCCESS + + consumer.configure(uri, block=True) + + +def test_open(cluster: Cluster): + """ + Test: open queue with the specified subscription. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, with subscription parameters for the consumer. + - Producer: produce messages M1-M3 with properties. + - Consumer: expect and confirm only M1-M2 in correct order. + + Concerns: + - Subscription expression must be passed with 'open' correctly. + - Only the messages expected by subscription must be received. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer = Consumer(cluster, uri, ["x <= 1"]) + + producer.post(x=0) + producer.post(x=1) + producer.post(x=2) + + consumer.expect_messages(["x: 0", "x: 1"], confirm=True) + + +def test_non_blocking(cluster: Cluster): + """ + Test: messages that must not be received by the given consumer do not + block it to receive the following messages. + - Create 1 producer / 1 consumer: C1. + - Open 1 priority queue, with subscription parameters for the consumer C1. + - Producer: produce 3 messages M1-M3 with properties. + - Consumer C1: expect and confirm only messages M2-M3 (M1 is not + suitable for subscription). + - Create another consumer C2 and open the same priority queue, with + subscription expression that is always true. + - Consumer C2: expect and confirm the only remaining message M1. + + Producer -> M1, M2, M3 + Consumer C1 <- M2, M3 + M1 is not routed to C1 because it is not suitable for its subscription + Consumer C2 <- M1 + the remaining M1 is routed to C2 + + Concerns: + - The first message M1 must not block consumer C1 to receive messages + M2, M3. + - Consumer C2 must receive the skipped message M1. + - Consumer C2 must not receive messages M2, M3 that were confirmed. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer1 = Consumer(cluster, uri, ["x >= 1"]) + + producer.post(x=0) + producer.post(x=1) + producer.post(x=2) + + consumer1.expect_messages(["x: 1", "x: 2"], confirm=True) + + consumer2 = Consumer( + cluster, + uri, + ["x >= 0 || x < 0"], + ) + consumer2.expect_messages(["x: 0"], confirm=True) + + +def test_configure_subscription(cluster: Cluster): + """ + Test: configure queue with the specified subscription. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, without subscription parameters. + - Configure queue for consumer with subscription parameters. + - Producer: produce messages M1-M3 with properties. + - Consumer: expect and confirm only M1-M2 in correct order. + + Concerns: + - Subscription expression must be passed with 'configure' correctly. + - Only the messages expected by subscription must be received. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer = Consumer(cluster, uri) + + consumer.configure(["x <= 1"]) + + producer.post(x=0) + producer.post(x=1) + producer.post(x=2) + + consumer.expect_messages(["x: 0", "x: 1"], confirm=True) + + +def test_reconfigure_subscription(cluster: Cluster): + """ + Test: reconfigure queue with the specified subscription. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, with subscription parameters for the consumer. + - Produce 6 messages with properties to the queue: M1-M6. + - Consumer: expect and confirm only the last 2 messages M5-M6. + + - Consumer: configure queue with another subscription parameters. + - Consumer: expect and confirm the current last 2 messages M3-M4. + + - Consumer: configure queue with another subscription parameters. + - Consumer: expect and confirm the remaining last 2 messages M1-M2. + + Producer -> [M1, M2, M3, M4, M5, M6] + Consumer + open <- [M5, M6] + configure <- [M3, M4] + configure <- [M1, M2] + + Concerns: + - Subscription expressions are replaced correctly each time with + 'configure'. + - Only the messages expected by each subscription must be received. + - Confirmed messages are not being received again. + - Valid substreams: the order of messages is the same as it was during + the 'post' for each subscription. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer = Consumer(cluster, uri, ["x >= 4"]) + + producer.post(x=0) + producer.post(x=1) + + producer.post(x=2) + m3 = producer.post(x=3) # pylint: disable=unused-variable + + producer.post(x=4) + producer.post(x=5) + + consumer.expect_messages(["x: 4", "x: 5"], confirm=True) + + consumer.configure(["x >= 2"]) + consumer.expect_messages(["x: 2", "x: 3"], confirm=True) + + consumer.configure(["x >= 0"]) + consumer.expect_messages(["x: 0", "x: 1"], confirm=True) + + +def test_non_overlapping(cluster: Cluster): + """ + Test: several non-overlapping subscriptions to the same queue. + - Create 1 producer / 3 consumers. + - Open 1 priority queue, each consumer has its own subscription + expression, these expressions are strictly non-overlapping. + - Produce N messages with properties to the queue. + - Expect and confirm only the messages that are good for the given + subscription expression for each consumer. + + Concerns: + - Routing of messages works correctly for multiple consumers with + subscriptions. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer1 = Consumer(cluster, uri, ["x > 0"]) + consumer2 = Consumer(cluster, uri, ["x < 0"]) + consumer3 = Consumer(cluster, uri, ["x == 0"]) + + values = [-1, 9, 0, 5, 678, 54, 17, -9, 0, -4, 3, -2, 10, 2, 11, -5] + expected = [producer.post(x=num) for num in values] + + expected1 = [payload for num, payload in zip(values, expected) if num > 0] + expected2 = [payload for num, payload in zip(values, expected) if num < 0] + expected3 = [payload for num, payload in zip(values, expected) if num == 0] + + consumer1.expect_messages(expected1, confirm=True) + consumer2.expect_messages(expected2, confirm=True) + consumer3.expect_messages(expected3, confirm=True) + + +def test_priorities(cluster: Cluster): + """ + Test: several subscriptions to the same queue with different priorities. + - Create 1 producer / 2 consumers: C1, C2. + - Open 1 priority queue, the subscription expression is the same for + each consumer but priorities are different. + - Produce N messages with properties to the queue. + + - Expect and confirm only the messages that are suitable for the given + subscription expression for each consumer. + + Concerns: + - Routing of messages works correctly for multiple consumers with + subscriptions. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer1 = Consumer(cluster, uri) + consumer2 = Consumer(cluster, uri) + + consumer1.configure( + [("x >= 0", 10)], + ) + consumer2.configure( + [("x >= 0", 5)], + ) - expected = producer.post_diff(num=5) - for consumer in consumers: - consumer.expect_messages(expected, confirm=True) + expected = producer.post_diff(num=5, offset=0) + consumer1.expect_messages(expected, confirm=True) - def test_round_robin(self, cluster: Cluster): - """ - Test: round-robin routing of messages with subscriptions. - - Create 1 producer / K consumers. - - Open 1 priority queue, the subscription expression is the same for - each consumer. - - Produce K*N messages with properties to the queue. - - Expect and confirm strictly N messages for each consumer. - - Concerns: - - Round-robin works for subscriptions: messages are evenly routed - between consumers. - """ - uri = tc.URI_PRIORITY - num_consumers = 10 - messages_per_consumer = 15 - - producer = Producer(cluster, uri) - consumers = [ - Consumer( - cluster, - uri, - ["x >= 0"], - ) - for _ in range(num_consumers) - ] - - # Note: we don't know the exact order of consumers for round-robin, - # this could change from launch to launch. Because of that it's not - # trivial to expect concrete messages for each consumer if all messages - # are different. To deal with this we generate messages in groups of - # size K, messages in one group have the same properties and payloads. - # So it doesn't matter in what order it will be routed with round-robin. - # producer > 0 0 0 1 1 1 2 2 2 3 3 3 - # consumer1 < 0 1 2 3 - # consumer2 < 0 1 2 3 - # consumer3 < 0 1 2 3 - expected = [] - for i in range(messages_per_consumer): - posted_payloads = producer.post_same(num=num_consumers, value=i) - # Posted payloads on specific step are equal, need to memorize just - # one sample from each group. - expected.append(posted_payloads[0]) - - for consumer in consumers: - consumer.expect_messages(expected, confirm=True) + consumer1.configure( + [("x >= 0", 1)], + ) - def test_redelivery(self, cluster: Cluster): - """ - Test: message redelivery when the consumer exits without confirmation. - - Create 1 producer / 1 consumer: C1. - - Open 1 priority queue, with subscription parameters. - - Producer: produce N messages with properties. - - Consumer C1: expect but not confirm all N messages. - - Consumer C1: close. - - Open another consumer C2 with the same parameters. - - Consumer C2: expect and confirm all N messages. - - Concerns: - - Unconfirmed messages must be redelivered correctly. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer = Consumer( + expected = producer.post_diff(num=5, offset=100) + consumer2.expect_messages(expected, confirm=True) + + +def test_fanout(cluster: Cluster): + """ + Test: subscriptions work in fanout mode. + - Create 1 producer / 3 consumers. + - Open 1 fanout queue, the subscription expression is the same for + each consumer but app uris are different. + - Produce N messages with properties to the queue. + - Expect all N messages for each consumer and confirm these. + + Concerns: + - Fanout works for subscriptions: every message delivered for each + consumer. + """ + producer_uri = tc.URI_FANOUT + app_uris = [tc.URI_FANOUT_FOO, tc.URI_FANOUT_BAR, tc.URI_FANOUT_BAZ] + + producer = Producer(cluster, producer_uri) + consumers = [ + Consumer( cluster, uri, ["x >= 0"], ) + for uri in app_uris + ] - expected = producer.post_diff(num=5) - consumer.expect_messages(expected, confirm=False) - # No confirm + expected = producer.post_diff(num=5) + for consumer in consumers: + consumer.expect_messages(expected, confirm=True) - consumer.close() - consumer = Consumer( +def test_round_robin(cluster: Cluster): + """ + Test: round-robin routing of messages with subscriptions. + - Create 1 producer / K consumers. + - Open 1 priority queue, the subscription expression is the same for + each consumer. + - Produce K*N messages with properties to the queue. + - Expect and confirm strictly N messages for each consumer. + + Concerns: + - Round-robin works for subscriptions: messages are evenly routed + between consumers. + """ + uri = tc.URI_PRIORITY + num_consumers = 10 + messages_per_consumer = 15 + + producer = Producer(cluster, uri) + consumers = [ + Consumer( cluster, uri, ["x >= 0"], ) + for _ in range(num_consumers) + ] + + # Note: we don't know the exact order of consumers for round-robin, + # this could change from launch to launch. Because of that it's not + # trivial to expect concrete messages for each consumer if all messages + # are different. To deal with this we generate messages in groups of + # size K, messages in one group have the same properties and payloads. + # So it doesn't matter in what order it will be routed with round-robin. + # producer > 0 0 0 1 1 1 2 2 2 3 3 3 + # consumer1 < 0 1 2 3 + # consumer2 < 0 1 2 3 + # consumer3 < 0 1 2 3 + expected = [] + for i in range(messages_per_consumer): + posted_payloads = producer.post_same(num=num_consumers, value=i) + # Posted payloads on specific step are equal, need to memorize just + # one sample from each group. + expected.append(posted_payloads[0]) + + for consumer in consumers: consumer.expect_messages(expected, confirm=True) - def test_max_unconfirmed(self, cluster: Cluster): - """ - Test: routing of messages between several subscriptions with the same - expression and priority when max_unconfirmed reached. - - Create 1 producer / 2 consumers: C1, C2. - - Open 1 priority queue, the subscription expression and priority are - the same for each consumer. - - Stage 1: - - Producer: produce 10 identical messages (group A). - - Expect 5 messages for each consumer (round-robin routed) - Note: here max_unconfirmed just reached for consumer C1 but it doesn't - affect routing yet. - Producer -> [A * 10] - 10 identical posted - C1 <- [A * 5] - 5 listed, no more unconfirmed allowed - C2 <- [A * 5][_ * 10] - 5 listed, 10 unconfirmed allowed - - Stage 2: - - Produce another 10 messages (group B), non identical. - - Expect 5 messages for consumer C1. - - Expect 15 messages for consumer C2. - Note: max_unconfirmed was reached for C1 so this block of messages goes - entirely to the consumer C2. In the end max_unconfirmed reached for C2. - Producer -> [B * 10] - 10 posted - C1 <- [A * 5] - 5 listed, no more unconfirmed allowed - C2 <- [A * 5][B * 10] - 15 listed, no more unconfirmed allowed - - Stage 3: - - Produce another 1 message (group C), non identical. - - Expect 5 messages for consumer C1. - - Expect 15 messages for consumer C2. - Note: max_unconfirmed was reached for C1 and C2, so this block of - messages is not routed to either consumer. - Producer -> [C * 1] - 1 posted, not routed to consumers - C1 <- [A * 5] - 5 received, no more unconfirmed allowed - C2 <- [A * 5][B * 10] - 15 received, no more unconfirmed allowed - - Stage 4: - - Confirm all 5 messages for consumer C1. - - Expect another 1 message for consumer C1 from group [C]. - - Expect 15 messages for consumer C2. - Note: max_unconfirmed was reached for C2, but C1 confirms all received - messages. It is possible to route messages from Stage 3 to consumer C1. - In the end max_unconfirmed reached for C1 again. - Producer -> no more posted - C1 <- [C * 1][_ * 4] - 1 listed, 4 more unconfirmed allowed - C2 <- [A * 5][B * 10] - 15 listed, no more unconfirmed allowed - Confirmed: [A * 5] - - Stage 5: - - Confirm all messages for consumers C1 and C2. - - Expect empty C1 and C2. - - Note: the group C contains only 1 message because we encounter - implementation-specific delay of unconfirmed messages delivery if we - try to post more messages here. - - Concerns: - - Round-robin works for subscriptions until max_unconfirmed limit reached. - - When max_unconfirmed reached for one consumer all messages go to the - other consumer if possible. - - When max_unconfirmed reached for all consumers extra messages are not - routed to these. - - Extra messages are delivered correctly when one of the consumers is - ready to receive again. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer1 = Consumer( - cluster, - uri, - ["x >= 0"], - max_unconfirmed_messages=5, - ) - consumer2 = Consumer( - cluster, - uri, - ["x >= 0"], - max_unconfirmed_messages=15, - ) - msgs_A = producer.post_same(num=10, value=0) - half_A = msgs_A[:5] - consumer1.expect_messages(half_A, confirm=False) - consumer2.expect_messages(half_A, confirm=False) +def test_redelivery(cluster: Cluster): + """ + Test: message redelivery when the consumer exits without confirmation. + - Create 1 producer / 1 consumer: C1. + - Open 1 priority queue, with subscription parameters. + - Producer: produce N messages with properties. + - Consumer C1: expect but not confirm all N messages. + - Consumer C1: close. + - Open another consumer C2 with the same parameters. + - Consumer C2: expect and confirm all N messages. + + Concerns: + - Unconfirmed messages must be redelivered correctly. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer = Consumer( + cluster, + uri, + ["x >= 0"], + ) - msgs_B = producer.post_diff(num=10, offset=100) + expected = producer.post_diff(num=5) + consumer.expect_messages(expected, confirm=False) + # No confirm - consumer1.expect_messages(half_A, confirm=False, timeout=0) - consumer2.expect_messages(half_A + msgs_B, confirm=False) + consumer.close() - msgs_C = producer.post_diff(num=1, offset=200) + consumer = Consumer( + cluster, + uri, + ["x >= 0"], + ) + consumer.expect_messages(expected, confirm=True) + + +def test_max_unconfirmed(cluster: Cluster): + """ + Test: routing of messages between several subscriptions with the same + expression and priority when max_unconfirmed reached. + - Create 1 producer / 2 consumers: C1, C2. + - Open 1 priority queue, the subscription expression and priority are + the same for each consumer. + + Stage 1: + - Producer: produce 10 identical messages (group A). + - Expect 5 messages for each consumer (round-robin routed) + Note: here max_unconfirmed just reached for consumer C1 but it doesn't + affect routing yet. + Producer -> [A * 10] - 10 identical posted + C1 <- [A * 5] - 5 listed, no more unconfirmed allowed + C2 <- [A * 5][_ * 10] - 5 listed, 10 unconfirmed allowed + + Stage 2: + - Produce another 10 messages (group B), non identical. + - Expect 5 messages for consumer C1. + - Expect 15 messages for consumer C2. + Note: max_unconfirmed was reached for C1 so this block of messages goes + entirely to the consumer C2. In the end max_unconfirmed reached for C2. + Producer -> [B * 10] - 10 posted + C1 <- [A * 5] - 5 listed, no more unconfirmed allowed + C2 <- [A * 5][B * 10] - 15 listed, no more unconfirmed allowed + + Stage 3: + - Produce another 1 message (group C), non identical. + - Expect 5 messages for consumer C1. + - Expect 15 messages for consumer C2. + Note: max_unconfirmed was reached for C1 and C2, so this block of + messages is not routed to either consumer. + Producer -> [C * 1] - 1 posted, not routed to consumers + C1 <- [A * 5] - 5 received, no more unconfirmed allowed + C2 <- [A * 5][B * 10] - 15 received, no more unconfirmed allowed + + Stage 4: + - Confirm all 5 messages for consumer C1. + - Expect another 1 message for consumer C1 from group [C]. + - Expect 15 messages for consumer C2. + Note: max_unconfirmed was reached for C2, but C1 confirms all received + messages. It is possible to route messages from Stage 3 to consumer C1. + In the end max_unconfirmed reached for C1 again. + Producer -> no more posted + C1 <- [C * 1][_ * 4] - 1 listed, 4 more unconfirmed allowed + C2 <- [A * 5][B * 10] - 15 listed, no more unconfirmed allowed + Confirmed: [A * 5] + + Stage 5: + - Confirm all messages for consumers C1 and C2. + - Expect empty C1 and C2. + + Note: the group C contains only 1 message because we encounter + implementation-specific delay of unconfirmed messages delivery if we + try to post more messages here. + + Concerns: + - Round-robin works for subscriptions until max_unconfirmed limit reached. + - When max_unconfirmed reached for one consumer all messages go to the + other consumer if possible. + - When max_unconfirmed reached for all consumers extra messages are not + routed to these. + - Extra messages are delivered correctly when one of the consumers is + ready to receive again. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer1 = Consumer( + cluster, + uri, + ["x >= 0"], + max_unconfirmed_messages=5, + ) + consumer2 = Consumer( + cluster, + uri, + ["x >= 0"], + max_unconfirmed_messages=15, + ) - consumer1.expect_messages(half_A, confirm=False, timeout=0) - consumer2.expect_messages(half_A + msgs_B, confirm=False, timeout=0) + msgs_A = producer.post_same(num=10, value=0) + half_A = msgs_A[:5] + consumer1.expect_messages(half_A, confirm=False) + consumer2.expect_messages(half_A, confirm=False) + + msgs_B = producer.post_diff(num=10, offset=100) + + consumer1.expect_messages(half_A, confirm=False, timeout=0) + consumer2.expect_messages(half_A + msgs_B, confirm=False) + + msgs_C = producer.post_diff(num=1, offset=200) + + consumer1.expect_messages(half_A, confirm=False, timeout=0) + consumer2.expect_messages(half_A + msgs_B, confirm=False, timeout=0) + + consumer1.confirm_all() + + consumer1.expect_messages(msgs_C, confirm=False) + consumer2.expect_messages(half_A + msgs_B, confirm=False, timeout=0) + + consumer1.confirm_all() + consumer2.confirm_all() + + consumer1.expect_empty() + consumer2.expect_empty() + + +def test_max_unconfirmed_low_priority_spillover(cluster: Cluster): + """ + Test: routing of messages between several subscriptions with the same + expression but with different priorities when max_unconfirmed reached. + - Create 1 producer / 2 consumers: C1, C2. + - Open 1 priority queue, the subscription expressions are the same but + priorities are different for each consumer. + + Stage 1: + - Producer: produce 15 sequential different messages M1-M15. + - Consumer C1: expect but not confirm 5 messages M1-M5. + - Consumer C2: expect empty (low priority). + + Stage 2: + - Consumer C1: confirm all messages. + - Consumer C1: expect but not confirm 5 next messages M6-M10. + - Consumer C2: expect empty (low priority). + + Stage 3: + - Consumer C2: reconfigure - set the same expression with a new highest + priority. + - Consumer C1: confirm all messages. + - Consumer C1: expect empty (low priority). + - Consumer C2: expect and confirm 5 last messages M11-M15. + + Concerns: + - Messages are not routed to low priority consumers even when + max_unconfirmed reached. + - Reconfiguration does not break this behavior. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer1 = Consumer( + cluster, + uri, + subscriptions=[("x >= 0", 20)], + max_unconfirmed_messages=5, + ) + consumer2 = Consumer( + cluster, + uri, + subscriptions=[("x >= 0", 1)], + max_unconfirmed_messages=15, + ) - consumer1.confirm_all() + expected = producer.post_diff(num=15, offset=0) - consumer1.expect_messages(msgs_C, confirm=False) - consumer2.expect_messages(half_A + msgs_B, confirm=False, timeout=0) + consumer1.expect_messages(expected[:5], confirm=False) + consumer2.expect_empty() - consumer1.confirm_all() - consumer2.confirm_all() + consumer1.confirm_all() - consumer1.expect_empty() - consumer2.expect_empty() + consumer1.expect_messages(expected[5:10], confirm=False) + consumer2.expect_empty() - def test_max_unconfirmed_low_priority_spillover(self, cluster: Cluster): - """ - Test: routing of messages between several subscriptions with the same - expression but with different priorities when max_unconfirmed reached. - - Create 1 producer / 2 consumers: C1, C2. - - Open 1 priority queue, the subscription expressions are the same but - priorities are different for each consumer. - - Stage 1: - - Producer: produce 15 sequential different messages M1-M15. - - Consumer C1: expect but not confirm 5 messages M1-M5. - - Consumer C2: expect empty (low priority). - - Stage 2: - - Consumer C1: confirm all messages. - - Consumer C1: expect but not confirm 5 next messages M6-M10. - - Consumer C2: expect empty (low priority). - - Stage 3: - - Consumer C2: reconfigure - set the same expression with a new highest - priority. - - Consumer C1: confirm all messages. - - Consumer C1: expect empty (low priority). - - Consumer C2: expect and confirm 5 last messages M11-M15. - - Concerns: - - Messages are not routed to low priority consumers even when - max_unconfirmed reached. - - Reconfiguration does not break this behavior. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer1 = Consumer( - cluster, - uri, - subscriptions=[("x >= 0", 20)], - max_unconfirmed_messages=5, - ) - consumer2 = Consumer( - cluster, - uri, - subscriptions=[("x >= 0", 1)], - max_unconfirmed_messages=15, - ) + consumer2.configure(subscriptions=[("x >= 0", 400)]) - expected = producer.post_diff(num=15, offset=0) + consumer1.confirm_all() - consumer1.expect_messages(expected[:5], confirm=False) - consumer2.expect_empty() + consumer1.expect_empty() + consumer2.expect_messages(expected[10:15], confirm=False) - consumer1.confirm_all() + consumer2.confirm_all() + consumer2.expect_empty() - consumer1.expect_messages(expected[5:10], confirm=False) - consumer2.expect_empty() - consumer2.configure(subscriptions=[("x >= 0", 400)]) +def test_many_subscriptions(cluster: Cluster): + """ + Test: open multiple subscriptions at once. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, with K different subscriptions for consumer. + - Produce K different messages suitable for each subscription 1-by-1. + - Expect and confirm all K messages. - consumer1.confirm_all() + Concerns: + - All messages must be received in correct order. + """ + uri = tc.URI_PRIORITY + num_subscriptions = 256 - consumer1.expect_empty() - consumer2.expect_messages(expected[10:15], confirm=False) + subscriptions = [f"x == {i}" for i in range(num_subscriptions)] - consumer2.confirm_all() - consumer2.expect_empty() + producer = Producer(cluster, uri) + consumer = Consumer( + cluster, + uri, + subscriptions, + ) - def test_many_subscriptions(self, cluster: Cluster): - """ - Test: open multiple subscriptions at once. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, with K different subscriptions for consumer. - - Produce K different messages suitable for each subscription 1-by-1. - - Expect and confirm all K messages. - - Concerns: - - All messages must be received in correct order. - """ - uri = tc.URI_PRIORITY - num_subscriptions = 256 + expected = producer.post_diff(num=num_subscriptions) - subscriptions = [f"x == {i}" for i in range(num_subscriptions)] + consumer.expect_messages(expected, confirm=True) - producer = Producer(cluster, uri) - consumer = Consumer( - cluster, - uri, - subscriptions, - ) - expected = producer.post_diff(num=num_subscriptions) +def test_complex_expressions(cluster: Cluster): + """ + Test: complex subscription expressions with several properties. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, with complex subscription expression for consumer. + - Produce K different messages suitable for subscription. + - Produce different messages not suitable for subscription. + - Expect and confirm only K expected messages. - consumer.expect_messages(expected, confirm=True) + Concerns: + - Complex expressions work. + - All expected messages must be received in correct order. + """ + uri = tc.URI_PRIORITY - def test_complex_expressions(self, cluster: Cluster): - """ - Test: complex subscription expressions with several properties. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, with complex subscription expression for consumer. - - Produce K different messages suitable for subscription. - - Produce different messages not suitable for subscription. - - Expect and confirm only K expected messages. - - Concerns: - - Complex expressions work. - - All expected messages must be received in correct order. - """ - uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer = Consumer( + cluster, + uri, + ["x >= 0 && x <= 100 && y <= 100"], + ) - producer = Producer(cluster, uri) - consumer = Consumer( - cluster, - uri, - ["x >= 0 && x <= 100 && y <= 100"], - ) + expected = [producer.post(x=x, y=10 + x) for x in range(10)] + producer.post(x=-10, y=100) + producer.post(x=-100, y=-10) + producer.post(x=-1, y=-1) + + consumer.expect_messages(expected, confirm=True) + + +def test_incorrect_expressions(cluster: Cluster): + """ + Test: incorrect expressions. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, with always false subscription. + + For each pre-defined incorrect expression EXPR: + - Try to configure consumer with expression EXPR. + - Expect fail (exception) during configuration. + - Post K different messages. + - Expect empty consumer. + - Reconfigure consumer with correct and always true expression. + - Expect consumer to receive previously posted K messages. + - Cleanup: reconfigure consumer with always false expression. + + Concerns: + - Broker and tool (which uses C++ SDK) do not crash when incorrect + expression specified. + - Incorrect expressions are rejected by the SDK. + - Broker and tool continue to work correctly. + """ + expressions = [ + " ", + "()", + "((true)", + "x > ", + "x x", + "&&", + " && 1 > 0", + "() && 1 > 0", + "true && (>>)", + ] + + uri = tc.URI_PRIORITY + + producer = Producer(cluster, uri) + consumer = Consumer(cluster, uri, ["x < 0"]) + + for i, expression in enumerate(expressions): + try: + consumer.configure([expression], timeout=5) + assert False # must not get here, expect exception + except ITError: + # expected + pass - expected = [producer.post(x=x, y=10 + x) for x in range(10)] - producer.post(x=-10, y=100) - producer.post(x=-100, y=-10) - producer.post(x=-1, y=-1) + expected = producer.post_diff(num=10, offset=i * 100) + consumer.expect_empty() + consumer.configure(["x >= 0"]) consumer.expect_messages(expected, confirm=True) - def test_incorrect_expressions(self, cluster: Cluster): - """ - Test: incorrect expressions. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, with always false subscription. - - For each pre-defined incorrect expression EXPR: - - Try to configure consumer with expression EXPR. - - Expect fail (exception) during configuration. - - Post K different messages. - - Expect empty consumer. - - Reconfigure consumer with correct and always true expression. - - Expect consumer to receive previously posted K messages. - - Cleanup: reconfigure consumer with always false expression. - - Concerns: - - Broker and tool (which uses C++ SDK) do not crash when incorrect - expression specified. - - Incorrect expressions are rejected by the SDK. - - Broker and tool continue to work correctly. - """ - expressions = [ - " ", - "()", - "((true)", - "x > ", - "x x", - "&&", - " && 1 > 0", - "() && 1 > 0", - "true && (>>)", - ] - - uri = tc.URI_PRIORITY - - producer = Producer(cluster, uri) - consumer = Consumer(cluster, uri, ["x < 0"]) - - for i, expression in enumerate(expressions): - try: - consumer.configure([expression], timeout=5) - assert False # must not get here, expect exception - except ITError: - # expected - pass - - expected = producer.post_diff(num=10, offset=i * 100) - consumer.expect_empty() - - consumer.configure(["x >= 0"]) - consumer.expect_messages(expected, confirm=True) - - consumer.configure(["x < 0"]) + consumer.configure(["x < 0"]) - def test_non_bool_expressions(self, cluster: Cluster): - """ - Test: expressions that are not evaluated as booleans. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, with always true subscription. - - For each pre-defined non-boolean expression EXPR: - - Configure consumer with expression EXPR. - - Produce K different messages. - - Consumer: expect empty. - Cleanup: - - Consumer: reconfigure with always true expression. - - Consumer: expect and confirm all K messages. - - Concerns: - - Broker and tool do not crash when non-bool expression specified. - - Non-bool expressions considered to be always false. - """ - expressions = ["x", "x + 33"] - - uri = tc.URI_PRIORITY - - producer = Producer(cluster, uri) - consumer = Consumer(cluster, uri, ["x >= 0"]) - - for i, expression in enumerate(expressions): - consumer.configure([expression]) - - expected = producer.post_diff(num=10, offset=i * 100) - consumer.expect_empty() - - consumer.configure(["x >= 0"]) - consumer.expect_messages(expected, confirm=True) - - def test_numeric_limits(self, cluster: Cluster, logger): - """ - Test: pass huge values in subscription expressions. - - Create 1 producer / 1 consumer. - - Open 1 priority queue. - - For each pre-defined huge value VAL: - - Starting state: configure consumer with always false expression. - - Try to re-configure consumer with expression including huge value VAL. - - Expect successful configuration for supported values. - - Expect exception during configuration for unsupported values. - - Produce K different messages which are suitable for subscription in - ideal mathematical means. - - Expect empty consumer for unsupported VAL. - - Expect and confirm all K messages for supported VAL. - - Reconfigure consumer if needed and confirm all messages for clean-up. - - Concerns: - - Expressions with out-of-bounds values are rejected by the C++ SDK. - - Broker and tool (which uses C++ SDK) do not crash when out-of-bounds - expression specified. - - Seemingly adequate range of integer values in expressions is still - supported. - """ - uri = tc.URI_PRIORITY - - supported_ints = [ - -(2**15), - 2**15 - 1, - -(2**31), - 2**31 - 1, - -(2**63) + 1, - 2**63 - 1, - -(2**63), - ] - unsupported_ints = [ - 2**63, - -(2**127), - 2**127 - 1, - -(2**255), - 2**255 - 1, - ] - - producer = Producer(cluster, uri) - consumer = Consumer( - cluster, - uri, - ) - - for i, value in enumerate(supported_ints + unsupported_ints): - consumer.configure(["x < 0"]) - - try: - consumer.configure([f"x != {value}"], timeout=5) - assert value in supported_ints - except ITError: - # exception is expected for values from 'unsupported_ints' - assert value in unsupported_ints - - expected = producer.post_diff(num=10, offset=i * 100) - if value in supported_ints: - logger.info("supported_ints[%s]: %s ", i, value) - consumer.expect_messages(expected, confirm=True) - else: - logger.info("unsupported_ints[%s]: %s", i - len(supported_ints), value) - consumer.expect_empty() - # Reconfigure for clean-up - consumer.configure(subscriptions=["x >= 0"]) - consumer.expect_messages(expected, confirm=True) - - def test_empty_subscription(self, cluster: Cluster): - """ - Test: empty subscription works for backward compatibility. - - Create 1 producer / 1 consumer. - - Open 1 priority queue, with empty subscription for consumer. +def test_non_bool_expressions(cluster: Cluster): + """ + Test: expressions that are not evaluated as booleans. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, with always true subscription. - - Produce [M1] messages with properties to the queue. - - Expect and confirm all [M1] messages in correct order by consumer. + For each pre-defined non-boolean expression EXPR: + - Configure consumer with expression EXPR. + - Produce K different messages. + - Consumer: expect empty. + Cleanup: + - Consumer: reconfigure with always true expression. + - Consumer: expect and confirm all K messages. - - Reconfigure consumer with always false expression. - - Produce [M2] messages with properties to the queue. - - Expect empty consumer. + Concerns: + - Broker and tool do not crash when non-bool expression specified. + - Non-bool expressions considered to be always false. + """ + expressions = ["x", "x + 33"] - - Reconfigure consumer with empty subscription. - - Expect and confirm all [M2] messages in correct order by consumer. + uri = tc.URI_PRIORITY - Concerns: - - Consumer with empty subscription must receive all messages. - - Reconfiguration does not break this behaviour. - """ - uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer = Consumer(cluster, uri, ["x >= 0"]) - producer = Producer(cluster, uri) - consumer = Consumer(cluster, uri, EMPTY_SUBSCRIPTION) + for i, expression in enumerate(expressions): + consumer.configure([expression]) - expected = producer.post_diff(num=10, offset=0) - consumer.expect_messages(expected, confirm=True) - - consumer.configure(["x < 0 && x > 0"]) # always false - - expected = producer.post_diff(num=10, offset=100) + expected = producer.post_diff(num=10, offset=i * 100) consumer.expect_empty() - consumer.configure(EMPTY_SUBSCRIPTION) + consumer.configure(["x >= 0"]) consumer.expect_messages(expected, confirm=True) - expected = producer.post_diff(num=10, offset=200) - consumer.expect_messages(expected, confirm=True) - def test_empty_expression(self, cluster: Cluster): - """ - Test: subscription with specific empty string "" expression rejected. - - Create 1 producer / 1 consumer. - - Open 1 priority queue for producer. - - Produce N messages with properties to the queue. - - Expect fail when try to open consumer with empty "" subscription - expression. - - Open consumer with always false expression. - - Expect empty consumer. - - Expect fail when try to configure consumer with empty "" subscription - expression, mixed together with always true expressions. - - Expect empty consumer. - - Reconfigure consumer with always true expression. - - Expect and confirm all N messages in correct order by consumer. - - Concerns: - - Empty "" subscription expression is rejected by both open/configure. - - These rejections do not break the broker. - - These rejections do not allow consumer to receive not routed yet - messages unintentionally. - - If valid expressions passed together with empty "" expression the - entire configure is rejected, so these valid subscriptions do not work. - """ - uri = tc.URI_PRIORITY - - producer = Producer(cluster, uri) - expected = producer.post_diff(num=10, offset=0) - always_true = "x >= 0" - always_false = "x < -100" +def test_numeric_limits(cluster: Cluster): + """ + Test: pass huge values in subscription expressions. + - Create 1 producer / 1 consumer. + - Open 1 priority queue. + + For each pre-defined huge value VAL: + - Starting state: configure consumer with always false expression. + - Try to re-configure consumer with expression including huge value VAL. + - Expect successful configuration for supported values. + - Expect exception during configuration for unsupported values. + - Produce K different messages which are suitable for subscription in + ideal mathematical means. + - Expect empty consumer for unsupported VAL. + - Expect and confirm all K messages for supported VAL. + - Reconfigure consumer if needed and confirm all messages for clean-up. + + Concerns: + - Expressions with out-of-bounds values are rejected by the C++ SDK. + - Broker and tool (which uses C++ SDK) do not crash when out-of-bounds + expression specified. + - Seemingly adequate range of integer values in expressions is still + supported. + """ + uri = tc.URI_PRIORITY + + supported_ints = [ + -(2**15), + 2**15 - 1, + -(2**31), + 2**31 - 1, + -(2**63) + 1, + 2**63 - 1, + -(2**63), + ] + unsupported_ints = [ + 2**63, + -(2**127), + 2**127 - 1, + -(2**255), + 2**255 - 1, + ] + + producer = Producer(cluster, uri) + consumer = Consumer( + cluster, + uri, + ) - try: - # Expect exception raise in constructor, so do not want to use this - # object later. Let garbage collector handle it. - _ = Consumer(cluster, uri, [""], timeout=5) - assert False, "Expected fail on empty subscription expression" - except ITError as ex: - exception_message = str(ex) - assert "did not complete" in exception_message - - consumer = Consumer(cluster, uri, [always_false]) - consumer.expect_empty() + for i, value in enumerate(supported_ints + unsupported_ints): + consumer.configure(["x < 0"]) try: - # Expect exception raise in 'configure'. - consumer.configure([always_true, "", always_true], timeout=5) - assert False, "Expected fail on empty subscription expression" - except ITError as ex: - exception_message = str(ex) - assert "did not complete" in exception_message - - consumer.expect_empty() - - consumer.configure([always_true]) - consumer.expect_messages(expected, confirm=True) - - def test_non_existent_property(self, cluster: Cluster): - """ - Test: expressions are evaluated even if some properties are missing. - - Create 1 producer / 2 consumers: C1, C2. - - Open 1 priority queue, provide different complex subscription - expressions for each consumer: - consumer C1 -> property "x" first OR "y" second. - consumer C2 -> property "y" first OR "x" second. - - Produce [M1] messages with property "x" only to the queue. - - Produce [M2] messages with property "y" only to the queue. - - Expect consumer C1 to receive and confirm only messages [M1]. - - Expect consumer C2 to receive and confirm only messages [M2]. - - Concerns: - - Expression strings evaluated even if some properties are missing, if - some non-existent property encountered the expression result is False. - - Expressions are evaluated from left to right, and it is possible to - return result early if logical expression allows it. - """ - uri = tc.URI_PRIORITY - - producer = Producer(cluster, uri) - consumer1 = Consumer(cluster, uri, ["x >= 0 || (y >= 0)"]) - consumer2 = Consumer(cluster, uri, ["y >= 0 || (x >= 0)"]) - - expected1 = [producer.post(x=10) for _ in range(10)] - expected2 = [producer.post(y=10) for _ in range(10)] - - consumer1.expect_messages(expected1, confirm=True) - consumer2.expect_messages(expected2, confirm=True) - - def test_date_time(self, cluster: Cluster): - """ - Test: expressions with date/time passed as string work. - - Create 1 producer / 1 consumers. - - Open 1 priority queue. - - Specify a list of sorted dates DATES. - - Consider START and END from DATES, where START < 1+ elements < END. - - For each such pair START/END do the following: - - - Configure consumer to receive only messages with date from (START; END). - - Produce N messages with all sorted and specified dates from DATES. - - Expect and confirm only messages with date from (START; END). - - Reconfigure consumer to receive messages from the reversed date range: - (-inf; START] union [END; +inf). - - Expect and confirm only messages with date from (-inf; START] union [END; +inf). - - Concerns: - - String properties and expressions work. - - Dates in ISO string format are compared in obvious and expected way. - """ - uri = tc.URI_PRIORITY - producer = Producer(cluster, uri) - consumer = Consumer(cluster, uri) - - # Contract: 'dates' elements are sorted. - dates = [ - datetime(1970, 1, 1, 0, 0, 0, 0), - datetime(1970, 1, 1, 12, 0, 0, 0), - datetime(1999, 10, 30, 4, 0, 0, 0), - datetime(2007, 4, 3, 11, 15, 25, 50), - datetime(2007, 8, 6, 23, 30, 50), - datetime(2007, 8, 6, 23, 30, 50, 100), - datetime(2022, 12, 12, 10, 56, 34, 678), - ] - - for id_end in range(len(dates)): # pylint: disable=consider-using-enumerate - for id_start in range(id_end - 1): - start = dates[id_start].isoformat() - end = dates[id_end].isoformat() - - expression = f'x > "{start}" && x < "{end}"' - reverse = f'x <= "{start}" || x >= "{end}"' - - consumer.configure([expression]) - - posted = [producer.post(x=date.isoformat()) for date in dates] - - # sample: - # [] [s] [] [] [] [e] [] - # [++++++] - expected = posted[id_start + 1 : id_end] - - # sample: - # [] [s] [] [] [] [e] [] - # [++++] [++++] - remaining = posted[: id_start + 1] + posted[id_end:] - - consumer.expect_messages(expected, confirm=True) - - consumer.configure([reverse]) - consumer.expect_messages(remaining, confirm=True) - - def test_consumer_multiple_priorities(self, cluster: Cluster): - """ - Test: messages are routed correctly with overlapping subscriptions with - different priorities. - - Create 1 producer / 2 consumers: C1, C2. - - Open 1 priority queue, provide different overlapping subscriptions - with different priorities. - Consumer C1: high priority, suitable "x" range [1000; +inf) - Consumer C2: low priority, suitable "x" range [0 .......; +inf) - - Producer: produce groups of messages with property "x" from [0; 1000) - range and [1000; +inf) range. - - Consumer C1: expect and confirm all messages with "x" in range [1000; +inf). - - Consumer C2: expect and confirm all messages with "x" in range [0; 1000). - - Concerns: - - High priority consumer receives all suitable messages according to - subscription. - - Low priority consumer will receive all suitable messages according to - subscription if there are no conflicting consumers with higher priority. - """ - uri = tc.URI_PRIORITY + consumer.configure([f"x != {value}"], timeout=5) + assert value in supported_ints + except ITError: + # exception is expected for values from 'unsupported_ints' + assert value in unsupported_ints - producer = Producer(cluster, uri) - consumer1 = Consumer(cluster, uri, [("x >= 1000", 5)]) - consumer2 = Consumer(cluster, uri, [("x >= 0", 1)]) + expected = producer.post_diff(num=10, offset=i * 100) - expected = producer.post_diff(num=10, offset=1000) - consumer1.expect_messages(expected, confirm=True) - - expected = producer.post_diff(num=10, offset=0) - consumer2.expect_messages(expected, confirm=True) - - expected = producer.post_diff(num=10, offset=2000) - consumer1.expect_messages(expected, confirm=True) - - expected = producer.post_diff(num=10, offset=100) - consumer2.expect_messages(expected, confirm=True) - - def test_no_capacity_all_optimization(self, cluster: Cluster): - """ - Test: delivery optimization works during routing when all subscriptions - have no capacity. - - DRQS 171509204 - - - Create 1 producer / 3 consumers: C1, C2, C3. - - Consumers: max_unconfirmed_messages = 1. - - Post several messages to cause NO_CAPACITY_ALL condition. - - Expect delivery optimization log messages when necessary. - - TODO: compare GUIDs of the posted messages and messages on which NO_CAPACITY_ALL - encountered. - - Concerns: - - Delivery optimization works when NO_CAPACITY_ALL condition observed. - """ - uri = tc.URI_PRIORITY - - producer = Producer(cluster, uri) - consumer1 = Consumer(cluster, uri, ["x > 0"], max_unconfirmed_messages=1) - - optimization_monitor = DeliveryOptimizationMonitor(cluster) - - # consumer1: capacity 0/1 - # pending messages: [] - - m1 = producer.post(x=1) - - consumer1.expect_messages([m1], confirm=False) - # consumer1: capacity 1/1 [m1] - # pending messages: [] - - # Do not expect a new delivery optimization message, because - # 'consumer1' just reached the full capacity with the last message 'm1', - # but this is not enough to enable the delivery optimization: need at least - # one message after the added one to see that all consumers are full. - assert not optimization_monitor.has_new_message() - - m2 = producer.post(x=2) - # consumer1: capacity 1/1 [m1] - # pending messages: [m2] - - # Extra message 'm2' is good for 'consumer1' but cannot be delivered: - # no capacity all condition observed. - assert optimization_monitor.has_new_message() - - m3 = producer.post(x=3) # pylint: disable=unused-variable - # consumer1: capacity 1/1 [m1] - # pending messages: [m2, m3] - - # Extra message 'm3' is good for 'consumer1' but cannot be delivered: - # no capacity all condition observed. - assert optimization_monitor.has_new_message() - - consumer1.confirm_all() - consumer1.expect_messages([m2], confirm=False) - # consumer1: capacity 1/1 [m2] - # pending messages: [m3] - - # Message 'm1' was confirmed, so routing is triggered. - # 'consumer1' receives the pending message 'm2' and becomes full again. - # The message 'm3' still exists and is good for 'consumer1', but cannot - # be delivered: no capacity all condition observed. - assert optimization_monitor.has_new_message() - - m4 = producer.post(y=1) - # consumer1: capacity 1/1 [m2] - # pending messages: [m3, m4] - - # Posting the message 'm4' triggers routing, but the only consumer is full: - # no capacity all condition observed. - assert optimization_monitor.has_new_message() - - consumer2 = Consumer(cluster, uri, ["y < 0"], max_unconfirmed_messages=1) - - m5 = producer.post(y=-1) - consumer2.expect_messages([m5], confirm=False) - # consumer1: capacity 1/1 [m2] - # consumer2: capacity 1/1 [m5] - # pending messages: [m3, m4] - - # Do not expect a new delivery optimization message, because - # 'consumer2' just reached the full capacity with the last message 'm5', - # but this is not enough to enable the delivery optimization: need at least - # one message after the added one to see that all consumers are full. - assert not optimization_monitor.has_new_message() - - consumer3 = Consumer(cluster, uri, ["y >= 0"], max_unconfirmed_messages=1) - consumer3.expect_messages([m4], confirm=False) - # consumer1: capacity 1/1 [m2] - # consumer2: capacity 1/1 [m5] - # consumer3: capacity 1/1 [m4] - # pending messages: [m3] - - # Do not expect a new delivery optimization message, because - # 'consumer3' just reached the full capacity with the last message 'm4', - # but this is not enough to enable the delivery optimization: need at least - # one message after the added one to see that all consumers are full. - assert not optimization_monitor.has_new_message() - - def test_no_capacity_all_fanout(self, cluster: Cluster): - """ - Test: delivery optimization encountered with one app does not affect - other apps. - - DRQS 171509204 - - - Create 1 producer / 2 consumers: C_foo, C_bar. - - C_foo: max_unconfirmed_messages = 128. - - C_bar: max_unconfirmed_messages = 1. - - Post several messages to cause NO_CAPACITY_ALL condition for C_bar. - - Expect delivery log messages when necessary. - - Concerns: - - Delivery optimization condition encountered with one app does not - affect the other app. - """ - producer_uri = tc.URI_FANOUT - uri_foo = tc.URI_FANOUT_FOO - uri_bar = tc.URI_FANOUT_BAR - - producer = Producer(cluster, producer_uri) - consumer_foo = Consumer( - cluster, uri_foo, ["x > 0"], max_unconfirmed_messages=128 - ) - consumer_bar = Consumer(cluster, uri_bar, ["x > 0"], max_unconfirmed_messages=1) - - optimization_monitor = DeliveryOptimizationMonitor(cluster) - - # consumer_foo: capacity 0/128 - # consumer_bar: capacity 0/1 - # pending messages (bar): [] - - m1 = producer.post(x=1) - - consumer_foo.expect_messages([m1], confirm=False) - consumer_bar.expect_messages([m1], confirm=False) - # consumer_foo: capacity 1/128 [m1] - # consumer_bar: capacity 1/1 [m1] - # pending messages (bar): [] - - # Do not expect a new delivery optimization message, because - # 'consumer_bar' just reached the full capacity with the last message 'm1', - # but this is not enough to enable the delivery optimization: need at least - # one message after the added one to see that all consumers are full. - assert not optimization_monitor.has_new_message("bar") - - m2 = producer.post(x=2) - # consumer_foo: capacity 2/128 [m1, m2] - # consumer_bar: capacity 1/1 [m1] - # pending messages (bar): [m2] - - # Extra message 'm2' is good for 'consumer_bar' but cannot be delivered: - # no capacity all condition observed. - assert optimization_monitor.has_new_message("bar") - - m3 = producer.post(x=3) # pylint: disable=unused-variable - # consumer_foo: capacity 3/128 [m1, m2, m3] - # consumer_bar: capacity 1/1 [m1] - # pending messages (bar): [m2, m3] - - # Extra message 'm3' is good for 'consumer_bar' but cannot be delivered: - # no capacity all condition observed. - assert optimization_monitor.has_new_message("bar") - - consumer_bar.confirm_all() - consumer_bar.expect_messages([m2], confirm=False) - # consumer_foo: capacity 3/128 [m1, m2, m3] - # consumer_bar: capacity 1/1 [m2] - # pending messages (bar): [m3] - - # Message 'm1' was confirmed, so routing is triggered. - # 'consumer_bar' receives the pending message 'm2' and becomes full again. - # The message 'm3' still exists and is good for 'consumer_bar', but cannot - # be delivered: no capacity all condition observed. - assert optimization_monitor.has_new_message("bar") - - # App 'foo' has not reached NO_CAPACITY_ALL condition, so there must - # not be any delivery optimization messages. - assert not optimization_monitor.has_new_message("foo") - - # App 'baz' does not have any consumers. It will have the NO_CAPACITY_ALL - # condition, because it's impossible to have any progress. - assert optimization_monitor.has_new_message("baz") - - def test_primary_node_crash(self, standard_cluster: Cluster): - """ - Test: configured subscriptions work after primary node crash. - - Create 1 producer / N consumers. - - Open 1 priority queue, provide non-overlapping subscriptions. - - Produce messages [A] suitable for each consumer. - - Expect but not confirm suitable messages from [A] for each consumer. - - - Kill primary node. - - Wait for a new leader. - - - Expect and confirm messages from [A] for each consumer. - - Produce messages [B] suitable for each consumer. - - Expect and confirm messages from [B] for each consumer. - - Concerns: - - Message listing works after primary node crash and change. - - Already configured subscriptions work after primary node crash and - change. - """ - uri = tc.URI_PRIORITY - - producer = Producer(standard_cluster, uri) - consumer1 = Consumer(standard_cluster, uri, ["x < 0"]) - consumer2 = Consumer(standard_cluster, uri, ["x > 0"]) - consumer3 = Consumer(standard_cluster, uri, ["x == 0"]) - - expected1 = producer.post_diff(num=10, offset=-100) - expected2 = producer.post_diff(num=10, offset=100) - expected3 = producer.post_same(num=10, value=0) - - consumer1.expect_messages(expected1, confirm=False) - consumer2.expect_messages(expected2, confirm=False) - consumer3.expect_messages(expected3, confirm=False) - - standard_cluster.drain() - leader = standard_cluster.last_known_leader - leader.check_exit_code = False - leader.kill() - leader.wait() - - leader = standard_cluster.wait_leader() - assert leader is not None - - # Consumer with always false expression in this context. - # Used for synchronization. - Consumer(standard_cluster, uri, ["x > 100000"]) - - consumer1.expect_messages(expected1, confirm=True, timeout=0) - consumer2.expect_messages(expected2, confirm=True, timeout=0) - consumer3.expect_messages(expected3, confirm=True, timeout=0) - - expected1 = producer.post_diff(num=10, offset=-200) - expected2 = producer.post_diff(num=10, offset=200) - expected3 = producer.post_same(num=10, value=0) - - consumer1.expect_messages(expected1, confirm=True) - consumer2.expect_messages(expected2, confirm=True) - consumer3.expect_messages(expected3, confirm=True) - - def test_redelivery_on_primary_node_crash(self, standard_cluster: Cluster): - """ - Test: configured subscriptions work after primary node crash. - - Create 1 producer / 3 consumers: C_high, C_low1, C_low2. - - Open 1 priority queue. - Consumer C_high: high priority, expression (-inf ; +inf) - Consumer C_low1: low priority, expression (-inf; 0] - Consumer C_low2: low priority, expression (0; +inf) - - Produce messages [A1] suitable both for C_high and C_low1. - - Produce messages [B1] suitable both for C_high and C_low2. - - Expect but not confirm messages [A1, B1] for consumer C_high. - - - Kill primary node. - - Close consumer C_high. - - Wait for a new leader. - - - Expect and confirm messages [A1] for consumer C_low1. - - Expect and confirm messages [B1] for consumer C_low2. - - Produce messages [A2] suitable both for C_low1. - - Produce messages [B2] suitable both for C_low2. - - Expect and confirm messages [A2] for consumer C_low1. - - Expect and confirm messages [B2] for consumer C_low2. - - Concerns: - - Message re-routing works after primary node crash and change. - - Already configured subscriptions work after primary node crash and - change. - """ - uri = tc.URI_PRIORITY - - producer = Producer(standard_cluster, uri) - consumer_high = Consumer(standard_cluster, uri, [("x <= 0 || x > 0", 2)]) - consumer_low1 = Consumer(standard_cluster, uri, [("x <= 0", 1)]) - consumer_low2 = Consumer(standard_cluster, uri, [("x > 0", 1)]) - - expected1 = producer.post_diff(num=10, offset=-100) - expected2 = producer.post_diff(num=10, offset=100) - - consumer_high.expect_messages(expected1 + expected2, confirm=False) - - standard_cluster.drain() - leader = standard_cluster.last_known_leader - leader.check_exit_code = False - leader.kill() - leader.wait() - - consumer_high.close() - - leader = standard_cluster.wait_leader() - assert leader is not None - - # Consumer with always false expression in this context. - # Used for synchronization. - Consumer(standard_cluster, uri, ["x > 100000"]) - - consumer_low1.expect_messages(expected1, confirm=True) - consumer_low2.expect_messages(expected2, confirm=True) + if value in supported_ints: + test_logger.info("supported_ints[%s]: %s ", i, value) + consumer.expect_messages(expected, confirm=True) + else: + test_logger.info("unsupported_ints[%s]: %s", i - len(supported_ints), value) + consumer.expect_empty() + # Reconfigure for clean-up + consumer.configure(subscriptions=["x >= 0"]) + consumer.expect_messages(expected, confirm=True) - expected1 = producer.post_diff(num=10, offset=-200) - expected2 = producer.post_diff(num=10, offset=200) - consumer_low1.expect_messages(expected1, confirm=True) - consumer_low2.expect_messages(expected2, confirm=True) +def test_empty_subscription(cluster: Cluster): + """ + Test: empty subscription works for backward compatibility. + - Create 1 producer / 1 consumer. + - Open 1 priority queue, with empty subscription for consumer. + + - Produce [M1] messages with properties to the queue. + - Expect and confirm all [M1] messages in correct order by consumer. + + - Reconfigure consumer with always false expression. + - Produce [M2] messages with properties to the queue. + - Expect empty consumer. + + - Reconfigure consumer with empty subscription. + - Expect and confirm all [M2] messages in correct order by consumer. + + Concerns: + - Consumer with empty subscription must receive all messages. + - Reconfiguration does not break this behaviour. + """ + uri = tc.URI_PRIORITY + + producer = Producer(cluster, uri) + consumer = Consumer(cluster, uri, EMPTY_SUBSCRIPTION) + + expected = producer.post_diff(num=10, offset=0) + consumer.expect_messages(expected, confirm=True) + + consumer.configure(["x < 0 && x > 0"]) # always false + + expected = producer.post_diff(num=10, offset=100) + consumer.expect_empty() + + consumer.configure(EMPTY_SUBSCRIPTION) + consumer.expect_messages(expected, confirm=True) + + expected = producer.post_diff(num=10, offset=200) + consumer.expect_messages(expected, confirm=True) + + +def test_empty_expression(cluster: Cluster): + """ + Test: subscription with specific empty string "" expression rejected. + - Create 1 producer / 1 consumer. + - Open 1 priority queue for producer. + - Produce N messages with properties to the queue. + - Expect fail when try to open consumer with empty "" subscription + expression. + - Open consumer with always false expression. + - Expect empty consumer. + - Expect fail when try to configure consumer with empty "" subscription + expression, mixed together with always true expressions. + - Expect empty consumer. + - Reconfigure consumer with always true expression. + - Expect and confirm all N messages in correct order by consumer. + + Concerns: + - Empty "" subscription expression is rejected by both open/configure. + - These rejections do not break the broker. + - These rejections do not allow consumer to receive not routed yet + messages unintentionally. + - If valid expressions passed together with empty "" expression the + entire configure is rejected, so these valid subscriptions do not work. + """ + uri = tc.URI_PRIORITY + + producer = Producer(cluster, uri) + expected = producer.post_diff(num=10, offset=0) + always_true = "x >= 0" + always_false = "x < -100" + + try: + # Expect exception raise in constructor, so do not want to use this + # object later. Let garbage collector handle it. + _ = Consumer(cluster, uri, [""], timeout=5) + assert False, "Expected fail on empty subscription expression" + except ITError as ex: + exception_message = str(ex) + assert "did not complete" in exception_message + + consumer = Consumer(cluster, uri, [always_false]) + consumer.expect_empty() + + try: + # Expect exception raise in 'configure'. + consumer.configure([always_true, "", always_true], timeout=5) + assert False, "Expected fail on empty subscription expression" + except ITError as ex: + exception_message = str(ex) + assert "did not complete" in exception_message + + consumer.expect_empty() + + consumer.configure([always_true]) + consumer.expect_messages(expected, confirm=True) + + +def test_non_existent_property(cluster: Cluster): + """ + Test: expressions are evaluated even if some properties are missing. + - Create 1 producer / 2 consumers: C1, C2. + - Open 1 priority queue, provide different complex subscription + expressions for each consumer: + consumer C1 -> property "x" first OR "y" second. + consumer C2 -> property "y" first OR "x" second. + - Produce [M1] messages with property "x" only to the queue. + - Produce [M2] messages with property "y" only to the queue. + - Expect consumer C1 to receive and confirm only messages [M1]. + - Expect consumer C2 to receive and confirm only messages [M2]. + + Concerns: + - Expression strings evaluated even if some properties are missing, if + some non-existent property encountered the expression result is False. + - Expressions are evaluated from left to right, and it is possible to + return result early if logical expression allows it. + """ + uri = tc.URI_PRIORITY + + producer = Producer(cluster, uri) + consumer1 = Consumer(cluster, uri, ["x >= 0 || (y >= 0)"]) + consumer2 = Consumer(cluster, uri, ["y >= 0 || (x >= 0)"]) + + expected1 = [producer.post(x=10) for _ in range(10)] + expected2 = [producer.post(y=10) for _ in range(10)] + + consumer1.expect_messages(expected1, confirm=True) + consumer2.expect_messages(expected2, confirm=True) + + +def test_date_time(cluster: Cluster): + """ + Test: expressions with date/time passed as string work. + - Create 1 producer / 1 consumers. + - Open 1 priority queue. + - Specify a list of sorted dates DATES. + - Consider START and END from DATES, where START < 1+ elements < END. + - For each such pair START/END do the following: + + - Configure consumer to receive only messages with date from (START; END). + - Produce N messages with all sorted and specified dates from DATES. + - Expect and confirm only messages with date from (START; END). + - Reconfigure consumer to receive messages from the reversed date range: + (-inf; START] union [END; +inf). + - Expect and confirm only messages with date from (-inf; START] union [END; +inf). + + Concerns: + - String properties and expressions work. + - Dates in ISO string format are compared in obvious and expected way. + """ + uri = tc.URI_PRIORITY + producer = Producer(cluster, uri) + consumer = Consumer(cluster, uri) + + # Contract: 'dates' elements are sorted. + dates = [ + datetime(1970, 1, 1, 0, 0, 0, 0), + datetime(1970, 1, 1, 12, 0, 0, 0), + datetime(1999, 10, 30, 4, 0, 0, 0), + datetime(2007, 4, 3, 11, 15, 25, 50), + datetime(2007, 8, 6, 23, 30, 50), + datetime(2007, 8, 6, 23, 30, 50, 100), + datetime(2022, 12, 12, 10, 56, 34, 678), + ] + + for id_end in range(len(dates)): # pylint: disable=consider-using-enumerate + for id_start in range(id_end - 1): + start = dates[id_start].isoformat() + end = dates[id_end].isoformat() + + expression = f'x > "{start}" && x < "{end}"' + reverse = f'x <= "{start}" || x >= "{end}"' - def test_reconfigure_on_primary_node_crash(self, standard_cluster: Cluster): - """ - Test: configured subscriptions work after primary node crash. - - Create 1 producer / 3 consumers: C_high, C_low1, C_low2. - - Open 1 priority queue. - Consumer C_high: high priority, expression (-inf ; +inf) - Consumer C_low1: low priority, expression (0; +inf) - Consumer C_low2: low priority, expression (-inf; 0] - - Produce messages [A1] suitable both for C_high and C_low1. - - Produce messages [B1] suitable both for C_high and C_low2. - - Expect but not confirm messages [A1, B1] for consumer C_high. - - - Kill primary node. - - Close consumer C_high. - - Reconfigure consumers C_low1, C_low2: reverse expressions. - Consumer C_low1: low priority, expression (-inf; 0] - Consumer C_low2: low priority, expression (0; +inf) - - Wait for a new leader. - - - Expect and confirm messages [B1] for consumer C_low1. - - Expect and confirm messages [A1] for consumer C_low2. - - Produce messages [B2] suitable both for C_low1. - - Produce messages [A2] suitable both for C_low2. - - Expect and confirm messages [B2] for consumer C_low1. - - Expect and confirm messages [A2] for consumer C_low2. - - Concerns: - - Configure works after primary node crash and change. - """ - uri = tc.URI_PRIORITY + consumer.configure([expression]) - producer = Producer(standard_cluster, uri) - consumer_high = Consumer(standard_cluster, uri, [("x <= 0 || x > 0", 2)]) - consumer_low1 = Consumer(standard_cluster, uri, [("x > 0", 1)]) - consumer_low2 = Consumer(standard_cluster, uri, [("x <= 0", 1)]) + posted = [producer.post(x=date.isoformat()) for date in dates] - expected1 = producer.post_diff(num=10, offset=-100) - expected2 = producer.post_diff(num=10, offset=100) + # sample: + # [] [s] [] [] [] [e] [] + # [++++++] + expected = posted[id_start + 1 : id_end] - consumer_high.expect_messages(expected1 + expected2, confirm=False) + # sample: + # [] [s] [] [] [] [e] [] + # [++++] [++++] + remaining = posted[: id_start + 1] + posted[id_end:] - standard_cluster.drain() - leader = standard_cluster.last_known_leader - leader.check_exit_code = False - leader.kill() - leader.wait() + consumer.expect_messages(expected, confirm=True) - # Expressions are reversed here intentionally - consumer_low1.configure(subscriptions=["x <= 0"]) - consumer_low2.configure(subscriptions=["x > 0"]) - consumer_high.close() + consumer.configure([reverse]) + consumer.expect_messages(remaining, confirm=True) - leader = standard_cluster.wait_leader() - assert leader is not None - # Consumer with always false expression in this context. - # Used for synchronization. - Consumer(standard_cluster, uri, ["x > 100000"]) +def test_consumer_multiple_priorities(cluster: Cluster): + """ + Test: messages are routed correctly with overlapping subscriptions with + different priorities. + - Create 1 producer / 2 consumers: C1, C2. + - Open 1 priority queue, provide different overlapping subscriptions + with different priorities. + Consumer C1: high priority, suitable "x" range [1000; +inf) + Consumer C2: low priority, suitable "x" range [0 .......; +inf) + - Producer: produce groups of messages with property "x" from [0; 1000) + range and [1000; +inf) range. + - Consumer C1: expect and confirm all messages with "x" in range [1000; +inf). + - Consumer C2: expect and confirm all messages with "x" in range [0; 1000). - consumer_low1.expect_messages(expected1, confirm=True) - consumer_low2.expect_messages(expected2, confirm=True) + Concerns: + - High priority consumer receives all suitable messages according to + subscription. + - Low priority consumer will receive all suitable messages according to + subscription if there are no conflicting consumers with higher priority. + """ + uri = tc.URI_PRIORITY - expected1 = producer.post_diff(num=10, offset=-200) - expected2 = producer.post_diff(num=10, offset=200) + producer = Producer(cluster, uri) + consumer1 = Consumer(cluster, uri, [("x >= 1000", 5)]) + consumer2 = Consumer(cluster, uri, [("x >= 0", 1)]) - consumer_low1.expect_messages(expected1, confirm=True) - consumer_low2.expect_messages(expected2, confirm=True) + expected = producer.post_diff(num=10, offset=1000) + consumer1.expect_messages(expected, confirm=True) - @pytest.mark.skip("TODO: fix this test") - @tweak.domain.max_delivery_attempts(2) - @tweak.cluster.message_throttle_config( - mqbcfg.MessageThrottleConfig( - high_threshold=1, - low_threshold=1, - high_interval=5000000, - low_interval=5000000, - ) + expected = producer.post_diff(num=10, offset=0) + consumer2.expect_messages(expected, confirm=True) + + expected = producer.post_diff(num=10, offset=2000) + consumer1.expect_messages(expected, confirm=True) + + expected = producer.post_diff(num=10, offset=100) + consumer2.expect_messages(expected, confirm=True) + + +def test_no_capacity_all_optimization(cluster: Cluster): + """ + Test: delivery optimization works during routing when all subscriptions + have no capacity. + + DRQS 171509204 + + - Create 1 producer / 3 consumers: C1, C2, C3. + - Consumers: max_unconfirmed_messages = 1. + - Post several messages to cause NO_CAPACITY_ALL condition. + - Expect delivery optimization log messages when necessary. + + TODO: compare GUIDs of the posted messages and messages on which NO_CAPACITY_ALL + encountered. + + Concerns: + - Delivery optimization works when NO_CAPACITY_ALL condition observed. + """ + uri = tc.URI_PRIORITY + + producer = Producer(cluster, uri) + consumer1 = Consumer(cluster, uri, ["x > 0"], max_unconfirmed_messages=1) + + optimization_monitor = DeliveryOptimizationMonitor(cluster) + + # consumer1: capacity 0/1 + # pending messages: [] + + m1 = producer.post(x=1) + + consumer1.expect_messages([m1], confirm=False) + # consumer1: capacity 1/1 [m1] + # pending messages: [] + + # Do not expect a new delivery optimization message, because + # 'consumer1' just reached the full capacity with the last message 'm1', + # but this is not enough to enable the delivery optimization: need at least + # one message after the added one to see that all consumers are full. + assert not optimization_monitor.has_new_message() + + m2 = producer.post(x=2) + # consumer1: capacity 1/1 [m1] + # pending messages: [m2] + + # Extra message 'm2' is good for 'consumer1' but cannot be delivered: + # no capacity all condition observed. + assert optimization_monitor.has_new_message() + + m3 = producer.post(x=3) # pylint: disable=unused-variable + # consumer1: capacity 1/1 [m1] + # pending messages: [m2, m3] + + # Extra message 'm3' is good for 'consumer1' but cannot be delivered: + # no capacity all condition observed. + assert optimization_monitor.has_new_message() + + consumer1.confirm_all() + consumer1.expect_messages([m2], confirm=False) + # consumer1: capacity 1/1 [m2] + # pending messages: [m3] + + # Message 'm1' was confirmed, so routing is triggered. + # 'consumer1' receives the pending message 'm2' and becomes full again. + # The message 'm3' still exists and is good for 'consumer1', but cannot + # be delivered: no capacity all condition observed. + assert optimization_monitor.has_new_message() + + m4 = producer.post(y=1) + # consumer1: capacity 1/1 [m2] + # pending messages: [m3, m4] + + # Posting the message 'm4' triggers routing, but the only consumer is full: + # no capacity all condition observed. + assert optimization_monitor.has_new_message() + + consumer2 = Consumer(cluster, uri, ["y < 0"], max_unconfirmed_messages=1) + + m5 = producer.post(y=-1) + consumer2.expect_messages([m5], confirm=False) + # consumer1: capacity 1/1 [m2] + # consumer2: capacity 1/1 [m5] + # pending messages: [m3, m4] + + # Do not expect a new delivery optimization message, because + # 'consumer2' just reached the full capacity with the last message 'm5', + # but this is not enough to enable the delivery optimization: need at least + # one message after the added one to see that all consumers are full. + assert not optimization_monitor.has_new_message() + + consumer3 = Consumer(cluster, uri, ["y >= 0"], max_unconfirmed_messages=1) + consumer3.expect_messages([m4], confirm=False) + # consumer1: capacity 1/1 [m2] + # consumer2: capacity 1/1 [m5] + # consumer3: capacity 1/1 [m4] + # pending messages: [m3] + + # Do not expect a new delivery optimization message, because + # 'consumer3' just reached the full capacity with the last message 'm4', + # but this is not enough to enable the delivery optimization: need at least + # one message after the added one to see that all consumers are full. + assert not optimization_monitor.has_new_message() + + +def test_no_capacity_all_fanout(cluster: Cluster): + """ + Test: delivery optimization encountered with one app does not affect + other apps. + + DRQS 171509204 + + - Create 1 producer / 2 consumers: C_foo, C_bar. + - C_foo: max_unconfirmed_messages = 128. + - C_bar: max_unconfirmed_messages = 1. + - Post several messages to cause NO_CAPACITY_ALL condition for C_bar. + - Expect delivery log messages when necessary. + + Concerns: + - Delivery optimization condition encountered with one app does not + affect the other app. + """ + producer_uri = tc.URI_FANOUT + uri_foo = tc.URI_FANOUT_FOO + uri_bar = tc.URI_FANOUT_BAR + + producer = Producer(cluster, producer_uri) + consumer_foo = Consumer(cluster, uri_foo, ["x > 0"], max_unconfirmed_messages=128) + consumer_bar = Consumer(cluster, uri_bar, ["x > 0"], max_unconfirmed_messages=1) + + optimization_monitor = DeliveryOptimizationMonitor(cluster) + + # consumer_foo: capacity 0/128 + # consumer_bar: capacity 0/1 + # pending messages (bar): [] + + m1 = producer.post(x=1) + + consumer_foo.expect_messages([m1], confirm=False) + consumer_bar.expect_messages([m1], confirm=False) + # consumer_foo: capacity 1/128 [m1] + # consumer_bar: capacity 1/1 [m1] + # pending messages (bar): [] + + # Do not expect a new delivery optimization message, because + # 'consumer_bar' just reached the full capacity with the last message 'm1', + # but this is not enough to enable the delivery optimization: need at least + # one message after the added one to see that all consumers are full. + assert not optimization_monitor.has_new_message("bar") + + m2 = producer.post(x=2) + # consumer_foo: capacity 2/128 [m1, m2] + # consumer_bar: capacity 1/1 [m1] + # pending messages (bar): [m2] + + # Extra message 'm2' is good for 'consumer_bar' but cannot be delivered: + # no capacity all condition observed. + assert optimization_monitor.has_new_message("bar") + + m3 = producer.post(x=3) # pylint: disable=unused-variable + # consumer_foo: capacity 3/128 [m1, m2, m3] + # consumer_bar: capacity 1/1 [m1] + # pending messages (bar): [m2, m3] + + # Extra message 'm3' is good for 'consumer_bar' but cannot be delivered: + # no capacity all condition observed. + assert optimization_monitor.has_new_message("bar") + + consumer_bar.confirm_all() + consumer_bar.expect_messages([m2], confirm=False) + # consumer_foo: capacity 3/128 [m1, m2, m3] + # consumer_bar: capacity 1/1 [m2] + # pending messages (bar): [m3] + + # Message 'm1' was confirmed, so routing is triggered. + # 'consumer_bar' receives the pending message 'm2' and becomes full again. + # The message 'm3' still exists and is good for 'consumer_bar', but cannot + # be delivered: no capacity all condition observed. + assert optimization_monitor.has_new_message("bar") + + # App 'foo' has not reached NO_CAPACITY_ALL condition, so there must + # not be any delivery optimization messages. + assert not optimization_monitor.has_new_message("foo") + + # App 'baz' does not have any consumers. It will have the NO_CAPACITY_ALL + # condition, because it's impossible to have any progress. + assert optimization_monitor.has_new_message("baz") + + +def test_primary_node_crash(multi_node: Cluster): + """ + Test: configured subscriptions work after primary node crash. + - Create 1 producer / N consumers. + - Open 1 priority queue, provide non-overlapping subscriptions. + - Produce messages [A] suitable for each consumer. + - Expect but not confirm suitable messages from [A] for each consumer. + + - Kill primary node. + - Wait for a new leader. + + - Expect and confirm messages from [A] for each consumer. + - Produce messages [B] suitable for each consumer. + - Expect and confirm messages from [B] for each consumer. + + Concerns: + - Message listing works after primary node crash and change. + - Already configured subscriptions work after primary node crash and + change. + """ + uri = tc.URI_PRIORITY + + producer = Producer(multi_node, uri) + consumer1 = Consumer(multi_node, uri, ["x < 0"]) + consumer2 = Consumer(multi_node, uri, ["x > 0"]) + consumer3 = Consumer(multi_node, uri, ["x == 0"]) + + expected1 = producer.post_diff(num=10, offset=-100) + expected2 = producer.post_diff(num=10, offset=100) + expected3 = producer.post_same(num=10, value=0) + + consumer1.expect_messages(expected1, confirm=False) + consumer2.expect_messages(expected2, confirm=False) + consumer3.expect_messages(expected3, confirm=False) + + multi_node.drain() + leader = multi_node.last_known_leader + leader.check_exit_code = False + leader.kill() + leader.wait() + + leader = multi_node.wait_leader() + assert leader is not None + + # Consumer with always false expression in this context. + # Used for synchronization. + Consumer(multi_node, uri, ["x > 100000"]) + + consumer1.expect_messages(expected1, confirm=True, timeout=0) + consumer2.expect_messages(expected2, confirm=True, timeout=0) + consumer3.expect_messages(expected3, confirm=True, timeout=0) + + expected1 = producer.post_diff(num=10, offset=-200) + expected2 = producer.post_diff(num=10, offset=200) + expected3 = producer.post_same(num=10, value=0) + + consumer1.expect_messages(expected1, confirm=True) + consumer2.expect_messages(expected2, confirm=True) + consumer3.expect_messages(expected3, confirm=True) + + +def test_redelivery_on_primary_node_crash(multi_node: Cluster): + """ + Test: configured subscriptions work after primary node crash. + - Create 1 producer / 3 consumers: C_high, C_low1, C_low2. + - Open 1 priority queue. + Consumer C_high: high priority, expression (-inf ; +inf) + Consumer C_low1: low priority, expression (-inf; 0] + Consumer C_low2: low priority, expression (0; +inf) + - Produce messages [A1] suitable both for C_high and C_low1. + - Produce messages [B1] suitable both for C_high and C_low2. + - Expect but not confirm messages [A1, B1] for consumer C_high. + + - Kill primary node. + - Close consumer C_high. + - Wait for a new leader. + + - Expect and confirm messages [A1] for consumer C_low1. + - Expect and confirm messages [B1] for consumer C_low2. + - Produce messages [A2] suitable both for C_low1. + - Produce messages [B2] suitable both for C_low2. + - Expect and confirm messages [A2] for consumer C_low1. + - Expect and confirm messages [B2] for consumer C_low2. + + Concerns: + - Message re-routing works after primary node crash and change. + - Already configured subscriptions work after primary node crash and + change. + """ + uri = tc.URI_PRIORITY + + producer = Producer(multi_node, uri) + consumer_high = Consumer(multi_node, uri, [("x <= 0 || x > 0", 2)]) + consumer_low1 = Consumer(multi_node, uri, [("x <= 0", 1)]) + consumer_low2 = Consumer(multi_node, uri, [("x > 0", 1)]) + + expected1 = producer.post_diff(num=10, offset=-100) + expected2 = producer.post_diff(num=10, offset=100) + + consumer_high.expect_messages(expected1 + expected2, confirm=False) + + multi_node.drain() + leader = multi_node.last_known_leader + leader.check_exit_code = False + leader.kill() + leader.wait() + + consumer_high.close() + + leader = multi_node.wait_leader() + assert leader is not None + + # Consumer with always false expression in this context. + # Used for synchronization. + Consumer(multi_node, uri, ["x > 100000"]) + + consumer_low1.expect_messages(expected1, confirm=True) + consumer_low2.expect_messages(expected2, confirm=True) + + expected1 = producer.post_diff(num=10, offset=-200) + expected2 = producer.post_diff(num=10, offset=200) + + consumer_low1.expect_messages(expected1, confirm=True) + consumer_low2.expect_messages(expected2, confirm=True) + + +def test_reconfigure_on_primary_node_crash(multi_node: Cluster): + """ + Test: configured subscriptions work after primary node crash. + - Create 1 producer / 3 consumers: C_high, C_low1, C_low2. + - Open 1 priority queue. + Consumer C_high: high priority, expression (-inf ; +inf) + Consumer C_low1: low priority, expression (0; +inf) + Consumer C_low2: low priority, expression (-inf; 0] + - Produce messages [A1] suitable both for C_high and C_low1. + - Produce messages [B1] suitable both for C_high and C_low2. + - Expect but not confirm messages [A1, B1] for consumer C_high. + + - Kill primary node. + - Close consumer C_high. + - Reconfigure consumers C_low1, C_low2: reverse expressions. + Consumer C_low1: low priority, expression (-inf; 0] + Consumer C_low2: low priority, expression (0; +inf) + - Wait for a new leader. + + - Expect and confirm messages [B1] for consumer C_low1. + - Expect and confirm messages [A1] for consumer C_low2. + - Produce messages [B2] suitable both for C_low1. + - Produce messages [A2] suitable both for C_low2. + - Expect and confirm messages [B2] for consumer C_low1. + - Expect and confirm messages [A2] for consumer C_low2. + + Concerns: + - Configure works after primary node crash and change. + """ + uri = tc.URI_PRIORITY + + producer = Producer(multi_node, uri) + consumer_high = Consumer(multi_node, uri, [("x <= 0 || x > 0", 2)]) + consumer_low1 = Consumer(multi_node, uri, [("x > 0", 1)]) + consumer_low2 = Consumer(multi_node, uri, [("x <= 0", 1)]) + + expected1 = producer.post_diff(num=10, offset=-100) + expected2 = producer.post_diff(num=10, offset=100) + + consumer_high.expect_messages(expected1 + expected2, confirm=False) + + multi_node.drain() + leader = multi_node.last_known_leader + leader.check_exit_code = False + leader.kill() + leader.wait() + + # Expressions are reversed here intentionally + consumer_low1.configure(subscriptions=["x <= 0"]) + consumer_low2.configure(subscriptions=["x > 0"]) + consumer_high.close() + + leader = multi_node.wait_leader() + assert leader is not None + + # Consumer with always false expression in this context. + # Used for synchronization. + Consumer(multi_node, uri, ["x > 100000"]) + + consumer_low1.expect_messages(expected1, confirm=True) + consumer_low2.expect_messages(expected2, confirm=True) + + expected1 = producer.post_diff(num=10, offset=-200) + expected2 = producer.post_diff(num=10, offset=200) + + consumer_low1.expect_messages(expected1, confirm=True) + consumer_low2.expect_messages(expected2, confirm=True) + + +@pytest.mark.skip("TODO: fix this test") +@tweak.domain.max_delivery_attempts(2) +@tweak.cluster.message_throttle_config( + mqbcfg.MessageThrottleConfig( + high_threshold=1, + low_threshold=1, + high_interval=5000000, + low_interval=5000000, ) - def test_poison(self, cluster: Cluster): - """ - Test: poison messages detection works for subscriptions. - - Configure max delivery attempts = 1 and adjust message throttling - parameters for long delay if poisoned message found. - - Create 1 producer / 2 consumers: C1, C2. - - Open 1 priority queue, consumers have non-overlapping subscriptions. - - Produce 2 messages: M1, M2. - - Expect M1 for consumer C1, M2 for consumer C2, but not confirm. - - Kill consumer C1. - - Produce 1 message M3 suitable for consumer C2. - - Consumer C2: expect and confirm only message M2 (M3 is not delivered - because of the delay). - - Producer -> M1, M2 - Consumer C1 <- M1 - Consumer C2 <- M2 - kill C1 - message M1 is marked as poisoned, the queue is suspended for delay - Producer -> M3 - Consumer C2 <- M2 - delay, M3 is not delivered to C2 - - Concerns: - - Delay condition works for the entire queue, not just for concrete - subscriptions. - """ - uri = tc.URI_PRIORITY - - producer = Producer(cluster, uri) - - consumer1 = Consumer(cluster, uri, ["x <= 0"]) - consumer2 = Consumer(cluster, uri, ["x > 0"]) - expected1 = [producer.post(x=0)] - expected2 = [producer.post(x=1)] - - consumer1.expect_messages(expected1, confirm=False) - consumer2.expect_messages(expected2, confirm=False) - - consumer1.kill() - - # The queue must be suspended for some time, so 'delayed' should not - # be delivered fast. - producer.post(x=2) - delayed = [producer.post(x=2)] # pylint: disable=unused-variable - - consumer2.expect_messages(expected2, confirm=True) +) +def test_poison(cluster: Cluster): + """ + Test: poison messages detection works for subscriptions. + - Configure max delivery attempts = 1 and adjust message throttling + parameters for long delay if poisoned message found. + - Create 1 producer / 2 consumers: C1, C2. + - Open 1 priority queue, consumers have non-overlapping subscriptions. + - Produce 2 messages: M1, M2. + - Expect M1 for consumer C1, M2 for consumer C2, but not confirm. + - Kill consumer C1. + - Produce 1 message M3 suitable for consumer C2. + - Consumer C2: expect and confirm only message M2 (M3 is not delivered + because of the delay). + + Producer -> M1, M2 + Consumer C1 <- M1 + Consumer C2 <- M2 + kill C1 + message M1 is marked as poisoned, the queue is suspended for delay + Producer -> M3 + Consumer C2 <- M2 + delay, M3 is not delivered to C2 + + Concerns: + - Delay condition works for the entire queue, not just for concrete + subscriptions. + """ + uri = tc.URI_PRIORITY + + producer = Producer(cluster, uri) + + consumer1 = Consumer(cluster, uri, ["x <= 0"]) + consumer2 = Consumer(cluster, uri, ["x > 0"]) + expected1 = [producer.post(x=0)] + expected2 = [producer.post(x=1)] + + consumer1.expect_messages(expected1, confirm=False) + consumer2.expect_messages(expected2, confirm=False) + + consumer1.kill() + + # The queue must be suspended for some time, so 'delayed' should not + # be delivered fast. + producer.post(x=2) + delayed = [producer.post(x=2)] # pylint: disable=unused-variable + + consumer2.expect_messages(expected2, confirm=True)