diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23fe15c..8b6358c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,12 +70,12 @@ jobs: - name: Run stubs check run: python3 tests/stubs_check.py - - name: Run examples check - run: python3 tests/examples_check.py - - name: Install pytest run: pip3 install pytest pytest-xdist + - name: Run examples check + run: pytest tests/examples_check.py -v --durations=0 + - name: Run pytest run: pytest -n auto --import-mode=append diff --git a/requirements-dev.txt b/requirements-dev.txt index ceccb8a..bfeb1fc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ maturin>1 +fixtures diff --git a/tests/examples_check.py b/tests/examples_check.py index 8f12e84..20b02e0 100644 --- a/tests/examples_check.py +++ b/tests/examples_check.py @@ -3,7 +3,11 @@ import time from os import getpgid, killpg, path from signal import SIGINT -from subprocess import PIPE, Popen +from subprocess import PIPE, Popen, TimeoutExpired + +import fixtures +import pytest +from testtools.content import text_content # Contributors: # ZettaScale Zenoh team, @@ -21,7 +25,7 @@ ret = "\r\n" -class Pyrun: +class Pyrun(fixtures.Fixture): def __init__(self, p, args=None) -> None: if args is None: args = [] @@ -29,7 +33,6 @@ def __init__(self, p, args=None) -> None: print(f"starting {self.name}") self.process: Popen = Popen( ["python3", path.join(examples, p), *args], - stdin=PIPE, stdout=PIPE, stderr=PIPE, start_new_session=True, @@ -57,8 +60,8 @@ def status(self, expecting=0): def wait(self): try: code = self.process.wait(timeout=10) - except: - self.process.kill() + except TimeoutExpired: + self.process.send_signal(SIGINT) code = self.process.wait(timeout=10) if self.end is None: self.end = time.time() @@ -89,207 +92,313 @@ def time(self): return None if self.end is None else (self.end - self.start) -errors = [] +class ZQueryableFixture(Pyrun): + + def __init__(self, key=""): + self.key = key + + def _setUp(self): + super().__init__("z_queryable.py", args=[f"-k={self.key}"]) + self.addCleanup(self.process.send_signal, SIGINT) + + +class ZStorageFixture(Pyrun): + + def _setUp(self): + super().__init__("z_storage.py") + self.addCleanup(self.process.send_signal, SIGINT) + self.addDetail("message", text_content(self.process.stdout)) + + +class ZBytesFixture(Pyrun): + + def __init__(self): + super().__init__("z_bytes.py") + + def _setUp(self): + # super().__init__("z_bytes.py") + self.addCleanup(self.process.send_signal, SIGINT) + self.addDetail("message", text_content(self.process.stdout)) + + +class ZInfoFixture(Pyrun): + def __init__(self): + super().__init__("z_info.py") + + def _setUp(self): + self.addCleanup(self.process.send_signal, SIGINT) + self.addDetail("message", text_content(self.process.stdout)) + + +class ZScoutFixture(Pyrun): + def __init__(self): + super().__init__("z_scout.py") + + def _setUp(self): + self.addCleanup(self.process.send_signal, SIGINT) + self.addDetail("message", text_content(self.process.stdout)) + + +class ZSubFixture(Pyrun): + def __init__(self): + super().__init__("z_sub.py") + + def _setUp(self): + self.addCleanup(self.process.send_signal, SIGINT) + self.addDetail("message", text_content(self.process.stdout)) + + +class ZDeleteFixture(Pyrun): + def __init__(self): + super().__init__("z_delete.py") + + def _setUp(self): + self.addCleanup(self.process.send_signal, SIGINT) + self.addDetail("message", text_content(self.process.stdout)) + + +@pytest.fixture +def z_pub(args=["--iter=2"]): + return Pyrun("z_pub.py", args) + + +class ZPutFixture(Pyrun): + + def __init__(self): + super().__init__("z_put.py") + + def _setUp(self): + self.addCleanup(self.process.send_signal, SIGINT) + self.addDetail("message", text_content(self.process.stdout)) + + +@pytest.fixture +def z_get(args=["-s=demo/example/zenoh-python-queryable"]): + return Pyrun("z_get.py", args) + + +def test_z_bytes(): + z_bytes = ZBytesFixture() + errors = [] + if sys.version_info >= (3, 9): + # Test z_bytes + print("=> Test z_bytes") + if error := z_bytes.status(): + z_bytes.dbg() + errors.append(error) + + assert not errors + -if sys.version_info >= (3, 9): - # Test z_bytes - print("=> Test z_bytes") - z_bytes = Pyrun("z_bytes.py") - if error := z_bytes.status(): - z_bytes.dbg() +def test_z_info_z_scout(): + z_info = ZInfoFixture() + z_scout = ZScoutFixture() + errors = [] + if error := z_info.status(): + z_info.dbg() + errors.append(error) + + if error := z_scout.status(): + z_scout.dbg() + errors.append(error) + + assert not errors + + +def test_z_get_z_queryable(): + """Test z_get & z_queryable""" + # print("=> Test z_get & z_queryable") + errors = [] + z_queryable = ZQueryableFixture("demo/example/zenoh-python-queryable") + z_queryable.setUp() + time.sleep(3) + ## z_get: Able to get reply from queryable + z_get = Pyrun("z_get.py", ["-s=demo/example/zenoh-python-queryable"]) + if error := z_get.status(): + z_get.dbg() + errors.append(error) + + z_queryable.cleanUp() + + if not ( + "Received ('demo/example/zenoh-python-queryable': 'Queryable from Python!')" + in "".join(z_get.stdout) + ): + z_get.dbg() + z_queryable.dbg() + errors.append("z_get didn't get a response from z_queryable") + queryableout = "".join(z_queryable.stdout) + if not ("Received Query 'demo/example/zenoh-python-queryable'" in queryableout): + errors.append("z_queryable didn't catch query") + if any(("z_queryable" in error) for error in errors): + z_queryable.dbg() + + assert not errors + + +def test_z_storage_z_sub(): + print("=> Test z_storage & z_sub") + errors = [] + z_storage = ZStorageFixture("z_storage.py") + z_sub = ZSubFixture() + time.sleep(3) + ## z_put: Put one message (to storage & sub) + z_put = ZPutFixture() + time.sleep(1) + ## z_pub: Put two messages (to storage & sub) + pub = Pyrun("z_pub.py", ["--iter=2"]) + time.sleep(1) + z_get = Pyrun("z_get.py", ["-s=demo/example/zenoh-python-put"]) + if error := z_put.status(): + z_put.dbg() + errors.append(error) + + if error := z_get.status(): + z_get.dbg() + errors.append(error) + + if not ( + "Received ('demo/example/zenoh-python-put': 'Put from Python!')" + in "".join(z_get.stdout) + ): + z_get.dbg() + errors.append("z_get didn't get a response from z_storage about put") + if any(("z_get" in error) for error in errors): + z_get.dbg() + time.sleep(1) + + z_delete = ZDeleteFixture() + if error := z_delete.status(): + z_delete.dbg() + errors.append(error) + time.sleep(1) + + ## z_get: Unable to get put from storage + z_get = Pyrun("z_get.py", ["-s=demo/example/zenoh-python-put"]) + if error := z_get.status(): + z_get.dbg() + errors.append(error) + if "Received ('demo/example/zenoh-python-put': 'Put from Python!')" in "".join( + z_get.stdout + ): + z_storage.dbg() + errors.append("z_get did get a response from z_storage about put after delete") + if any(("z_get" in error) for error in errors): + z_get.dbg() + time.sleep(1) + + ## z_sub: Should receive put, pub and delete + if error := z_sub.process.send_signal(SIGINT): + z_sub.dbg() + errors.append(error) + subout = "".join(z_sub.stdout) + if not ( + "Received SampleKind.PUT ('demo/example/zenoh-python-put': 'Put from Python!')" + in subout + ): + errors.append("z_sub didn't catch put") + if not ( + "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" + in subout + ): + errors.append("z_sub didn't catch second z_pub") + if not ( + "Received SampleKind.DELETE ('demo/example/zenoh-python-put': '')" in subout + ): + errors.append("z_sub didn't catch delete") + if any(("z_sub" in error) for error in errors): + z_sub.dbg() + + ## z_storage: Should receive put, pub, delete, and query + if error := z_storage.process.send_signal(SIGINT): + z_storage.dbg() + errors.append(error) + storageout = "".join(z_storage.stdout) + if not ( + "Received SampleKind.PUT ('demo/example/zenoh-python-put': 'Put from Python!')" + in storageout + ): + errors.append("z_storage didn't catch put") + if not ( + "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" + in storageout + ): + errors.append("z_storage didn't catch second z_pub") + if not ( + "Received SampleKind.DELETE ('demo/example/zenoh-python-put': '')" in storageout + ): + errors.append("z_storage didn't catch delete") + if not ("Received Query 'demo/example/zenoh-python-put'" in storageout): + errors.append("z_storage didn't catch query") + if any(("z_storage" in error) for error in errors): + z_storage.dbg() + + assert not errors + + +def test_z_pull_z_sub_queued(): + print("=> Test z_pull & z_sub_queued") + errors = [] + ## Run z_pull and z_sub_queued + sub_queued = Pyrun("z_sub_queued.py") + time.sleep(3) + pull = Pyrun("z_pull.py", ["--size=1", "--interval=1"]) + time.sleep(3) + ## z_pub: Put two messages (to storage & sub) + pub = Pyrun("z_pub.py", ["--iter=2", "--interval=0"]) + if error := pub.status(): + pub.dbg() + errors.append(error) + ## z_sub_queued: Should receive two messages + if error := sub_queued.interrupt(): + sub_queued.dbg() + errors.append(error) + sub_queued_out = "".join(sub_queued.stdout) + if not ( + "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 0] Pub from Python!')" + in sub_queued_out + ): + errors.append("z_sub_queued didn't catch the first z_pub") + if not ( + "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" + in sub_queued_out + ): + errors.append("z_sub_queued didn't catch the second z_pub") + if any(("z_sub_queued" in error) for error in errors): + sub_queued.dbg() + ## z_pull: Should only receive the last messages + time.sleep(3) + if error := pull.interrupt(): + pull.dbg() + errors.append(error) + pullout = "".join(pull.stdout) + if ( + "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 0] Pub from Python!')" + in pullout + ): + errors.append("z_pull shouldn't catch the old z_pub") + if not ( + "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" + in pullout + ): + errors.append("z_pull didn't catch the last z_pub") + if any(("z_pull" in error) for error in errors): + pull.dbg() + + assert not errors + + +def test_z_sub_thr_z_pub_thr(): + print("=> Test z_sub_thr & z_pub_thr") + errors = [] + sub_thr = Pyrun("z_sub_thr.py") + pub_thr = Pyrun("z_pub_thr.py", ["128"]) + time.sleep(5) + if error := sub_thr.interrupt(): + sub_thr.dbg() + errors.append(error) + if error := pub_thr.interrupt(): + pub_thr.dbg() errors.append(error) -# Test z_info & z_scout -print("=> Test z_info & z_scout") -info = Pyrun("z_info.py") -if error := info.status(): - info.dbg() - errors.append(error) -scout = Pyrun("z_scout.py") -if error := scout.status(): - scout.dbg() - errors.append(error) - -# Test z_get & z_queryable -print("=> Test z_get & z_queryable") -## Run z_queryable -queryable = Pyrun("z_queryable.py", ["-k=demo/example/zenoh-python-queryable"]) -time.sleep(3) -## z_get: Able to get reply from queryable -get = Pyrun("z_get.py", ["-s=demo/example/zenoh-python-queryable"]) -if error := get.status(): - get.dbg() - errors.append(error) -if not ( - "Received ('demo/example/zenoh-python-queryable': 'Queryable from Python!')" - in "".join(get.stdout) -): - get.dbg() - queryable.dbg() - errors.append("z_get didn't get a response from z_queryable") -## Stop z_queryable -if error := queryable.interrupt(): - queryable.dbg() - errors.append(error) -queryableout = "".join(queryable.stdout) -if not ("Received Query 'demo/example/zenoh-python-queryable'" in queryableout): - errors.append("z_queryable didn't catch query") -if any(("z_queryable" in error) for error in errors): - queryable.dbg() - -# Test z_storage & z_sub -print("=> Test z_storage & z_sub") -storage = Pyrun("z_storage.py") -sub = Pyrun("z_sub.py") -time.sleep(3) -## z_put: Put one message (to storage & sub) -put = Pyrun("z_put.py") -if error := put.status(): - put.dbg() - errors.append(error) -time.sleep(3) -## z_pub: Put two messages (to storage & sub) -pub = Pyrun("z_pub.py", ["--iter=2"]) -time.sleep(4) -## z_get: Able to get put from storage -get = Pyrun("z_get.py", ["-s=demo/example/zenoh-python-put"]) -if error := get.status(): - get.dbg() - errors.append(error) -if not ( - "Received ('demo/example/zenoh-python-put': 'Put from Python!')" - in "".join(get.stdout) -): - get.dbg() - errors.append("z_get didn't get a response from z_storage about put") -if any(("z_get" in error) for error in errors): - get.dbg() -time.sleep(3) -## z_delete: Delete put in storage -delete = Pyrun("z_delete.py") -if error := delete.status(): - delete.dbg() - errors.append(error) -time.sleep(3) -## z_get: Unable to get put from storage -get = Pyrun("z_get.py", ["-s=demo/example/zenoh-python-put"]) -if error := get.status(): - get.dbg() - errors.append(error) -if "Received ('demo/example/zenoh-python-put': 'Put from Python!')" in "".join( - get.stdout -): - # storage.dbg() - errors.append("z_get did get a response from z_storage about put after delete") -if any(("z_get" in error) for error in errors): - get.dbg() -time.sleep(3) -## z_sub: Should receive put, pub and delete -if error := sub.interrupt(): - sub.dbg() - errors.append(error) -subout = "".join(sub.stdout) -if not ( - "Received SampleKind.PUT ('demo/example/zenoh-python-put': 'Put from Python!')" - in subout -): - errors.append("z_sub didn't catch put") -if not ( - "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" - in subout -): - errors.append("z_sub didn't catch second z_pub") -if not ("Received SampleKind.DELETE ('demo/example/zenoh-python-put': '')" in subout): - errors.append("z_sub didn't catch delete") -if any(("z_sub" in error) for error in errors): - sub.dbg() -## z_storage: Should receive put, pub, delete, and query -if error := storage.interrupt(): - storage.dbg() - errors.append(error) -storageout = "".join(storage.stdout) -if not ( - "Received SampleKind.PUT ('demo/example/zenoh-python-put': 'Put from Python!')" - in storageout -): - errors.append("z_storage didn't catch put") -if not ( - "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" - in storageout -): - errors.append("z_storage didn't catch second z_pub") -if not ( - "Received SampleKind.DELETE ('demo/example/zenoh-python-put': '')" in storageout -): - errors.append("z_storage didn't catch delete") -if not ("Received Query 'demo/example/zenoh-python-put'" in storageout): - errors.append("z_storage didn't catch query") -if any(("z_storage" in error) for error in errors): - storage.dbg() - -# Test z_pull & s_sub_queued -print("=> Test z_pull & z_sub_queued") -## Run z_pull and z_sub_queued -sub_queued = Pyrun("z_sub_queued.py") -time.sleep(3) -pull = Pyrun("z_pull.py", ["--size=1", "--interval=1"]) -time.sleep(3) -## z_pub: Put two messages (to storage & sub) -pub = Pyrun("z_pub.py", ["--iter=2", "--interval=0"]) -if error := pub.status(): - pub.dbg() - errors.append(error) -## z_sub_queued: Should receive two messages -if error := sub_queued.interrupt(): - sub_queued.dbg() - errors.append(error) -sub_queued_out = "".join(sub_queued.stdout) -if not ( - "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 0] Pub from Python!')" - in sub_queued_out -): - errors.append("z_sub_queued didn't catch the first z_pub") -if not ( - "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" - in sub_queued_out -): - errors.append("z_sub_queued didn't catch the second z_pub") -if any(("z_sub_queued" in error) for error in errors): - sub_queued.dbg() -## z_pull: Should only receive the last messages -time.sleep(3) -if error := pull.interrupt(): - pull.dbg() - errors.append(error) -pullout = "".join(pull.stdout) -if ( - "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 0] Pub from Python!')" - in pullout -): - errors.append("z_pull shouldn't catch the old z_pub") -if not ( - "Received SampleKind.PUT ('demo/example/zenoh-python-pub': '[ 1] Pub from Python!')" - in pullout -): - errors.append("z_pull didn't catch the last z_pub") -if any(("z_pull" in error) for error in errors): - pull.dbg() - -# Test z_sub_thr & z_pub_thr -print("=> Test z_sub_thr & z_pub_thr") -sub_thr = Pyrun("z_sub_thr.py") -pub_thr = Pyrun("z_pub_thr.py", ["128"]) -time.sleep(5) -if error := sub_thr.interrupt(): - sub_thr.dbg() - errors.append(error) -if error := pub_thr.interrupt(): - pub_thr.dbg() - errors.append(error) - - -if len(errors): - message = f"Found {len(errors)} errors: {(ret+tab) + (ret+tab).join(errors)}" - raise Exception(message) -else: - print("Pass examples_check") + assert not errors