diff --git a/Makefile b/Makefile index b6f12dd..7db9fc7 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,6 @@ install: install -m 00644 $(top_srcdir)/tools/kirk/libkirk/*.py $(BASE_DIR)/libkirk install -m 00775 $(top_srcdir)/tools/kirk/kirk $(BASE_DIR)/kirk + cd $(BASE_DIR) && ln -sf kirk runltp-ng + include $(top_srcdir)/include/mk/generic_leaf_target.mk diff --git a/README.md b/README.md index a88754a..72e5c56 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Some basic commands are the following: # run LTP syscalls testing suite via SSH ./kirk --framework ltp \ - --sut=ssh:host myhost.com:user=root:key_file=myhost_id_rsa \ + --sut ssh:host=myhost.com:user=root:key_file=myhost_id_rsa \ --run-suite syscalls # run LTP syscalls testing suite in parallel on host using 16 workers @@ -101,7 +101,7 @@ Some basic commands are the following: # run LTP syscalls testing suite in parallel via SSH using 16 workers ./kirk --framework ltp \ - --sut=ssh:host myhost.com:user=root:key_file=myhost_id_rsa \ + --sut ssh:host=myhost.com:user=root:key_file=myhost_id_rsa \ --run-suite syscalls --workers 16 It's possible to run a single command before running testing suites using diff --git a/libkirk/data.py b/libkirk/data.py index e446667..b0f4c11 100644 --- a/libkirk/data.py +++ b/libkirk/data.py @@ -122,3 +122,17 @@ def env(self): Environment variables """ return self._env + + @property + def full_command(self): + """ + Return the full command, with arguments as well. + For example, if `command="ls"` and `arguments="-l -a"`, + `full_command="ls -l -a"`. + """ + cmd = self.command + if len(self.arguments) > 0: + cmd += ' ' + cmd += ' '.join(self.arguments) + + return cmd diff --git a/libkirk/framework.py b/libkirk/framework.py index e2f359e..12ce0a9 100644 --- a/libkirk/framework.py +++ b/libkirk/framework.py @@ -34,6 +34,19 @@ async def get_suites(self, sut: SUT) -> list: """ raise NotImplementedError() + async def find_command(self, sut: SUT, command: str) -> Test: + """ + Search for command inside Framework folder and, if it's not found, + search for command in the operating system. Then return a Test object + which can be used to execute command. + :param sut: SUT object to communicate with + :type sut: SUT + :param command: command to execute + :type command: str + :returns: Test + """ + raise NotImplementedError() + async def find_suite(self, sut: SUT, name: str) -> Suite: """ Search for suite with given name inside SUT. diff --git a/libkirk/host.py b/libkirk/host.py index 7baf952..b43df68 100644 --- a/libkirk/host.py +++ b/libkirk/host.py @@ -144,7 +144,7 @@ async def run_command( try: kwargs = { "stdout": asyncio.subprocess.PIPE, - "stderr": asyncio.subprocess.PIPE, + "stderr": asyncio.subprocess.STDOUT, "cwd": cwd, "preexec_fn": os.setsid } diff --git a/libkirk/kselftests.py b/libkirk/kselftests.py index 681fa3a..90f17dd 100644 --- a/libkirk/kselftests.py +++ b/libkirk/kselftests.py @@ -6,6 +6,7 @@ .. moduleauthor:: Andrea Cervesato """ import os +import shlex import logging from libkirk import KirkException from libkirk.sut import SUT @@ -116,6 +117,43 @@ async def get_suites(self, sut: SUT) -> list: return ["cgroup", "bpf"] + async def find_command(self, sut: SUT, command: str) -> Test: + if not sut: + raise ValueError("SUT is None") + + if not command: + raise ValueError("command is empty") + + cmd_args = shlex.split(command) + suite_folder = None + + for suite in await self.get_suites(sut): + folder = os.path.join(self._root, suite) + binary = os.path.join(folder, cmd_args[0]) + + ret = await sut.run_command(f"test -f {binary}") + if ret["returncode"] == 0: + suite_folder = folder + break + + cwd = None + env = None + + ret = await sut.run_command(f"test -d {suite_folder}") + if ret["returncode"] == 0: + cwd = suite_folder + env={"PATH": suite_folder} + + test = Test( + name=cmd_args[0], + cmd=cmd_args[0], + args=cmd_args[1:] if len(cmd_args) > 0 else None, + cwd=cwd, + env=env, + parallelizable=False) + + return test + async def find_suite(self, sut: SUT, name: str) -> Suite: if not sut: raise ValueError("SUT is None") diff --git a/libkirk/liburing.py b/libkirk/liburing.py index e7bf428..f380e22 100644 --- a/libkirk/liburing.py +++ b/libkirk/liburing.py @@ -7,6 +7,7 @@ """ import os import re +import shlex import logging from libkirk.sut import SUT from libkirk.data import Test @@ -101,6 +102,32 @@ async def _is_parallelizable(sut: SUT, cmd: str) -> bool: return parallel + async def find_command(self, sut: SUT, command: str) -> Test: + if not sut: + raise ValueError("SUT is None") + + if not command: + raise ValueError("command is empty") + + cmd_args = shlex.split(command) + cwd = None + env = None + + ret = await sut.run_command(f"test -d {self._root}") + if ret["returncode"] == 0: + cwd = self._root + env={"PATH": self._root} + + test = Test( + name=cmd_args[0], + cmd=cmd_args[0], + args=cmd_args[1:] if len(cmd_args) > 0 else None, + cwd=cwd, + env=env, + parallelizable=False) + + return test + async def find_suite(self, sut: SUT, name: str) -> Suite: if not sut: raise ValueError("SUT is None") diff --git a/libkirk/ltp.py b/libkirk/ltp.py index dd3c908..3d25ee4 100644 --- a/libkirk/ltp.py +++ b/libkirk/ltp.py @@ -8,6 +8,7 @@ import os import re import json +import shlex import logging from libkirk.results import TestResults from libkirk.results import ResultStatus @@ -39,6 +40,7 @@ def __init__(self) -> None: self._root = None self._env = None self._max_runtime = None + self._tc_folder = None @ property def config_help(self) -> dict: @@ -68,6 +70,8 @@ def setup(self, **kwargs: dict) -> None: self._root = root self._env["LTPROOT"] = self._root + self._tc_folder = os.path.join(self._root, "testcases", "bin") + runtime = kwargs.get("max_runtime", None) if runtime: @@ -82,11 +86,9 @@ async def _read_path(self, sut: SUT) -> dict: """ Read PATH and initialize it with testcases folder as well. """ - tc_path = os.path.join(self._root, "testcases", "bin") - env = self._env.copy() if 'PATH' in env: - env["PATH"] = env["PATH"] + f":{tc_path}" + env["PATH"] = env["PATH"] + f":{self._tc_folder}" else: ret = await sut.run_command("echo -n $PATH") if ret["returncode"] != 0: @@ -145,7 +147,6 @@ async def _read_runtest( tests = [] lines = content.split('\n') - tc_path = os.path.join(self._root, "testcases", "bin") for line in lines: if not line.strip() or line.strip().startswith("#"): @@ -199,7 +200,7 @@ async def _read_runtest( name=test_name, cmd=test_cmd, args=test_args, - cwd=tc_path, + cwd=self._tc_folder, env=env, parallelizable=parallelizable) @@ -241,6 +242,32 @@ async def get_suites(self, sut: SUT) -> list: suites = [line for line in stdout.split('\n') if line] return suites + async def find_command(self, sut: SUT, command: str) -> Test: + if not sut: + raise ValueError("SUT is None") + + if not command: + raise ValueError("command is empty") + + cmd_args = shlex.split(command) + cwd = None + env = None + + ret = await sut.run_command(f"test -d {self._tc_folder}") + if ret["returncode"] == 0: + cwd = self._tc_folder + env = await self._read_path(sut) + + test = Test( + name=cmd_args[0], + cmd=cmd_args[0], + args=cmd_args[1:] if len(cmd_args) > 0 else None, + cwd=cwd, + env=env, + parallelizable=False) + + return test + async def find_suite(self, sut: SUT, name: str) -> Suite: if not sut: raise ValueError("SUT is None") diff --git a/libkirk/scheduler.py b/libkirk/scheduler.py index 9bc39f7..4704a27 100644 --- a/libkirk/scheduler.py +++ b/libkirk/scheduler.py @@ -131,18 +131,6 @@ def __init__(self, **kwargs: dict) -> None: if not self._framework: raise ValueError("Framework object is empty") - @ staticmethod - def _command_from_test(test: Test) -> str: - """ - Returns a command from test. - """ - cmd = test.command - if len(test.arguments) > 0: - cmd += ' ' - cmd += ' '.join(test.arguments) - - return cmd - async def _get_tainted_status(self) -> tuple: """ Check tainted status of the Kernel. @@ -220,7 +208,7 @@ async def _run_test(self, test: Test, sem: asyncio.Semaphore) -> None: await self._write_kmsg(test) iobuffer = RedirectTestStdout(test) - cmd = self._command_from_test(test) + cmd = test.full_command start_t = time.time() exec_time = 0 test_data = None diff --git a/libkirk/session.py b/libkirk/session.py index 3774beb..a54783e 100644 --- a/libkirk/session.py +++ b/libkirk/session.py @@ -235,9 +235,13 @@ async def _exec_command(self, command: str) -> None: try: await libkirk.events.fire("run_cmd_start", command) + test = await self._framework.find_command(self._sut, command) + ret = await asyncio.wait_for( self._sut.run_command( - command, + test.full_command, + cwd=test.cwd, + env=test.env, iobuffer=RedirectSUTStdout(self._sut, True)), timeout=self._exec_timeout ) diff --git a/libkirk/sut.py b/libkirk/sut.py index 33c1ce9..077da21 100644 --- a/libkirk/sut.py +++ b/libkirk/sut.py @@ -253,7 +253,13 @@ async def get_tainted_info(self) -> tuple: stdout = ret["stdout"].rstrip() tainted_num = len(TAINED_MSG) - code = int(stdout.rstrip()) + code = stdout.rstrip() + + # output is likely message in stderr + if not code.isdigit(): + raise SUTError(code) + + code = int(code) bits = format(code, f"0{tainted_num}b")[::-1] messages = [] diff --git a/libkirk/tests/conftest.py b/libkirk/tests/conftest.py index 65292b9..a0b7840 100644 --- a/libkirk/tests/conftest.py +++ b/libkirk/tests/conftest.py @@ -48,6 +48,9 @@ def config_help(self) -> dict: async def get_suites(self, sut: SUT) -> list: return ["suite01", "suite02", "sleep", "environ", "kernel_panic"] + async def find_command(self, sut: SUT, command: str) -> Test: + return Test(name=command, cmd=command) + async def find_suite(self, sut: SUT, name: str) -> Suite: if name in "suite01": test0 = Test( diff --git a/libkirk/tests/test_kselftests.py b/libkirk/tests/test_kselftests.py index 03040fa..145ed18 100644 --- a/libkirk/tests/test_kselftests.py +++ b/libkirk/tests/test_kselftests.py @@ -89,6 +89,18 @@ async def test_get_suites(self, framework, sut): suites = await framework.get_suites(sut) assert suites == self.GROUPS + async def test_find_command(self, framework, sut, tmpdir): + """ + Test find_command method. + """ + test = await framework.find_command(sut, "test_progs") + assert test.name == "test_progs" + assert test.command == "test_progs" + assert not test.arguments + assert not test.parallelizable + assert test.env == {"PATH": str(tmpdir / "bpf")} + assert test.cwd == str(tmpdir / "bpf") + async def test_find_suite(self, framework, sut, tmpdir): """ Test find_suite method. diff --git a/libkirk/tests/test_liburing.py b/libkirk/tests/test_liburing.py index 8e06bac..d8571aa 100644 --- a/libkirk/tests/test_liburing.py +++ b/libkirk/tests/test_liburing.py @@ -66,6 +66,18 @@ async def test_get_suites(self, framework, sut): suites = await framework.get_suites(sut) assert suites == ["default"] + async def test_find_command(self, framework, sut, tmpdir): + """ + Test find_command method. + """ + test = await framework.find_command(sut, "test0 ciao bepi") + assert test.name == "test0" + assert test.command == "test0" + assert test.arguments == ["ciao", "bepi"] + assert not test.parallelizable + assert test.env == {"PATH": str(tmpdir)} + assert test.cwd == str(tmpdir) + async def test_find_suite(self, framework, sut, tmpdir): """ Test find_suite method. diff --git a/libkirk/tests/test_ltp.py b/libkirk/tests/test_ltp.py index 4181b3e..118cd6e 100644 --- a/libkirk/tests/test_ltp.py +++ b/libkirk/tests/test_ltp.py @@ -50,7 +50,7 @@ def prepare_tmpdir(self, tmpdir): for i in range(self.TESTS_NUM): content += f"test0{i} echo ciao\n" - tmpdir.mkdir("testcases").mkdir("bin") + testcases = tmpdir.mkdir("testcases").mkdir("bin") runtest = tmpdir.mkdir("runtest") for i in range(self.SUITES_NUM): @@ -75,6 +75,10 @@ def prepare_tmpdir(self, tmpdir): metadata = tmpdir.mkdir("metadata") / "ltp.json" metadata.write(json.dumps(metadata_d)) + # create shell test + test_sh = testcases / "test.sh" + test_sh.write("#!/bin/bash\necho $1 $2\n") + def test_name(self, framework): """ Test that name property is not empty. @@ -91,6 +95,18 @@ async def test_get_suites(self, framework, sut, tmpdir): assert "suite2" in suites assert "slow_suite" in suites + async def test_find_command(self, framework, sut, tmpdir): + """ + Test find_command method. + """ + test = await framework.find_command(sut, "test.sh ciao bepi") + assert test.name == "test.sh" + assert test.command == "test.sh" + assert test.arguments == ["ciao", "bepi"] + assert not test.parallelizable + assert test.cwd == tmpdir / "testcases" / "bin" + assert test.env + async def test_find_suite(self, framework, sut, tmpdir): """ Test find_suite method.