Fix command execution

Makefile

@@ -15,4 +15,6 @@ install:
  install -m 00644 $(top_srcdir)/tools/kirk/libkirk/*.py $(BASE_DIR)/libkirk
  install -m 00775 $(top_srcdir)/tools/kirk/kirk $(BASE_DIR)/kirk
 
+ cd $(BASE_DIR) && ln -sf kirk runltp-ng
+
 include $(top_srcdir)/include/mk/generic_leaf_target.mk

README.md

@@ -93,15 +93,15 @@ Some basic commands are the following:
 
  # run LTP syscalls testing suite via SSH
  ./kirk --framework ltp \
- --sut=ssh:host myhost.com:user=root:key_file=myhost_id_rsa \
+ --sut ssh:host=myhost.com:user=root:key_file=myhost_id_rsa \
  --run-suite syscalls
 
  # run LTP syscalls testing suite in parallel on host using 16 workers
  ./kirk --framework ltp --run-suite syscalls --workers 16
 
  # run LTP syscalls testing suite in parallel via SSH using 16 workers
  ./kirk --framework ltp \
- --sut=ssh:host myhost.com:user=root:key_file=myhost_id_rsa \
+ --sut ssh:host=myhost.com:user=root:key_file=myhost_id_rsa \
  --run-suite syscalls --workers 16
 
 It's possible to run a single command before running testing suites using

libkirk/data.py

@@ -122,3 +122,17 @@ def env(self):
  Environment variables
  """
  return self._env
+
+ @property
+ def full_command(self):
+ """
+ Return the full command, with arguments as well.
+ For example, if `command="ls"` and `arguments="-l -a"`,
+ `full_command="ls -l -a"`.
+ """
+ cmd = self.command
+ if len(self.arguments) > 0:
+ cmd += ' '
+ cmd += ' '.join(self.arguments)
+
+ return cmd

libkirk/framework.py

@@ -34,6 +34,19 @@ async def get_suites(self, sut: SUT) -> list:
  """
  raise NotImplementedError()
 
+ async def find_command(self, sut: SUT, command: str) -> Test:
+ """
+ Search for command inside Framework folder and, if it's not found,
+ search for command in the operating system. Then return a Test object
+ which can be used to execute command.
+ :param sut: SUT object to communicate with
+ :type sut: SUT
+ :param command: command to execute
+ :type command: str
+ :returns: Test
+ """
+ raise NotImplementedError()
+
  async def find_suite(self, sut: SUT, name: str) -> Suite:
  """
  Search for suite with given name inside SUT.

libkirk/host.py

@@ -144,7 +144,7 @@ async def run_command(
  try:
  kwargs = {
  "stdout": asyncio.subprocess.PIPE,
- "stderr": asyncio.subprocess.PIPE,
+ "stderr": asyncio.subprocess.STDOUT,
  "cwd": cwd,
  "preexec_fn": os.setsid
  }

libkirk/kselftests.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Andrea Cervesato <[email protected]>
 """
 import os
+import shlex
 import logging
 from libkirk import KirkException
 from libkirk.sut import SUT
@@ -116,6 +117,43 @@ async def get_suites(self, sut: SUT) -> list:
 
  return ["cgroup", "bpf"]
 
+ async def find_command(self, sut: SUT, command: str) -> Test:
+ if not sut:
+ raise ValueError("SUT is None")
+
+ if not command:
+ raise ValueError("command is empty")
+
+ cmd_args = shlex.split(command)
+ suite_folder = None
+
+ for suite in await self.get_suites(sut):
+ folder = os.path.join(self._root, suite)
+ binary = os.path.join(folder, cmd_args[0])
+
+ ret = await sut.run_command(f"test -f {binary}")
+ if ret["returncode"] == 0:
+ suite_folder = folder
+ break
+
+ cwd = None
+ env = None
+
+ ret = await sut.run_command(f"test -d {suite_folder}")
+ if ret["returncode"] == 0:
+ cwd = suite_folder
+ env={"PATH": suite_folder}
+
+ test = Test(
+ name=cmd_args[0],
+ cmd=cmd_args[0],
+ args=cmd_args[1:] if len(cmd_args) > 0 else None,
+ cwd=cwd,
+ env=env,
+ parallelizable=False)
+
+ return test
+
  async def find_suite(self, sut: SUT, name: str) -> Suite:
  if not sut:
  raise ValueError("SUT is None")

libkirk/liburing.py

@@ -7,6 +7,7 @@
 """
 import os
 import re
+import shlex
 import logging
 from libkirk.sut import SUT
 from libkirk.data import Test
@@ -101,6 +102,32 @@ async def _is_parallelizable(sut: SUT, cmd: str) -> bool:
 
  return parallel
 
+ async def find_command(self, sut: SUT, command: str) -> Test:
+ if not sut:
+ raise ValueError("SUT is None")
+
+ if not command:
+ raise ValueError("command is empty")
+
+ cmd_args = shlex.split(command)
+ cwd = None
+ env = None
+
+ ret = await sut.run_command(f"test -d {self._root}")
+ if ret["returncode"] == 0:
+ cwd = self._root
+ env={"PATH": self._root}
+
+ test = Test(
+ name=cmd_args[0],
+ cmd=cmd_args[0],
+ args=cmd_args[1:] if len(cmd_args) > 0 else None,
+ cwd=cwd,
+ env=env,
+ parallelizable=False)
+
+ return test
+
  async def find_suite(self, sut: SUT, name: str) -> Suite:
  if not sut:
  raise ValueError("SUT is None")

libkirk/ltp.py

@@ -8,6 +8,7 @@
 import os
 import re
 import json
+import shlex
 import logging
 from libkirk.results import TestResults
 from libkirk.results import ResultStatus
@@ -39,6 +40,7 @@ def __init__(self) -> None:
  self._root = None
  self._env = None
  self._max_runtime = None
+ self._tc_folder = None
 
  @ property
  def config_help(self) -> dict:
@@ -68,6 +70,8 @@ def setup(self, **kwargs: dict) -> None:
  self._root = root
  self._env["LTPROOT"] = self._root
 
+ self._tc_folder = os.path.join(self._root, "testcases", "bin")
+
  runtime = kwargs.get("max_runtime", None)
 
  if runtime:
@@ -82,11 +86,9 @@ async def _read_path(self, sut: SUT) -> dict:
  """
  Read PATH and initialize it with testcases folder as well.
  """
- tc_path = os.path.join(self._root, "testcases", "bin")
-
  env = self._env.copy()
  if 'PATH' in env:
- env["PATH"] = env["PATH"] + f":{tc_path}"
+ env["PATH"] = env["PATH"] + f":{self._tc_folder}"
  else:
  ret = await sut.run_command("echo -n $PATH")
  if ret["returncode"] != 0:
@@ -145,7 +147,6 @@ async def _read_runtest(
 
  tests = []
  lines = content.split('\n')
- tc_path = os.path.join(self._root, "testcases", "bin")
 
  for line in lines:
  if not line.strip() or line.strip().startswith("#"):
@@ -199,7 +200,7 @@ async def _read_runtest(
  name=test_name,
  cmd=test_cmd,
  args=test_args,
- cwd=tc_path,
+ cwd=self._tc_folder,
  env=env,
  parallelizable=parallelizable)
 
@@ -241,6 +242,32 @@ async def get_suites(self, sut: SUT) -> list:
  suites = [line for line in stdout.split('\n') if line]
  return suites
 
+ async def find_command(self, sut: SUT, command: str) -> Test:
+ if not sut:
+ raise ValueError("SUT is None")
+
+ if not command:
+ raise ValueError("command is empty")
+
+ cmd_args = shlex.split(command)
+ cwd = None
+ env = None
+
+ ret = await sut.run_command(f"test -d {self._tc_folder}")
+ if ret["returncode"] == 0:
+ cwd = self._tc_folder
+ env = await self._read_path(sut)
+
+ test = Test(
+ name=cmd_args[0],
+ cmd=cmd_args[0],
+ args=cmd_args[1:] if len(cmd_args) > 0 else None,
+ cwd=cwd,
+ env=env,
+ parallelizable=False)
+
+ return test
+
  async def find_suite(self, sut: SUT, name: str) -> Suite:
  if not sut:
  raise ValueError("SUT is None")

libkirk/scheduler.py

@@ -131,18 +131,6 @@ def __init__(self, **kwargs: dict) -> None:
  if not self._framework:
  raise ValueError("Framework object is empty")
 
- @ staticmethod
- def _command_from_test(test: Test) -> str:
- """
- Returns a command from test.
- """
- cmd = test.command
- if len(test.arguments) > 0:
- cmd += ' '
- cmd += ' '.join(test.arguments)
-
- return cmd
-
  async def _get_tainted_status(self) -> tuple:
  """
  Check tainted status of the Kernel.
@@ -220,7 +208,7 @@ async def _run_test(self, test: Test, sem: asyncio.Semaphore) -> None:
  await self._write_kmsg(test)
 
  iobuffer = RedirectTestStdout(test)
- cmd = self._command_from_test(test)
+ cmd = test.full_command
  start_t = time.time()
  exec_time = 0
  test_data = None

libkirk/session.py

@@ -235,9 +235,13 @@ async def _exec_command(self, command: str) -> None:
  try:
  await libkirk.events.fire("run_cmd_start", command)
 
+ test = await self._framework.find_command(self._sut, command)
+
  ret = await asyncio.wait_for(
  self._sut.run_command(
- command,
+ test.full_command,
+ cwd=test.cwd,
+ env=test.env,
  iobuffer=RedirectSUTStdout(self._sut, True)),
  timeout=self._exec_timeout
  )

libkirk/sut.py

@@ -253,7 +253,13 @@ async def get_tainted_info(self) -> tuple:
  stdout = ret["stdout"].rstrip()
 
  tainted_num = len(TAINED_MSG)
- code = int(stdout.rstrip())
+ code = stdout.rstrip()
+
+ # output is likely message in stderr
+ if not code.isdigit():
+ raise SUTError(code)
+
+ code = int(code)
  bits = format(code, f"0{tainted_num}b")[::-1]
 
  messages = []

libkirk/tests/conftest.py

@@ -48,6 +48,9 @@ def config_help(self) -> dict:
  async def get_suites(self, sut: SUT) -> list:
  return ["suite01", "suite02", "sleep", "environ", "kernel_panic"]
 
+ async def find_command(self, sut: SUT, command: str) -> Test:
+ return Test(name=command, cmd=command)
+
  async def find_suite(self, sut: SUT, name: str) -> Suite:
  if name in "suite01":
  test0 = Test(

libkirk/tests/test_kselftests.py

@@ -89,6 +89,18 @@ async def test_get_suites(self, framework, sut):
  suites = await framework.get_suites(sut)
  assert suites == self.GROUPS
 
+ async def test_find_command(self, framework, sut, tmpdir):
+ """
+ Test find_command method.
+ """
+ test = await framework.find_command(sut, "test_progs")
+ assert test.name == "test_progs"
+ assert test.command == "test_progs"
+ assert not test.arguments
+ assert not test.parallelizable
+ assert test.env == {"PATH": str(tmpdir / "bpf")}
+ assert test.cwd == str(tmpdir / "bpf")
+
  async def test_find_suite(self, framework, sut, tmpdir):
  """
  Test find_suite method.

libkirk/tests/test_liburing.py

@@ -66,6 +66,18 @@ async def test_get_suites(self, framework, sut):
  suites = await framework.get_suites(sut)
  assert suites == ["default"]
 
+ async def test_find_command(self, framework, sut, tmpdir):
+ """
+ Test find_command method.
+ """
+ test = await framework.find_command(sut, "test0 ciao bepi")
+ assert test.name == "test0"
+ assert test.command == "test0"
+ assert test.arguments == ["ciao", "bepi"]
+ assert not test.parallelizable
+ assert test.env == {"PATH": str(tmpdir)}
+ assert test.cwd == str(tmpdir)
+
  async def test_find_suite(self, framework, sut, tmpdir):
  """
  Test find_suite method.

libkirk/tests/test_ltp.py

@@ -50,7 +50,7 @@ def prepare_tmpdir(self, tmpdir):
  for i in range(self.TESTS_NUM):
  content += f"test0{i} echo ciao\n"
 
- tmpdir.mkdir("testcases").mkdir("bin")
+ testcases = tmpdir.mkdir("testcases").mkdir("bin")
  runtest = tmpdir.mkdir("runtest")
 
  for i in range(self.SUITES_NUM):
@@ -75,6 +75,10 @@ def prepare_tmpdir(self, tmpdir):
  metadata = tmpdir.mkdir("metadata") / "ltp.json"
  metadata.write(json.dumps(metadata_d))
 
+ # create shell test
+ test_sh = testcases / "test.sh"
+ test_sh.write("#!/bin/bash\necho $1 $2\n")
+
  def test_name(self, framework):
  """
  Test that name property is not empty.
@@ -91,6 +95,18 @@ async def test_get_suites(self, framework, sut, tmpdir):
  assert "suite2" in suites
  assert "slow_suite" in suites
 
+ async def test_find_command(self, framework, sut, tmpdir):
+ """
+ Test find_command method.
+ """
+ test = await framework.find_command(sut, "test.sh ciao bepi")
+ assert test.name == "test.sh"
+ assert test.command == "test.sh"
+ assert test.arguments == ["ciao", "bepi"]
+ assert not test.parallelizable
+ assert test.cwd == tmpdir / "testcases" / "bin"
+ assert test.env
+
  async def test_find_suite(self, framework, sut, tmpdir):
  """
  Test find_suite method.