Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added timeout detection on commands #2

Merged
merged 1 commit into from
Oct 19, 2023
Merged

Added timeout detection on commands #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 57 additions & 10 deletions GradescopeGrader/Cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import os
import subprocess
import threading
import time

from typing import Union
Expand All @@ -22,32 +23,78 @@ def __init__(
cmd: Union[str, list],
workDir: str = os.getcwd(),
env: dict = os.environ,
timeout: Union[None, float] = None,
) -> None:
super(Cmd, self).__init__()

self.workDir = workDir
self.cmd = cmd
self.env = env
self.timeout = timeout

self.proc = None
self.stdout = None
self.stderr = None
self.returncode = None
self.returncode = None # None value by default
self.runtimeNS = None
self.hasTimedOut = False
self.timer = None

def Kill(self) -> None:
if self.proc:
self.proc.kill()
self.hasTimedOut = True

def StartKillTimer(self) -> None:
if self.timer is not None:
self.CancelKillTimer()

if self.timeout is not None:
self.hasTimedOut = False # reset
self.timer = threading.Timer(
interval=self.timeout,
function=self.Kill
)
self.timer.start()

def CancelKillTimer(self) -> None:
if self.timer:
self.timer.cancel()
self.timer = None

def Run(self) -> None:
startTime = time.time_ns()
with subprocess.Popen(
self.cmd,
cwd=self.workDir,
env=self.env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as proc:
self.stdout, self.stderr = proc.communicate()
self.returncode = proc.returncode

try:
with subprocess.Popen(
self.cmd,
cwd=self.workDir,
env=self.env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as proc:
self.proc = proc
self.StartKillTimer()
self.stdout, self.stderr = proc.communicate()
# update return code based on the process return code
# or -9 if the process is killed by timeout
self.returncode = proc.returncode if not self.hasTimedOut else -9
finally:
self.CancelKillTimer()
self.proc = None

endTime = time.time_ns()
self.runtimeNS = endTime - startTime

def GetRunTimeNS(self) -> int:
return self.runtimeNS

def GetRunTimeMS(self) -> float:
return self.GetRunTimeNS() / 1000 / 1000

def GetRunTime(self) -> float:
return self.GetRunTimeMS() / 1000

def __str__(self) -> str:
if isinstance(self.cmd, str):
return self.cmd
Expand Down
9 changes: 7 additions & 2 deletions GradescopeGrader/CmdAllOrNothingTest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def __init__(
self.preCmd = preCmd
self.postCmd = postCmd

# failed score and status by default
self.score = 0
self.status = 'failed'

Expand Down Expand Up @@ -66,6 +67,7 @@ def Run(self) -> None:
self.runtimeNS = self.cmd.runtimeNS

if self.cmd.returncode == 0:
# only update score and status when th return code is 0
self.score = self.maxScore
self.status = 'passed'

Expand All @@ -75,7 +77,7 @@ def Run(self) -> None:

def _GenOutput(self) -> Tuple[str, str]:
output = ''
output += 'Execution Time: {:.2f} ms\n'.format(self.runtimeNS / 1000000)
output += 'Execution Time: {:.2f} ms\n'.format(self.GetRunTimeMS())
output += 'Return Code: {}\n'.format(self.returncode)
output += '\n'

Expand All @@ -97,8 +99,11 @@ def _GenOutput(self) -> Tuple[str, str]:
def GetRunTimeNS(self) -> int:
return self.runtimeNS

def GetRunTimeMS(self) -> float:
return self.GetRunTimeNS() / 1000 / 1000

def GetRunTime(self) -> float:
return self.GetRunTimeNS() / 1000000
return self.GetRunTimeMS() / 1000

def GetResult(self) -> dict:
'''
Expand Down
62 changes: 62 additions & 0 deletions tests/unittest/Cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,65 @@ def test_fail_run(self):
self.assertEqual(inst.stderr, b'')
self.assertEqual(inst.returncode, 12)
self.assertGreater(inst.runtimeNS, 0)

def test_timeout_run_sleep(self):
cmd = [ sys.executable, '-c', 'import time; print("before sleep"); time.sleep(10); print("after sleep")' ]
inst = Cmd.Cmd(
workDir='/tmp',
cmd=cmd,
env={},
timeout=1.0,
)
inst.Run()
#self.assertEqual(inst.stdout, b'')
self.assertEqual(inst.stderr, b'')
self.assertEqual(inst.returncode, -9)
self.assertGreater(inst.GetRunTime(), 0.500) # 0.5s
self.assertLess(inst.GetRunTime(), 2.0) # 2s
self.assertGreater(inst.GetRunTimeMS(), 500.0) # 500ms
self.assertLess(inst.GetRunTimeMS(), 2000.0) # 2000ms
self.assertEqual(inst.hasTimedOut, True)

def test_timeout_run_loop(self):
cmd = [ sys.executable, '-c', 'i = 1\nwhile True:\n\ti += 1' ]
inst = Cmd.Cmd(
workDir='/tmp',
cmd=cmd,
env={},
timeout=1.0,
)
inst.Run()
self.assertEqual(inst.stdout, b'')
self.assertEqual(inst.stderr, b'')
self.assertEqual(inst.returncode, -9)
self.assertGreater(inst.GetRunTime(), 0.500) # 0.5s
self.assertLess(inst.GetRunTime(), 2.0) # 2s
self.assertGreater(inst.GetRunTimeMS(), 500.0) # 500ms
self.assertLess(inst.GetRunTimeMS(), 2000.0) # 2000ms
self.assertEqual(inst.hasTimedOut, True)

def test_runtime_500ms(self):
inst = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(0.5); exit(123)']
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 123)
self.assertGreater(inst.GetRunTimeNS(), 400000000)
self.assertLess( inst.GetRunTimeNS(), 600000000)
self.assertGreater(inst.GetRunTimeMS(), 400.0)
self.assertLess( inst.GetRunTimeMS(), 600.0)
self.assertGreater(inst.GetRunTime(), 0.4)
self.assertLess( inst.GetRunTime(), 0.6)

def test_runtime_2s(self):
inst = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(2.0); exit(132)']
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 132)
self.assertGreater(inst.GetRunTimeNS(), 1500000000)
self.assertLess( inst.GetRunTimeNS(), 2500000000)
self.assertGreater(inst.GetRunTimeMS(), 1500.0)
self.assertLess( inst.GetRunTimeMS(), 2500.0)
self.assertGreater(inst.GetRunTime(), 1.5)
self.assertLess( inst.GetRunTime(), 2.5)
36 changes: 36 additions & 0 deletions tests/unittest/CmdAllOrNothingTest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,42 @@ def test_serializable(self):
self.assertEqual(resJsonDict['output_format'], 'md')
self.assertEqual(res, resJsonDict)

def test_runtime_500ms(self):
okCmd = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(0.5); exit(0)']
)
inst = CmdAllOrNothingTest(
testId='test_inst',
cmd=okCmd,
maxScore=123,
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 0)
self.assertGreater(inst.GetRunTimeNS(), 400000000)
self.assertLess( inst.GetRunTimeNS(), 600000000)
self.assertGreater(inst.GetRunTimeMS(), 400.0)
self.assertLess( inst.GetRunTimeMS(), 600.0)
self.assertGreater(inst.GetRunTime(), 0.4)
self.assertLess( inst.GetRunTime(), 0.6)

def test_runtime_2s(self):
okCmd = Cmd.Cmd(
cmd=[sys.executable, '-c', 'import time; time.sleep(2.0); exit(0)']
)
inst = CmdAllOrNothingTest(
testId='test_inst',
cmd=okCmd,
maxScore=123,
)
inst.Run()
self.assertGreaterEqual(inst.returncode, 0)
self.assertGreater(inst.GetRunTimeNS(), 1500000000)
self.assertLess( inst.GetRunTimeNS(), 2500000000)
self.assertGreater(inst.GetRunTimeMS(), 1500.0)
self.assertLess( inst.GetRunTimeMS(), 2500.0)
self.assertGreater(inst.GetRunTime(), 1.5)
self.assertLess( inst.GetRunTime(), 2.5)

class TestGrader(unittest.TestCase):

def test_constructor(self):
Expand Down