Feature/execute failing command

gator/checks/check_ExecuteFailingCommand.py

@@ -0,0 +1,56 @@
+"""Check that a command executes with an expected error."""
+
+import argparse
+
+from gator import checkers
+from gator import invoke
+
+
+def get_parser():
+    """Get a parser for the arguments provided on the command-line."""
+    # create the parser with the default help formatter
+    # use a new description since this is a stand-alone check
+    parser = argparse.ArgumentParser(
+        prog="ExecuteFailingCommand",
+        description="Check Provided by GatorGrader: ExecuteFailingCommand",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    # Required Named Checker Arguments {{{
+
+    required_group = parser.add_argument_group("required checker arguments")
+
+    # COMMAND: the command to execute
+    # REQUIRED? Yes
+    required_group.add_argument(
+        "--command", type=str, help="command to execute", required=True
+    )
+
+    # }}}
+
+    # Optional Named Checker Arguments {{{
+
+    # None required for this checker
+
+    # }}}
+    return parser
+
+
+def parse(args):
+    """Use the parser on the provided arguments."""
+    return checkers.parse(get_parser, args)
+
+
+# pylint: disable=unused-argument
+def act(main_parsed_arguments, check_remaining_arguments):
+    """Perform the action for this check."""
+    # extract the two arguments for this check:
+    # --> command is required to specify the commit count threshold
+    check_parsed_arguments = parse(check_remaining_arguments)
+    # Directly run the check since at least one of the argument's for it is mandatory.
+    # This means that the use of check_ExecuteCommand would have already failed by this
+    # point since argparse will exit the program if a command-line argument is not provided
+    command = check_parsed_arguments.command
+    # Invoke the command with the inverse check so that the command with return code 1 should pass the check
+    invocations = invoke.invoke_all_command_executes_checks(command, inverse_check=True)
+    return invocations

gator/invoke.py

@@ -696,7 +696,7 @@ def invoke_all_command_regex_checks(
     )
 
 
-def invoke_all_command_executes_checks(command):
+def invoke_all_command_executes_checks(command, inverse_check=False):
     """Perform the check for whether or not a command runs without error."""
     # pylint: disable=unused-variable
     # note that the program does not use all of these
@@ -706,11 +706,19 @@ def invoke_all_command_executes_checks(command):
     # this is the opposite of what is used for processes
     # but, all other GatorGrader checks return 0 on failure and 1 on success
     command_passed = False
-    if (
-        command_error == constants.markers.Empty
-        and command_returncode == constants.codes.Success
-    ):
-        command_passed = True
+    # Assume that command passes when it makes error and return code 1 in inverse check mode
+    if inverse_check is False:
+        if (
+            command_error == constants.markers.Empty
+            and command_returncode == constants.codes.Success
+        ):
+            command_passed = True
+    else:
+        if (
+            command_error != constants.markers.Empty
+            or command_returncode != constants.codes.Success
+        ):
+            command_passed = True
     # create the message and diagnostic and report the result
     message = "The command '" + str(command) + "'" + " executes correctly"
     diagnostic = "The command returned the error code " + str(command_returncode)

tests/checks/test_check_ExecuteFailingCommand.py

@@ -0,0 +1,73 @@
+"""Tests for ExecuteFailingCommand's input and verification of command-line arguments."""
+
+import pytest
+import os
+import sys
+
+from unittest.mock import patch
+
+
+from gator import arguments
+from gator import report
+from gator.exceptions import InvalidCheckArgumentsError
+from gator.checks import check_ExecuteFailingCommand
+
+
+@pytest.mark.parametrize(
+    "commandline_arguments",
+    [
+        ([]),
+        (["--commandWRONG", "echo"]),
+        (["--command", "run", "--WRONG"]),
+        (["--command"]),
+    ],
+)
+def test_required_commandline_arguments_cannot_parse(commandline_arguments, capsys):
+    """Check that incorrect optional command-line arguments check correctly."""
+    with pytest.raises(InvalidCheckArgumentsError) as excinfo:
+        _ = check_ExecuteFailingCommand.parse(commandline_arguments)
+    captured = capsys.readouterr()
+    # there is no standard output or error
+    assert captured.err == ""
+    assert captured.out == ""
+    assert excinfo.value.check_name == "ExecuteFailingCommand"
+    assert excinfo.value.usage
+    assert excinfo.value.error
+
+
+@pytest.mark.parametrize(
+    "commandline_arguments",
+    [(["--command", "run_command_first"]), (["--command", "run_command_second"])],
+)
+def test_required_commandline_arguments_can_parse(commandline_arguments, not_raises):
+    """Check that correct optional command-line arguments check correctly."""
+    with not_raises(InvalidCheckArgumentsError):
+        _ = check_ExecuteFailingCommand.parse(commandline_arguments)
+
+
+@pytest.mark.parametrize(
+    "commandline_arguments, expected_result",
+    [
+        (["ExecuteFailingCommand", "--command", "WrongCommand"], True),
+        (["ExecuteFailingCommand", "--command", 'echo "CorrectCommand"'], False),
+    ],
+)
+def test_act_produces_output(commandline_arguments, expected_result, load_check):
+    """Check that using the check produces output."""
+    testargs = [os.getcwd()]
+    with patch.object(sys, "argv", testargs):
+        parsed_arguments, remaining_arguments = arguments.parse(commandline_arguments)
+        args_verified = arguments.verify(parsed_arguments)
+        assert args_verified is True
+        check = load_check(parsed_arguments)
+        check_result = check.act(parsed_arguments, remaining_arguments)
+        # check the result
+        assert check_result is expected_result
+        # check the contents of the report
+        assert report.get_result() is not None
+        assert len(report.get_result()["check"]) > 1
+        assert report.get_result()["outcome"] is expected_result
+        if expected_result:
+            assert report.get_result()["diagnostic"] == ""
+        else:
+            assert report.get_result()["diagnostic"] != ""

tests/test_invoke.py

@@ -750,7 +750,7 @@ def test_run_command_grab_output_as_string_count_lines_exact(
 
 
 def test_command_executes_checks_does_not_execute_correctly():
-    """Check to see if a command does not run correctly and gets a zero return value."""
+    """Check to see if a command does not run correctly and gets a zero return value and fails to pass."""
     # note that a zero-code means that the command did not work
     # this is the opposite of what is used for processes
     # but, all other GatorGrader checks return 0 on failure and 1 on success
@@ -759,14 +759,34 @@ def test_command_executes_checks_does_not_execute_correctly():
 
 
 def test_command_executes_checks_does_execute_correctly():
-    """Check to see if a command does run correctly and gets a non-zero return value."""
+    """Check to see if a command does run correctly and gets a non-zero return value and succeeds to pass."""
     # note that a zero-code means that the command did not work
     # this is the opposite of what is used for processes
     # but, all other GatorGrader checks return 0 on failure and 1 on success
     status_code = invoke.invoke_all_command_executes_checks("true")
     assert status_code is True
 
 
+def test_command_executes_checks_not_pass_with_correct_command_in_inverse_mode():
+    """Check to see if a command does run correctly and gets a non-zero return value but not pass."""
+    # note that a zero-code means that the command did not work
+    # this is the opposite of what is used for processes
+    # but, all other GatorGrader checks return 0 on failure and 1 on success
+    status_code = invoke.invoke_all_command_executes_checks("true", inverse_check=True)
+    assert status_code is False
+
+
+def test_command_executes_checks_pass_with_wrong_command_in_inverse_mode():
+    """Check to see if a command does not run correctly and gets a zero return value but pass."""
+    # note that a zero-code means that the command did not work
+    # this is the opposite of what is used for processes
+    # but, all other GatorGrader checks return 0 on failure and 1 on success
+    status_code = invoke.invoke_all_command_executes_checks(
+        "willnotwork", inverse_check=True
+    )
+    assert status_code is True
+
+
 # pylint: disable=unused-argument
 def test_run_command_does_execute_correctly_would_make_output(
     reset_results_dictionary, tmpdir