Rerun tests when timeout or crash in matrix mode

pitest-entry/src/main/java/org/pitest/mutationtest/MutationResult.java

@@ -45,6 +45,18 @@ public List<String> getSucceedingTests() {
     return this.status.getSucceedingTests();
   }
 
+  public List<String> getTimeoutTests() {
+    return this.status.getTimeOutTests();
+  }
+
+  public List<String> getMemoryErrorTests() {
+    return this.status.getMemoryErrorTests();
+  }
+
+  public List<String> getRunErrorTests() {
+    return this.status.getRunErrorTests();
+  }
+
   public DetectionStatus getStatus() {
     return this.status.getStatus();
   }

pitest-entry/src/main/java/org/pitest/mutationtest/MutationStatusMap.java

@@ -14,6 +14,7 @@
  */
 package org.pitest.mutationtest;
 
+import static org.pitest.functional.prelude.Prelude.or;
 import static org.pitest.functional.prelude.Prelude.putToMap;
 
 import java.util.Collection;
@@ -74,6 +75,15 @@ public Collection<MutationDetails> getUnfinishedRuns() {
         .collect(Collectors.toList());
   }
 
+  public Collection<MutationDetails> getCrashed() {
+    return this.mutationMap.entrySet().stream()
+            .filter(or(hasStatus(DetectionStatus.RUN_ERROR),
+                    or(hasStatus(DetectionStatus.MEMORY_ERROR),
+                            hasStatus(DetectionStatus.TIMED_OUT))))
+            .map(toMutationDetails())
+            .collect(Collectors.toList());
+  }
+
   public Set<MutationDetails> allMutations() {
     return this.mutationMap.keySet();
   }

pitest-entry/src/main/java/org/pitest/mutationtest/build/MutationTestUnit.java

@@ -16,13 +16,17 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.logging.Logger;
 
 import org.pitest.classinfo.ClassName;
+import org.pitest.coverage.TestInfo;
 import org.pitest.mutationtest.DetectionStatus;
 import org.pitest.mutationtest.MutationMetaData;
 import org.pitest.mutationtest.MutationStatusMap;
+import org.pitest.mutationtest.MutationStatusTestPair;
 import org.pitest.mutationtest.engine.MutationDetails;
+import org.pitest.mutationtest.engine.MutationIdentifier;
 import org.pitest.mutationtest.execute.MutationTestProcess;
 import org.pitest.util.ExitCode;
 import org.pitest.util.Log;
@@ -75,17 +79,55 @@ private void runTestInSeperateProcessForMutationRange(
 
     final Collection<MutationDetails> remainingMutations = mutations
         .getUnrunMutations();
-    final MutationTestProcess worker = this.workerFactory.createWorker(
+
+    //First run mutants normally
+    MutationTestProcess worker = this.workerFactory.createWorker(
         remainingMutations, this.testClasses);
     worker.start();
 
     setFirstMutationToStatusOfStartedInCaseMinionFailsAtBoot(mutations,
         remainingMutations);
 
-    final ExitCode exitCode = waitForMinionToDie(worker);
+    ExitCode exitCode = waitForMinionToDie(worker);
     worker.results(mutations);
 
     correctResultForProcessExitCode(mutations, exitCode);
+
+    //rerun crashing mutants with isolation
+    if (this.workerFactory.isFullMutationMatrix() && !exitCode.isOk()) {
+        Collection<MutationDetails> crashedRuns = mutations.getCrashed();
+        //not rerun crashed from previous range
+        crashedRuns.retainAll(remainingMutations);
+
+        LOG.info("Rerunning " + crashedRuns.size() + " mutant(s) because of minion crash");
+        for (MutationDetails d : crashedRuns) {
+            MutationStatusTestPair result = null;
+            for (TestInfo t : d.getTestsInOrder()) {
+                MutationDetails singleTest = new MutationDetails(new MutationIdentifier(d.getId().getLocation(),
+                        d.getId().getIndexes(),d.getMutator()), d.getFilename(), d.getDescription(),
+                        d.getLineNumber(), d.getBlock());
+                singleTest.addTestsInOrder(Collections.singleton(t));
+                worker = this.workerFactory.createWorker(Collections.singleton(singleTest), this.testClasses);
+                worker.start();
+                exitCode = waitForMinionToDie(worker);
+                MutationStatusTestPair r = worker.results(singleTest);
+
+                if (exitCode != ExitCode.OK) {
+                    r.setErrorStatusAndName(DetectionStatus.getForErrorExitCode(exitCode), t.getName());
+                }
+
+                if (result == null) {
+                    result = r;
+                } else {
+                    result.accumulate(r, t.getName());
+                }
+            }
+
+            if (result != null) {
+                mutations.setStatusForMutation(d, result);
+            }
+        }
+    }
   }
 
   private static ExitCode waitForMinionToDie(final MutationTestProcess worker) {
@@ -123,7 +165,4 @@ private static void correctResultForProcessExitCode(
   private static MutationMetaData reportResults(final MutationStatusMap mutationsMap) {
     return new MutationMetaData(mutationsMap.createMutationResults());
   }
-
-
-
-}
+}

pitest-entry/src/main/java/org/pitest/mutationtest/build/WorkerFactory.java

@@ -75,4 +75,8 @@ private SideEffect1<String> captureStdOutIfVerbose() {
 
   }
 
+  public boolean isFullMutationMatrix() {
+    return this.fullMutationMatrix;
+  }
+
 }

pitest-entry/src/main/java/org/pitest/mutationtest/execute/MutationTestProcess.java

@@ -31,6 +31,10 @@ public void start() throws IOException, InterruptedException {
     this.process.start();
   }
 
+  public MutationStatusTestPair results(MutationDetails mutant) {
+    return this.thread.getStatus(mutant.getId());
+  }
+
   public void results(final MutationStatusMap allmutations) throws IOException {
 
     for (final MutationDetails each : allmutations.allMutations()) {

pitest-entry/src/main/java/org/pitest/mutationtest/report/xml/XMLReportListener.java

@@ -20,13 +20,16 @@
 import static org.pitest.mutationtest.report.xml.Tag.killingTest;
 import static org.pitest.mutationtest.report.xml.Tag.killingTests;
 import static org.pitest.mutationtest.report.xml.Tag.lineNumber;
+import static org.pitest.mutationtest.report.xml.Tag.memoryErrorTests;
 import static org.pitest.mutationtest.report.xml.Tag.methodDescription;
 import static org.pitest.mutationtest.report.xml.Tag.mutatedClass;
 import static org.pitest.mutationtest.report.xml.Tag.mutatedMethod;
 import static org.pitest.mutationtest.report.xml.Tag.mutation;
 import static org.pitest.mutationtest.report.xml.Tag.mutator;
+import static org.pitest.mutationtest.report.xml.Tag.runErrorTests;
 import static org.pitest.mutationtest.report.xml.Tag.sourceFile;
 import static org.pitest.mutationtest.report.xml.Tag.succeedingTests;
+import static org.pitest.mutationtest.report.xml.Tag.timeoutTests;
 
 import java.io.IOException;
 import java.io.Writer;
@@ -42,7 +45,8 @@
 import org.pitest.util.Unchecked;
 
 enum Tag {
-  mutation, sourceFile, mutatedClass, mutatedMethod, methodDescription, lineNumber, mutator, index, killingTest, killingTests, succeedingTests, description, block;
+  mutation, sourceFile, mutatedClass, mutatedMethod, methodDescription, lineNumber, mutator, index, killingTest,
+  killingTests, succeedingTests, description, block, timeoutTests, runErrorTests, memoryErrorTests;
 }
 
 public class XMLReportListener implements MutationResultListener {
@@ -95,6 +99,12 @@ private String makeMutationNode(final MutationResult mutation) {
             createTestDesc(mutation.getKillingTests()), killingTests)
         + makeNodeWhenConditionSatisfied(fullMutationMatrix,
             createTestDesc(mutation.getSucceedingTests()), succeedingTests)
+        + makeNodeWhenConditionSatisfied(fullMutationMatrix,
+            createTestDesc(mutation.getTimeoutTests()), timeoutTests)
+        + makeNodeWhenConditionSatisfied(fullMutationMatrix,
+            createTestDesc(mutation.getRunErrorTests()), runErrorTests)
+        + makeNodeWhenConditionSatisfied(fullMutationMatrix,
+            createTestDesc(mutation.getMemoryErrorTests()), memoryErrorTests)
         + makeNode(clean(details.getDescription()), description);
   }
 

pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/SimpleCalculator.java

@@ -1,5 +1,6 @@
 package com.example.coverage.execute.samples.mutationMatrix;
 
+
 public class SimpleCalculator {
 
   public static int sum(int x, int y) {

pitest-entry/src/test/java/com/example/coverage/execute/samples/mutationMatrix/TestsForSimpleCalculator.java

@@ -4,6 +4,9 @@
 
 import org.junit.Test;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class TestsForSimpleCalculator {
 
   @Test
@@ -20,4 +23,24 @@ public void testSumWithNegativeNumber() {
   public void pseudoTestSum() {
     SimpleCalculator.sum(2, 1);
   }
+
+  @Test
+  public void unknownErrorOnMutant() {
+    if (SimpleCalculator.sum(2,1) != 3) {
+      System.exit(13);
+    }
+  }
+
+  @Test
+  public void timeoutOnMutant() {
+    if (SimpleCalculator.sum(2,1) != 3) {
+//      System.exit(14);
+      try {
+        Thread.sleep(5000);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+  }
+
 }

pitest-entry/src/test/java/org/pitest/mutationtest/MutationCoverageReportSystemTest.java

@@ -39,6 +39,7 @@
 import org.pitest.SystemTest;
 import org.pitest.classpath.ClassPath;
 import org.pitest.help.PitHelpError;
+import org.pitest.mutationtest.build.PercentAndConstantTimeoutStrategy;
 import org.pitest.mutationtest.engine.gregor.Generated;
 import org.pitest.util.FileUtil;
 import org.pitest.util.IsolationUtils;
@@ -212,18 +213,24 @@ public void computesFullMutationMatrix() {
     .setTargetTests(predicateFor("com.example.coverage.execute.samples.mutationMatrix.*"));
     this.data.setTargetClasses(asList("com.example.coverage.execute.samples.mutationMatrix.*"));
     this.data.setExcludedClasses(asGlobs(TestsForSimpleCalculator.class));
+    this.data.setExcludedMethods(asList("crash"));
     this.data.setFullMutationMatrix(true);
     this.data.addOutputFormats(Arrays.asList("XML"));
     this.data.setMutators(Arrays.asList("MATH"));
+    this.data.setTimeoutConstant(PercentAndConstantTimeoutStrategy.DEFAULT_CONSTANT);
+    this.data.setTimeoutFactor(PercentAndConstantTimeoutStrategy.DEFAULT_FACTOR);
     createAndRun();
     List<MutationResult> resultData = this.metaDataExtractor.getData();
     assertEquals(1, resultData.size());
 
     MutationResult mutation = resultData.get(0);
     assertEquals(KILLED, mutation.getStatus());
-    assertEquals(3, mutation.getNumberOfTestsRun());
+    assertEquals(5, mutation.getNumberOfTestsRun());
     assertEquals(2, mutation.getKillingTests().size());
     assertEquals(1, mutation.getSucceedingTests().size());
+    assertEquals(0, mutation.getMemoryErrorTests().size());
+    assertEquals(1, mutation.getTimeoutTests().size());
+    assertEquals(1, mutation.getRunErrorTests().size());
   }
 
   @Test(expected = PitHelpError.class)

pitest-entry/src/test/java/org/pitest/mutationtest/report/xml/XMLReportListenerTest.java

@@ -66,7 +66,7 @@ public void shouldOutputFullMutationMatrixWhenEnabled() throws IOException {
             new MutationStatusTestPair(3, DetectionStatus.KILLED, Arrays.asList("foo", "foo2"), Arrays.asList("bar")));
     this.testee
         .handleMutationResult(MutationTestResultMother.createClassResults(mr));
-    final String expected = "<mutation detected='true' status='KILLED' numberOfTestsRun='3'><sourceFile>file</sourceFile><mutatedClass>clazz</mutatedClass><mutatedMethod>method</mutatedMethod><methodDescription>()I</methodDescription><lineNumber>42</lineNumber><mutator>mutator</mutator><index>1</index><block>0</block><killingTests>foo|foo2</killingTests><succeedingTests>bar</succeedingTests><description>desc</description></mutation>\n";
+    final String expected = "<mutation detected='true' status='KILLED' numberOfTestsRun='3'><sourceFile>file</sourceFile><mutatedClass>clazz</mutatedClass><mutatedMethod>method</mutatedMethod><methodDescription>()I</methodDescription><lineNumber>42</lineNumber><mutator>mutator</mutator><index>1</index><block>0</block><killingTests>foo|foo2</killingTests><succeedingTests>bar</succeedingTests><timeoutTests></timeoutTests><runErrorTests></runErrorTests><memoryErrorTests></memoryErrorTests><description>desc</description></mutation>\n";
     assertEquals(expected, this.out.toString());
   }