Feature/fail test if any exceptions thrown (#28)

* Refactoring default error behaviour

build.gradle

@@ -7,7 +7,7 @@ apply plugin: 'jacoco'
 group = 'com.github.noconnor'
 sourceCompatibility = 1.8
 // http://semver.org/
-version = '1.7.0'
+version = '1.8.0'
 
 repositories {
     mavenCentral()

src/main/java/com/github/noconnor/junitperf/JUnitPerfTestRequirement.java

@@ -13,9 +13,9 @@
   String percentiles() default "";
 
   // Expected test throughput (executions per second)
-  int executionsPerSec() default 1;
+  int executionsPerSec() default 0;
 
-  // Expected % of test failures. Failures are measured as test case exceptions
-  float allowedErrorPercentage() default 1;
+  // Expected % of test failures. Failures are measured as test case exceptions, default 0% errors allowed
+  float allowedErrorPercentage() default 0;
 
 }

src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java

@@ -35,13 +35,11 @@ public class EvaluationContext {
   private int configuredRateLimit;
 
   @Getter
-  private Map<Integer, Float> requiredPercentiles;
+  private Map<Integer, Float> requiredPercentiles = emptyMap();
   @Getter
-  private int requiredThroughput;
+  private int requiredThroughput = 0;
   @Getter
-  private float requiredAllowedErrorsRate;
-  @Getter
-  private boolean validationRequired;
+  private float requiredAllowedErrorsRate = 0;
 
   @Getter
   @Setter
@@ -75,10 +73,8 @@ public void loadConfiguration(JUnitPerfTest testSettings) {
   }
 
   public void loadRequirements(JUnitPerfTestRequirement requirements) {
-    validationRequired = nonNull(requirements);
-    if (validationRequired) {
+    if (nonNull(requirements)) {
       validateRequirements(requirements);
-      validationRequired = true;
       requiredThroughput = requirements.executionsPerSec();
       requiredAllowedErrorsRate = requirements.allowedErrorPercentage();
       requiredPercentiles = parsePercentileLimits(requirements.percentiles());
@@ -87,17 +83,10 @@ public void loadRequirements(JUnitPerfTestRequirement requirements) {
 
   public void runValidation() {
     checkState(nonNull(statistics), "Statistics must be calculated before running validation");
-    if (validationRequired) {
-      isThroughputAchieved = getThroughputQps() >= requiredThroughput;
-      isErrorThresholdAchieved = statistics.getErrorPercentage() <= (requiredAllowedErrorsRate * 100);
-      percentileResults = evaluateLatencyPercentiles();
-      isSuccessful = isThroughputAchieved && isErrorThresholdAchieved && noLatencyPercentileFailures();
-    } else {
-      isSuccessful = true;
-      isThroughputAchieved = true;
-      isErrorThresholdAchieved = true;
-      percentileResults = emptyMap();
-    }
+    isThroughputAchieved = getThroughputQps() >= requiredThroughput;
+    isErrorThresholdAchieved = statistics.getErrorPercentage() <= (requiredAllowedErrorsRate * 100);
+    percentileResults = evaluateLatencyPercentiles();
+    isSuccessful = isThroughputAchieved && isErrorThresholdAchieved && noLatencyPercentileFailures();
   }
 
   private boolean noLatencyPercentileFailures() {
@@ -140,7 +129,7 @@ private void validateTestSettings(JUnitPerfTest testSettings) {
 
   private void validateRequirements(JUnitPerfTestRequirement requirements) {
     checkState(requirements.allowedErrorPercentage() >= 0, "AllowedErrorPercentage must be >= 0");
-    checkState(requirements.executionsPerSec() > 0, "ExecutionsPerSec must be > 0");
+    checkState(requirements.executionsPerSec() >= 0, "ExecutionsPerSec must be >= 0");
   }
 
 }

src/test/examples/com/github/noconnor/junitperf/examples/ExampleFailureTests.java

@@ -11,7 +11,7 @@ public class ExampleFailureTests {
   @Rule
   public JUnitPerfRule perfRule = new JUnitPerfRule();
 
-  @Test
+  @Test(expected = AssertionError.class)
   @JUnitPerfTest(threads = 1, durationMs = 1_000, maxExecutionsPerSecond = 1_000)
   @JUnitPerfTestRequirement(executionsPerSec = 10_000)
   public void whenThroughputRequirementIsNotMet_thenTestShouldFail() throws InterruptedException {
@@ -21,10 +21,18 @@ public void whenThroughputRequirementIsNotMet_thenTestShouldFail() throws Interr
 
   @Test
   @JUnitPerfTest(threads = 1, durationMs = 1_000, maxExecutionsPerSecond = 1_000)
-  @JUnitPerfTestRequirement(executionsPerSec = 10_000, percentiles = "99:1")
+  @JUnitPerfTestRequirement(executionsPerSec = 10, percentiles = "99:1")
   public void whenLatencyRequirementIsNotMet_thenTestShouldFail() throws InterruptedException {
     // Mock some processing logic
     Thread.sleep(2);
   }
 
+  @Test
+  @JUnitPerfTest(threads = 1, durationMs = 1_000, maxExecutionsPerSecond = 1_000)
+  public void whenNoRequirementsAreSpecified_andExceptionIsThrown_thenTestShouldFail() throws InterruptedException {
+    // Mock some processing logic
+    Thread.sleep(2);
+    throw new IllegalStateException("testing failure");
+  }
+
 }

src/test/java/com/github/noconnor/junitperf/data/EvaluationContextTest.java

@@ -95,18 +95,11 @@ public void whenLoadingJUnitPerfTestSettings_andSettingsAreNull_thenExceptionSho
   @Test
   public void whenLoadingJUnitPerfTestRequirements_thenAppropriateContextSettingsShouldBeUpdated() {
     context.loadRequirements(perfTestRequirement);
-    assertThat(context.isValidationRequired(), is(true));
     assertThat(context.getRequiredAllowedErrorsRate(), is(perfTestRequirement.allowedErrorPercentage()));
     assertThat(context.getRequiredThroughput(), is(perfTestRequirement.executionsPerSec()));
     assertThat(context.getRequiredPercentiles(), is(ImmutableMap.of(90, 0.5F, 95, 9F)));
   }
 
-  @Test
-  public void whenLoadingJUnitPerfTestRequirements_andRequirementsAreNull_thenValidationShouldNotBeRequired() {
-    context.loadRequirements(null);
-    assertThat(context.isValidationRequired(), is(false));
-  }
-
   @Test
   public void whenLoadingJUnitPerfTestRequirements_thenAllowedErrorPercentageShouldBeSensible() {
     when(perfTestRequirement.allowedErrorPercentage()).thenReturn(-1.6F);
@@ -116,9 +109,7 @@ public void whenLoadingJUnitPerfTestRequirements_thenAllowedErrorPercentageShoul
   @Test
   public void whenLoadingJUnitPerfTestRequirements_thenAExecutionsPerSecShouldBeSensible() {
     when(perfTestRequirement.executionsPerSec()).thenReturn(-1);
-    expectRequirementsValidationError("ExecutionsPerSec must be > 0");
-    when(perfTestRequirement.executionsPerSec()).thenReturn(0);
-    expectRequirementsValidationError("ExecutionsPerSec must be > 0");
+    expectRequirementsValidationError("ExecutionsPerSec must be >= 0");
   }
 
   @Test(expected = IllegalStateException.class)
@@ -262,8 +253,8 @@ private void initialisePerfTestRequirementAnnotation() {
 
   private void initialiseStatisticsMockToPassValidation() {
     when(statisticsMock.getEvaluationCount()).thenReturn(15_000L);
-    when(statisticsMock.getErrorCount()).thenReturn(1L);
-    when(statisticsMock.getErrorPercentage()).thenReturn(1.0F);
+    when(statisticsMock.getErrorCount()).thenReturn(0L);
+    when(statisticsMock.getErrorPercentage()).thenReturn(0.0F);
     when(statisticsMock.getLatencyPercentile(90, NANOSECONDS)).thenReturn(2000F);
     when(statisticsMock.getLatencyPercentile(95, NANOSECONDS)).thenReturn(4000F);
   }