Noconnor/update j unit perf test requirement to support additional va…

…lidations (#29)

* Adding new requirement fields

* Adding min latency validation

* Adding max latency validation

* Adding mean latency validation

* Fixing unittests

* Updating documentation

* Updating consile reporter

* Fixing typo

* Updating HTML report

* Updating CSV reporter test files

README.md

@@ -112,6 +112,9 @@ More information on statistic calculations can be found [here](#statistics)
 | percentiles            | Comma separated list of percentile targets, format: percentile1:limit,percentile2:limit (ie. 90:3.3,99:6.8)                 |        ""      |
 | executionsPerSec       | Target executions per second                                                                                                |        1       |
 | allowedErrorPercentage | Allowed % of errors (uncaught exceptions) during test execution (value between 0 and 1, where 1 = 100% errors allowed)      |        1       |
+| minLatency             | Expected minimum latency in ms, if minimum latency is above this value, test will fail                                      |   disabled     |
+| maxLatency             | Expected maximum latency in ms, if maximum latency is above this value, test will fail                                      |   disabled     |
+| meanLatency            | Expected mean latency in ms, if mean latency is above this value, test will fail                                            |   disabled     |
 
 
 ## Reports

src/main/java/com/github/noconnor/junitperf/JUnitPerfTestRequirement.java

@@ -18,4 +18,13 @@
   // Expected % of test failures. Failures are measured as test case exceptions, default 0% errors allowed
   float allowedErrorPercentage() default 0;
 
+  // Expected minimum latency in ms, if minimum latency is above this value, test will fail
+  float minLatency() default -1;
+
+  // Expected maximum latency in ms, if maximum latency is above this value, test will fail
+  float maxLatency() default -1;
+
+  // Expected mean latency in ms, if mean latency is above this value, test will fail
+  float meanLatency() default -1;
+
 }

src/main/java/com/github/noconnor/junitperf/data/EvaluationContext.java

@@ -40,6 +40,12 @@ public class EvaluationContext {
   private int requiredThroughput = 0;
   @Getter
   private float requiredAllowedErrorsRate = 0;
+  @Getter
+  private float requiredMinLatency = -1;
+  @Getter
+  private float requiredMaxLatency = -1;
+  @Getter
+  private float requiredMeanLatency = -1;
 
   @Getter
   @Setter
@@ -48,6 +54,12 @@ public class EvaluationContext {
   @Getter
   private boolean isThroughputAchieved;
   @Getter
+  private boolean isMinLatencyAchieved;
+  @Getter
+  private boolean isMaxLatencyAchieved;
+  @Getter
+  private boolean isMeanLatencyAchieved;
+  @Getter
   private boolean isErrorThresholdAchieved;
   @Getter
   private Map<Integer, Boolean> percentileResults;
@@ -78,15 +90,32 @@ public void loadRequirements(JUnitPerfTestRequirement requirements) {
       requiredThroughput = requirements.executionsPerSec();
       requiredAllowedErrorsRate = requirements.allowedErrorPercentage();
       requiredPercentiles = parsePercentileLimits(requirements.percentiles());
+      requiredMinLatency = requirements.minLatency();
+      requiredMaxLatency = requirements.maxLatency();
+      requiredMeanLatency = requirements.meanLatency();
     }
   }
 
   public void runValidation() {
     checkState(nonNull(statistics), "Statistics must be calculated before running validation");
     isThroughputAchieved = getThroughputQps() >= requiredThroughput;
     isErrorThresholdAchieved = statistics.getErrorPercentage() <= (requiredAllowedErrorsRate * 100);
+    isMinLatencyAchieved = validateLatency(statistics.getMinLatency(NANOSECONDS), requiredMinLatency);
+    isMaxLatencyAchieved = validateLatency(statistics.getMaxLatency(NANOSECONDS), requiredMaxLatency);
+    isMeanLatencyAchieved = validateLatency(statistics.getMeanLatency(NANOSECONDS), requiredMeanLatency);
     percentileResults = evaluateLatencyPercentiles();
-    isSuccessful = isThroughputAchieved && isErrorThresholdAchieved && noLatencyPercentileFailures();
+
+    isSuccessful = isThroughputAchieved &&
+      isMaxLatencyAchieved &&
+      isMinLatencyAchieved &&
+      isMeanLatencyAchieved &&
+      isErrorThresholdAchieved &&
+      noLatencyPercentileFailures();
+  }
+
+  private boolean validateLatency(float actualNs, float requiredMs) {
+    long thresholdNs = (long)(requiredMs * MILLISECONDS.toNanos(1));
+    return requiredMaxLatency < 0 || actualNs <= thresholdNs;
   }
 
   private boolean noLatencyPercentileFailures() {

src/main/java/com/github/noconnor/junitperf/reporting/providers/ConsoleReportGenerator.java

@@ -36,9 +36,15 @@ public void generateReport(Set<EvaluationContext> testContexts) {
         context.getThroughputQps(),
         context.getRequiredThroughput(),
         throughputStatus);
-      log.info("Min. latency:   {}ms", statistics.getMinLatency(MILLISECONDS));
-      log.info("Max latency:    {}ms", statistics.getMaxLatency(MILLISECONDS));
-      log.info("Ave latency:    {}ms", statistics.getMeanLatency(MILLISECONDS));
+      log.info("Min. latency:   {}ms (Required: {}ms) - {}",
+        statistics.getMinLatency(MILLISECONDS),
+        context.getRequiredMinLatency());
+      log.info("Max latency:    {}ms (Required: {}ms) - {}",
+        statistics.getMaxLatency(MILLISECONDS),
+        context.getRequiredMaxLatency());
+      log.info("Ave latency:    {}ms (Required: {}ms) - {}",
+        statistics.getMeanLatency(MILLISECONDS),
+        context.getRequiredMeanLatency());
       context.getRequiredPercentiles().forEach((percentile, threshold) -> {
         String percentileStatus = context.getPercentileResults().get(percentile) ? PASSED : FAILED;
         log.info("{}:    {}ms (Required: {}ms) - {}",

src/main/resources/templates/report.twig

@@ -31,7 +31,7 @@
 
         {% for context in contextData %}
         <a name='{{ context.testName }}'><h2 style='color:#2b67a4'>{{ context.testName }}</h2></a>
-        <table width='900'>
+        <table width='970'>
             <tr>
                 <td>
                     <!-- ADD scatter Chart here!! -->
@@ -131,14 +131,16 @@
                             <td align='right'><b style='color:{{ colour }};'>{{ number_format(context.requiredThroughput, 0, ".", ",") }} / s</b></td>
                         </tr>
                         <tr>
-                            <th align='right' valign='top'>Min. latency:</th>
-                            <td align='right'>{{ number_format(context.statistics.getMinLatency(milliseconds), 2, ".", " ") }} ms</td>
-                            <td align='right'></td>
+                            {% set colour = (context.isMinLatencyAchieved) ? "#2b67a4" : "#d9534f" %}
+                            <th align='right' valign='top'><b style='color:{{ colour }}'>Min. latency:</b></th>
+                            <td align='right'><b style='color:{{ colour }}'>{{ number_format(context.statistics.getMinLatency(milliseconds), 2, ".", " ") }} ms</b></td>
+                            <td align='right'><b style='color:{{ colour }}'>{{ number_format(context.requiredMinLatency, 2, ".", " ") }} ms</b></td>
                         </tr>
                         <tr>
-                            <th align='right' valign='top'>Average latency:</th>
-                            <td align='right'>{{ number_format(context.statistics.getMeanLatency(milliseconds), 2, ".", " ") }} ms</td>
-                            <td align='right'></td>
+                            {% set colour = (context.isMeanLatencyAchieved) ? "#2b67a4" : "#d9534f" %}
+                            <th align='right' valign='top'><b style='color:{{ colour }}'>Average latency:</b></th>
+                            <td align='right'><b style='color:{{ colour }}'>{{ number_format(context.statistics.getMeanLatency(milliseconds), 2, ".", " ") }} ms</b></td>
+                            <td align='right'><b style='color:{{ colour }}'>{{ number_format(context.requiredMeanLatency, 2, ".", " ") }} ms</b></td>
                         </tr>
 
                         {% for percentile, target in context.requiredPercentiles %}
@@ -152,9 +154,10 @@
                         </tr>
                         {% endfor %}
                         <tr>
-                            <th align='right' valign='top'>Max latency:</th>
-                            <td align='right'>{{ number_format(context.statistics.getMaxLatency(milliseconds), 2, ".", ",") }} ms</td>
-                            <td align='right'></td>
+                            {% set colour = (context.isMaxLatencyAchieved) ? "#2b67a4" : "#d9534f" %}
+                            <th align='right' valign='top'><b style='color:{{ colour }}'>Max latency:</b></th>
+                            <td align='right'><b style='color:{{ colour }}'>{{ number_format(context.statistics.getMaxLatency(milliseconds), 2, ".", ",") }} ms</b></td>
+                            <td align='right'><b style='color:{{ colour }}'>{{ number_format(context.requiredMaxLatency, 2, ".", " ") }} ms</b></td>
                         </tr>
                     </table>
                 </td>

src/test/java/com/github/noconnor/junitperf/data/EvaluationContextTest.java

@@ -232,6 +232,57 @@ public void whenCalculatingThroughputQps_thenCorrectValueShouldBeCalculated() {
     assertThat(context.getThroughputQps(), is(expected));
   }
 
+  @Test
+  public void whenRunningEvaluation_thenMinLatencyRequirementsShouldBeChecked() {
+    initialiseContext();
+    context.runValidation();
+    assertThat(context.isMinLatencyAchieved(), is(true));
+    assertThat(context.isSuccessful(), is(true));
+  }
+
+  @Test
+  public void whenRunningEvaluation_andMinLatencyCheckFails_thenIsSuccessfulShouldBeFalse() {
+    when(statisticsMock.getMinLatency(NANOSECONDS)).thenReturn(60_090_000.9F);
+    initialiseContext();
+    context.runValidation();
+    assertThat(context.isMinLatencyAchieved(), is(false));
+    assertThat(context.isSuccessful(), is(false));
+  }
+
+  @Test
+  public void whenRunningEvaluation_thenMaxLatencyRequirementsShouldBeChecked() {
+    initialiseContext();
+    context.runValidation();
+    assertThat(context.isMaxLatencyAchieved(), is(true));
+    assertThat(context.isSuccessful(), is(true));
+  }
+
+  @Test
+  public void whenRunningEvaluation_andMaxLatencyCheckFails_thenIsSuccessfulShouldBeFalse() {
+    when(statisticsMock.getMaxLatency(NANOSECONDS)).thenReturn(190_090_000.9F);
+    initialiseContext();
+    context.runValidation();
+    assertThat(context.isMaxLatencyAchieved(), is(false));
+    assertThat(context.isSuccessful(), is(false));
+  }
+
+  @Test
+  public void whenRunningEvaluation_thenMeanLatencyRequirementsShouldBeChecked() {
+    initialiseContext();
+    context.runValidation();
+    assertThat(context.isMeanLatencyAchieved(), is(true));
+    assertThat(context.isSuccessful(), is(true));
+  }
+
+  @Test
+  public void whenRunningEvaluation_andMeanLatencyCheckFails_thenIsSuccessfulShouldBeFalse() {
+    when(statisticsMock.getMeanLatency(NANOSECONDS)).thenReturn(10_090_000.9F);
+    initialiseContext();
+    context.runValidation();
+    assertThat(context.isMeanLatencyAchieved(), is(false));
+    assertThat(context.isSuccessful(), is(false));
+  }
+
   private void initialiseContext() {
     context.loadConfiguration(perfTestAnnotation);
     context.loadRequirements(perfTestRequirement);
@@ -249,11 +300,17 @@ private void initialisePerfTestRequirementAnnotation() {
     when(perfTestRequirement.executionsPerSec()).thenReturn(10_000);
     when(perfTestRequirement.allowedErrorPercentage()).thenReturn(0.5f);
     when(perfTestRequirement.percentiles()).thenReturn("90:0.5,95:9");
+    when(perfTestRequirement.meanLatency()).thenReturn(4.8F);
+    when(perfTestRequirement.minLatency()).thenReturn(1.6F);
+    when(perfTestRequirement.maxLatency()).thenReturn(100.6F);
   }
 
   private void initialiseStatisticsMockToPassValidation() {
     when(statisticsMock.getEvaluationCount()).thenReturn(15_000L);
     when(statisticsMock.getErrorCount()).thenReturn(0L);
+    when(statisticsMock.getMaxLatency(NANOSECONDS)).thenReturn(130.0F);
+    when(statisticsMock.getMinLatency(NANOSECONDS)).thenReturn(5.1F);
+    when(statisticsMock.getMeanLatency(NANOSECONDS)).thenReturn(30.9F);
     when(statisticsMock.getErrorPercentage()).thenReturn(0.0F);
     when(statisticsMock.getLatencyPercentile(90, NANOSECONDS)).thenReturn(2000F);
     when(statisticsMock.getLatencyPercentile(95, NANOSECONDS)).thenReturn(4000F);

src/test/java/com/github/noconnor/junitperf/reporting/BaseReportGeneratorTest.java

@@ -106,8 +106,8 @@ private StatisticsCalculator createAllSuccessMock() {
     when(statisticsMock.getMinLatency(NANOSECONDS)).thenReturn(1636367F);
     when(statisticsMock.getMeanLatency(NANOSECONDS)).thenReturn(17540000F);
     when(statisticsMock.getMaxLatency(MILLISECONDS)).thenReturn(38548467F / 1_000_000);
-    when(statisticsMock.getMinLatency(MILLISECONDS)).thenReturn(17540000F / 1_000_000);
-    when(statisticsMock.getMeanLatency(MILLISECONDS)).thenReturn(28343467F / 1_000_000);
+    when(statisticsMock.getMinLatency(MILLISECONDS)).thenReturn(1636367F / 1_000_000);
+    when(statisticsMock.getMeanLatency(MILLISECONDS)).thenReturn(17540000F / 1_000_000);
     return statisticsMock;
   }
 
@@ -119,12 +119,12 @@ private StatisticsCalculator createSomeFailuresMock() {
     when(statisticsMock.getLatencyPercentile(100, NANOSECONDS)).thenReturn(98548467F);
     when(statisticsMock.getLatencyPercentile(98, MILLISECONDS)).thenReturn(1636367F / 1_000_000);
     when(statisticsMock.getLatencyPercentile(99, MILLISECONDS)).thenReturn(28343467F / 1_000_000);
-    when(statisticsMock.getLatencyPercentile(100, MILLISECONDS)).thenReturn(98548467F / 1_000_000);
+    when(statisticsMock.getLatencyPercentile(100, MILLISECONDS)).thenReturn(28343467F / 1_000_000);
     when(statisticsMock.getEvaluationCount()).thenReturn(130_000L);
     when(statisticsMock.getErrorCount()).thenReturn(78_000L);
     when(statisticsMock.getMaxLatency(NANOSECONDS)).thenReturn(38548467F);
-    when(statisticsMock.getMinLatency(NANOSECONDS)).thenReturn(1636367F);
-    when(statisticsMock.getMeanLatency(NANOSECONDS)).thenReturn(17540000F);
+    when(statisticsMock.getMinLatency(NANOSECONDS)).thenReturn(17540000F);
+    when(statisticsMock.getMeanLatency(NANOSECONDS)).thenReturn(28343467F);
     when(statisticsMock.getMaxLatency(MILLISECONDS)).thenReturn(38548467F / 1_000_000);
     when(statisticsMock.getMinLatency(MILLISECONDS)).thenReturn(17540000F / 1_000_000);
     when(statisticsMock.getMeanLatency(MILLISECONDS)).thenReturn(28343467F / 1_000_000);
@@ -139,15 +139,15 @@ private StatisticsCalculator createAllFailureMock() {
     when(statisticsMock.getLatencyPercentile(100, NANOSECONDS)).thenReturn(58548467F);
     when(statisticsMock.getLatencyPercentile(98, MILLISECONDS)).thenReturn(4636367F / 1_000_000);
     when(statisticsMock.getLatencyPercentile(99, MILLISECONDS)).thenReturn(48343467F / 1_000_000);
-    when(statisticsMock.getLatencyPercentile(100, MILLISECONDS)).thenReturn(58548467F / 1_000_000);
+    when(statisticsMock.getLatencyPercentile(100, MILLISECONDS)).thenReturn(234680000F / 1_000_000);
     when(statisticsMock.getEvaluationCount()).thenReturn(1000L);
     when(statisticsMock.getErrorCount()).thenReturn(400L);
-    when(statisticsMock.getMaxLatency(NANOSECONDS)).thenReturn(100002F);
-    when(statisticsMock.getMinLatency(NANOSECONDS)).thenReturn(500000F);
-    when(statisticsMock.getMeanLatency(NANOSECONDS)).thenReturn(600000F);
-    when(statisticsMock.getMaxLatency(MILLISECONDS)).thenReturn(100002F / 1_000_000);
-    when(statisticsMock.getMinLatency(MILLISECONDS)).thenReturn(500000F / 1_000_000);
-    when(statisticsMock.getMeanLatency(MILLISECONDS)).thenReturn(600000F / 1_000_000);
+    when(statisticsMock.getMaxLatency(NANOSECONDS)).thenReturn(234680000F);
+    when(statisticsMock.getMinLatency(NANOSECONDS)).thenReturn(12700000F);
+    when(statisticsMock.getMeanLatency(NANOSECONDS)).thenReturn(61700000F);
+    when(statisticsMock.getMaxLatency(MILLISECONDS)).thenReturn(234680000F / 1_000_000);
+    when(statisticsMock.getMinLatency(MILLISECONDS)).thenReturn(12700000F / 1_000_000);
+    when(statisticsMock.getMeanLatency(MILLISECONDS)).thenReturn(61700000F / 1_000_000);
     return statisticsMock;
   }
 
@@ -162,6 +162,9 @@ protected void initialisePerfTestRequirementAnnotationMock() {
     when(perfTestRequirementAnnotationMock.percentiles()).thenReturn("98:3.3,99:32.6,100:46.9999");
     when(perfTestRequirementAnnotationMock.allowedErrorPercentage()).thenReturn(0.3F);
     when(perfTestRequirementAnnotationMock.executionsPerSec()).thenReturn(13_000);
+    when(perfTestRequirementAnnotationMock.minLatency()).thenReturn(10.0F);
+    when(perfTestRequirementAnnotationMock.maxLatency()).thenReturn(200.66F);
+    when(perfTestRequirementAnnotationMock.meanLatency()).thenReturn(55.1F);
   }
 
   @SuppressWarnings("unused")

src/test/resources/csv/failed.csv

@@ -1,3 +1,3 @@
 testName,duration,threadCount,throughput,minLatencyNs,maxLatencyNs,meanLatencyNs,1st,2nd,3rd,4th,5th,6th,7th,8th,9th,10th,11th,12th,13th,14th,15th,16th,17th,18th,19th,20th,21st,22nd,23rd,24th,25th,26th,27th,28th,29th,30th,31st,32nd,33rd,34th,35th,36th,37th,38th,39th,40th,41st,42nd,43rd,44th,45th,46th,47th,48th,49th,50th,51st,52nd,53rd,54th,55th,56th,57th,58th,59th,60th,61st,62nd,63rd,64th,65th,66th,67th,68th,69th,70th,71st,72nd,73rd,74th,75th,76th,77th,78th,79th,80th,81st,82nd,83rd,84th,85th,86th,87th,88th,89th,90th,91st,92nd,93rd,94th,95th,96th,97th,98th,99th,100th
-unittest1,10000,50,101,0.5000,0.1000,0.6000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,4.6364,48.3435,58.5485
-unittest2,10000,50,101,0.5000,0.1000,0.6000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,4.6364,48.3435,58.5485
+unittest1,10000,50,101,12.7000,234.6800,61.7000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,4.6364,48.3435,234.6800
+unittest2,10000,50,101,12.7000,234.6800,61.7000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,4.6364,48.3435,234.6800