Skip to content

Commit

Permalink
Added new python profiler for performance testing (#89)
Browse files Browse the repository at this point in the history 
*Description of changes:*

Added a new profiler.py script that uses psutils to get per process info
such as Network Bytes Sent/Received, CPU, Resident Memory (Phyical RAM
used), virtual memory and the peak number of threads. Updated the
persisters as well to read the new data and populate `summary.txt` and
`results.csv` as expected. Below is an example from a test run.

Also added a bunch of TODO comments to clean up old JFR logic that is no
longer required here and to keep this PR changes to a minimum.

```
----------------------------------------------------------
 Run at Mon Mar 04 15:53:15 PST 2024
 all-100-tps : Compares all DistroConfigs (100TPS test)
 5 users, 10s duration
----------------------------------------------------------
DistroConfig                  :                      none     app_signals_disabled    app_signals_no_traces       app_signals_traces
Run duration                  :                  00:00:19                 00:00:15                 00:00:21                 00:00:17
Startup time (ms)             :                      4040                     5051                     5042                     5058
Req. Count                    :                    504.00                   504.00                   462.00                   448.00
Req. Rate                     :                     31.97                    44.39                    25.53                    33.31
Req. Lat. mean (ms)           :                    113.25                   106.52                   130.56                   121.27
Req. Lat. p0 (ms)             :                      1.56                     2.11                     2.09                     2.07
Req. Lat. p50 (ms)            :                     54.95                    56.82                    56.22                    56.85
Req. Lat. p90 (ms)            :                    336.17                   340.21                   353.73                   350.77
Req. Lat. p99 (ms)            :                    420.83                   429.16                   441.63                   458.23
Req. Lat. p100 (ms)           :                   5410.95                  5436.79                 10113.78                  5405.26
Net Sent mean (B)             :                  47226.00                 64212.00                 39314.00                 49675.00
Net Sent p0 (B)               :                      0.00                     0.00                     0.00                     0.00
Net Sent p50 (B)              :                  62080.00                 72615.00                 29153.00                 55584.00
Net Sent p90 (B)              :                  97788.00                104820.00                104455.00                 87291.00
Net Sent p99 (B)              :                 107597.00                128103.00                112795.00                111525.00
Net Sent p100 (B)             :                 107597.00                128103.00                112795.00                111525.00
Net Recv mean (B)             :                  46004.00                 58401.00                 38055.00                 45899.00
Net Recv p0 (B)               :                      0.00                     0.00                     0.00                     0.00
Net Recv p50 (B)              :                  60268.00                 70027.00                 29353.00                 55344.00
Net Recv p90 (B)              :                  94835.00                 94095.00                102147.00                 81180.00
Net Recv p99 (B)              :                 105284.00                101951.00                108862.00                107865.00
Net Recv p100 (B)             :                 105284.00                101951.00                108862.00                107865.00
CPU Usage mean %              :                      1.73                     2.71                     1.81                     2.17
CPU Usage p0 %                :                      0.00                     0.00                     0.00                     0.00
CPU Usage p50 %               :                      2.07                     2.99                     1.24                     2.49
CPU Usage p90 %               :                      3.73                     4.56                     4.81                     4.48
CPU Usage p99 %               :                      4.23                     4.73                     5.22                     5.30
CPU Usage p100 %              :                      4.23                     4.73                     5.22                     5.30
RSS Mem mean (MB)             :                     50.65                    81.07                    83.21                    80.96
RSS Mem p0 (MB)               :                     49.78                    78.30                    79.71                    79.33
RSS Mem p50 (MB)              :                     50.78                    81.30                    83.53                    81.21
RSS Mem p90 (MB)              :                     50.91                    82.17                    83.53                    81.58
RSS Mem p99 (MB)              :                     50.91                    82.17                    87.15                    81.58
RSS Mem p100 (MB)             :                     50.91                    82.17                    87.15                    81.58
VMS Mem mean (MB)             :                    445.26                  2100.96                  1928.66                  2114.42
VMS Mem p0 (MB)               :                    203.14                  1746.01                  1601.97                  1746.04
VMS Mem p50 (MB)              :                    491.16                  2179.04                  1962.99                  2179.07
VMS Mem p90 (MB)              :                    491.16                  2179.04                  1962.99                  2179.07
VMS Mem p99 (MB)              :                    491.16                  2179.04                  1964.99                  2179.07
VMS Mem p100 (MB)             :                    491.16                  2179.04                  1964.99                  2179.07
Peak threads                  :                         7                       29                       27                       29


```

### Testing

```
asakem@88665a24d661 performance-tests % ./gradlew  test

Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.

You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.

For more on this, please refer to https://docs.gradle.org/8.6/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.

BUILD SUCCESSFUL in 15m 14s
3 actionable tasks: 2 executed, 1 up-to-date

```

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.
  • Loading branch information
@AsakerMohd
AsakerMohd authored Mar 5, 2024
1 parent b81f2f9 commit bdd81df
Show file tree
Hide file tree
Showing 11 changed files with 515 additions and 52 deletions.
3 changes: 2 additions & 1 deletion performance-tests/Dockerfile-VehicleInventoryService-base
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,5 @@ COPY ./sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp /ve
COPY ./dist/$DISTRO /vehicle-inventory-app/

# Install dependencies and distro
RUN pip install --upgrade pip && pip install -r requirements.txt && pip install ${DISTRO} --force-reinstall
RUN pip install --upgrade pip && pip install -r requirements.txt && pip install psutil && pip install ${DISTRO} \
--force-reinstall
17 changes: 7 additions & 10 deletions performance-tests/src/test/java/io/opentelemetry/OverheadTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
imageService.start();

GenericContainer<?> vehicleInventoryService =
new VehicleInventoryServiceContainer(NETWORK, collector, distroConfig).build();
new VehicleInventoryServiceContainer(NETWORK, collector, distroConfig, namingConventions)
.build();
long start = System.currentTimeMillis();
vehicleInventoryService.start();
writeStartupTimeFile(distroConfig, start);
Expand All @@ -114,7 +115,7 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {
}

long testStart = System.currentTimeMillis();
// startRecording(distroConfig, vehicleInventoryService);
startRecording(distroConfig, vehicleInventoryService);

GenericContainer<?> k6 =
new K6Container(NETWORK, distroConfig, config, namingConventions).build();
Expand All @@ -130,15 +131,11 @@ void runAppOnce(TestConfig config, DistroConfig distroConfig) throws Exception {

private void startRecording(
DistroConfig distroConfig, GenericContainer<?> vehicleInventoryService) throws Exception {
Path outFile = namingConventions.container.jfrFile(distroConfig);
String[] command = {
"jcmd",
"1",
"JFR.start",
"settings=/app/overhead.jfc",
"dumponexit=true",
"name=petclinic",
"filename=" + outFile
"sh",
"executeProfiler.sh",
namingConventions.container.performanceMetricsFileWithoutPath(distroConfig),
namingConventions.container.root()
};
vehicleInventoryService.execInContainer(command);
}
Expand Down
16 changes: 15 additions & 1 deletion ...nce-tests/src/test/java/io/opentelemetry/containers/VehicleInventoryServiceContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package io.opentelemetry.containers;

import io.opentelemetry.distros.DistroConfig;
import io.opentelemetry.util.NamingConventions;
import io.opentelemetry.util.RuntimeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -28,12 +29,17 @@ public class VehicleInventoryServiceContainer {
private final Network network;
private final Startable collector;
private final DistroConfig distroConfig;
private final NamingConventions namingConventions;

public VehicleInventoryServiceContainer(
Network network, Startable collector, DistroConfig distroConfig) {
Network network,
Startable collector,
DistroConfig distroConfig,
NamingConventions namingConventions) {
this.network = network;
this.collector = collector;
this.distroConfig = distroConfig;
this.namingConventions = namingConventions;
}

public GenericContainer<?> build() {
Expand All @@ -44,9 +50,17 @@ public GenericContainer<?> build() {
.withLogConsumer(new Slf4jLogConsumer(logger))
.withExposedPorts(PORT)
.waitingFor(Wait.forHttp("/vehicle-inventory/health-check").forPort(PORT))
.withFileSystemBind(
namingConventions.localResults(), namingConventions.containerResults())
.withCopyFileToContainer(
MountableFile.forClasspathResource("runVehicleInventory.sh"),
"vehicle-inventory-app/run.sh")
.withCopyFileToContainer(
MountableFile.forClasspathResource("profiler.py"),
"vehicle-inventory-app/profiler.py")
.withCopyFileToContainer(
MountableFile.forClasspathResource("executeProfiler.sh"),
"vehicle-inventory-app/executeProfiler.sh")
.withEnv("DJANGO_SETTINGS_MODULE", "VehicleInventoryApp.settings")
.withEnv("PORT", Integer.toString(PORT))
.withEnv("POSTGRES_DATABASE", PostgresContainer.DATABASE_NAME)
Expand Down
111 changes: 102 additions & 9 deletions performance-tests/src/test/java/io/opentelemetry/results/AppPerfResults.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,49 @@ public class AppPerfResults {
final double requestLatencyP90;
final double requestLatencyP99;
final double requestLatencyP100;
final long networkBytesSentAvg;
final long networkBytesSentP0;
final long networkBytesSentP50;
final long networkBytesSentP90;
final long networkBytesSentP99;
final long networkBytesSentP100;
final long networkBytesRecvAvg;
final long networkBytesRecvP0;
final long networkBytesRecvP50;
final long networkBytesRecvP90;
final long networkBytesRecvP99;
final long networkBytesRecvP100;
final double cpuAvg;
final double cpuP0;
final double cpuP50;
final double cpuP90;
final double cpuP99;
final double cpuP100;
final long rssMemAvg;
final long rssMemP0;
final long rssMemP50;
final long rssMemP90;
final long rssMemP99;
final long rssMemP100;
final long vmsMemAvg;
final long vmsMemP0;
final long vmsMemP50;
final long vmsMemP90;
final long vmsMemP99;
final long vmsMemP100;
final long peakThreadCount;
final long startupDurationMs;
final long runDurationMs;
// TODO: cleanup
final long totalGCTime;
final long totalAllocated;
final MinMax heapUsed;
final float maxThreadContextSwitchRate;
final long startupDurationMs;
final long peakThreadCount;
final long averageNetworkRead;
final long averageNetworkWrite;
final float averageJvmUserCpu;
final float maxJvmUserCpu;
final float averageMachineCpuTotal;
final long runDurationMs;
final long totalGcPauseNanos;

private AppPerfResults(Builder builder) {
Expand All @@ -46,18 +77,48 @@ private AppPerfResults(Builder builder) {
this.requestLatencyP90 = builder.requestLatencyP90;
this.requestLatencyP99 = builder.requestLatencyP99;
this.requestLatencyP100 = builder.requestLatencyP100;
this.networkBytesSentAvg = builder.networkBytesSentAvg;
this.networkBytesSentP0 = builder.networkBytesSentP0;
this.networkBytesSentP50 = builder.networkBytesSentP50;
this.networkBytesSentP90 = builder.networkBytesSentP90;
this.networkBytesSentP99 = builder.networkBytesSentP99;
this.networkBytesSentP100 = builder.networkBytesSentP100;
this.networkBytesRecvAvg = builder.networkBytesRecvAvg;
this.networkBytesRecvP0 = builder.networkBytesRecvP0;
this.networkBytesRecvP50 = builder.networkBytesRecvP50;
this.networkBytesRecvP90 = builder.networkBytesRecvP90;
this.networkBytesRecvP99 = builder.networkBytesRecvP99;
this.networkBytesRecvP100 = builder.networkBytesRecvP100;
this.cpuAvg = builder.cpuAvg;
this.cpuP0 = builder.cpuP0;
this.cpuP50 = builder.cpuP50;
this.cpuP90 = builder.cpuP90;
this.cpuP99 = builder.cpuP99;
this.cpuP100 = builder.cpuP100;
this.rssMemAvg = builder.rssMemAvg;
this.rssMemP0 = builder.rssMemP0;
this.rssMemP50 = builder.rssMemP50;
this.rssMemP90 = builder.rssMemP90;
this.rssMemP99 = builder.rssMemP99;
this.rssMemP100 = builder.rssMemP100;
this.vmsMemAvg = builder.vmsMemAvg;
this.vmsMemP0 = builder.vmsMemP0;
this.vmsMemP50 = builder.vmsMemP50;
this.vmsMemP90 = builder.vmsMemP90;
this.vmsMemP99 = builder.vmsMemP99;
this.vmsMemP100 = builder.vmsMemP100;
this.peakThreadCount = builder.peakThreadCount;
this.startupDurationMs = builder.startupDurationMs;
this.runDurationMs = builder.runDurationMs;
this.totalGCTime = builder.totalGCTime;
this.totalAllocated = builder.totalAllocated;
this.heapUsed = builder.heapUsed;
this.maxThreadContextSwitchRate = builder.maxThreadContextSwitchRate;
this.startupDurationMs = builder.startupDurationMs;
this.peakThreadCount = builder.peakThreadCount;
this.averageNetworkRead = builder.averageNetworkRead;
this.averageNetworkWrite = builder.averageNetworkWrite;
this.averageJvmUserCpu = builder.averageJvmUserCpu;
this.maxJvmUserCpu = builder.maxJvmUserCpu;
this.averageMachineCpuTotal = builder.averageMachineCpuTotal;
this.runDurationMs = builder.runDurationMs;
this.totalGcPauseNanos = builder.totalGcPauseNanos;
}

Expand All @@ -73,7 +134,7 @@ private AppPerfResults(Builder builder) {
return bytesToMegs(this.heapUsed.max);
}

private double bytesToMegs(long x) {
public double bytesToMegs(long x) {
return x / (1024.0 * 1024.0);
}

Expand All @@ -97,17 +158,48 @@ static class Builder {
public double requestLatencyP90;
public double requestLatencyP99;
public double requestLatencyP100;
public long networkBytesSentAvg;
public long networkBytesSentP0;
public long networkBytesSentP50;
public long networkBytesSentP90;
public long networkBytesSentP99;
public long networkBytesSentP100;
public long networkBytesRecvAvg;
public long networkBytesRecvP0;
public long networkBytesRecvP50;
public long networkBytesRecvP90;
public long networkBytesRecvP99;
public long networkBytesRecvP100;
public double cpuAvg;
public double cpuP0;
public double cpuP50;
public double cpuP90;
public double cpuP99;
public double cpuP100;
public long rssMemAvg;
public long rssMemP0;
public long rssMemP50;
public long rssMemP90;
public long rssMemP99;
public long rssMemP100;
public long vmsMemAvg;
public long vmsMemP0;
public long vmsMemP50;
public long vmsMemP90;
public long vmsMemP99;
public long vmsMemP100;
public long peakThreadCount;
public long runDurationMs;
// TODO: cleanup
private long totalGCTime;
private long totalAllocated;
private MinMax heapUsed;
private float maxThreadContextSwitchRate;
private long peakThreadCount;
public long averageNetworkRead;
public long averageNetworkWrite;
public float averageJvmUserCpu;
public float maxJvmUserCpu;
public float averageMachineCpuTotal;
public long runDurationMs;
public long totalGcPauseNanos;

AppPerfResults build() {
Expand All @@ -124,6 +216,7 @@ Builder config(TestConfig config) {
return this;
}

// TODO: cleanup
Builder totalGCTime(long totalGCTime) {
this.totalGCTime = totalGCTime;
return this;
Expand Down
51 changes: 37 additions & 14 deletions performance-tests/src/test/java/io/opentelemetry/results/CsvPersister.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@

package io.opentelemetry.results;

import static java.util.concurrent.TimeUnit.NANOSECONDS;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
Expand All @@ -23,11 +21,6 @@ class CsvPersister implements ResultsPersister {
private static final List<FieldSpec> FIELDS =
Arrays.asList(
FieldSpec.of("startupDurationMs", r -> r.startupDurationMs),
// FieldSpec.of("minHeapUsed", r -> r.heapUsed.min),
// FieldSpec.of("maxHeapUsed", r -> r.heapUsed.max),
FieldSpec.of("totalAllocatedMB", r -> r.getTotalAllocatedMB()),
FieldSpec.of("totalGCTime", r -> r.totalGCTime),
FieldSpec.of("maxThreadContextSwitchRate", r -> r.maxThreadContextSwitchRate),
FieldSpec.of("requestCount", r -> r.requestCount),
FieldSpec.of("requestRate", r -> r.requestRate),
FieldSpec.of("requestLatencyAvg", r -> r.requestLatencyAvg),
Expand All @@ -36,14 +29,44 @@ class CsvPersister implements ResultsPersister {
FieldSpec.of("requestLatencyP90", r -> r.requestLatencyP90),
FieldSpec.of("requestLatencyP99", r -> r.requestLatencyP99),
FieldSpec.of("requestLatencyP100", r -> r.requestLatencyP100),
FieldSpec.of("netReadAvg", r -> r.averageNetworkRead),
FieldSpec.of("netWriteAvg", r -> r.averageNetworkWrite),
FieldSpec.of("networkBytesSentAvg", r -> r.networkBytesSentAvg),
FieldSpec.of("networkBytesSentP0", r -> r.networkBytesSentP0),
FieldSpec.of("networkBytesSentP50", r -> r.networkBytesSentP50),
FieldSpec.of("networkBytesSentP90", r -> r.networkBytesSentP90),
FieldSpec.of("networkBytesSentP99", r -> r.networkBytesSentP99),
FieldSpec.of("networkBytesSentP100", r -> r.networkBytesSentP100),
FieldSpec.of("networkBytesRecvAvg", r -> r.networkBytesRecvAvg),
FieldSpec.of("networkBytesRecvP0", r -> r.networkBytesRecvP0),
FieldSpec.of("networkBytesRecvP50", r -> r.networkBytesRecvP50),
FieldSpec.of("networkBytesRecvP90", r -> r.networkBytesRecvP90),
FieldSpec.of("networkBytesRecvP99", r -> r.networkBytesRecvP99),
FieldSpec.of("networkBytesRecvP100", r -> r.networkBytesRecvP100),
FieldSpec.of("cpuAvg", r -> r.cpuAvg),
FieldSpec.of("cpuP0", r -> r.cpuP0),
FieldSpec.of("cpuP50", r -> r.cpuP50),
FieldSpec.of("cpuP90", r -> r.cpuP90),
FieldSpec.of("cpuP99", r -> r.cpuP99),
FieldSpec.of("cpuP100", r -> r.cpuP100),
FieldSpec.of("cpuAvg", r -> r.cpuAvg),
FieldSpec.of("cpuP0", r -> r.cpuP0),
FieldSpec.of("cpuP50", r -> r.cpuP50),
FieldSpec.of("cpuP90", r -> r.cpuP90),
FieldSpec.of("cpuP99", r -> r.cpuP99),
FieldSpec.of("cpuP100", r -> r.cpuP100),
FieldSpec.of("rssMemAvg", r -> r.rssMemAvg),
FieldSpec.of("rssMemP0", r -> r.rssMemP0),
FieldSpec.of("rssMemP50", r -> r.rssMemP50),
FieldSpec.of("rssMemP90", r -> r.rssMemP90),
FieldSpec.of("rssMemP99", r -> r.rssMemP99),
FieldSpec.of("rssMemP100", r -> r.rssMemP100),
FieldSpec.of("vmsMemAvg", r -> r.vmsMemAvg),
FieldSpec.of("vmsMemP0", r -> r.vmsMemP0),
FieldSpec.of("vmsMemP50", r -> r.vmsMemP50),
FieldSpec.of("vmsMemP90", r -> r.vmsMemP90),
FieldSpec.of("vmsMemP99", r -> r.vmsMemP99),
FieldSpec.of("vmsMemP100", r -> r.vmsMemP100),
FieldSpec.of("peakThreadCount", r -> r.peakThreadCount),
FieldSpec.of("averageCpuUser", r -> r.averageJvmUserCpu),
FieldSpec.of("maxCpuUser", r -> r.maxJvmUserCpu),
FieldSpec.of("averageMachineCpuTotal", r -> r.averageMachineCpuTotal),
FieldSpec.of("runDurationMs", r -> r.runDurationMs),
FieldSpec.of("gcPauseMs", r -> NANOSECONDS.toMillis(r.totalGcPauseNanos)));
FieldSpec.of("runDurationMs", r -> r.runDurationMs));

private final Path resultsFile;

Expand Down
Loading

0 comments on commit bdd81df

Please sign in to comment.