Merge pull request #3 from kaleido-io/perf-report

Perf report

.gitignore

@@ -2,6 +2,7 @@ main
 .DS_Store
 ffperf/ffperf
 ff-perf.log
+ffperf-report.html
 .vscode
 dist/
 *.iml

cmd/run.go

@@ -28,6 +28,7 @@ import (
 	"github.com/hyperledger/firefly-perf-cli/internal/perf"
 	"github.com/hyperledger/firefly-perf-cli/internal/server"
 	"github.com/hyperledger/firefly-perf-cli/internal/types"
+	"github.com/hyperledger/firefly-perf-cli/internal/util"
 	"github.com/hyperledger/firefly/pkg/core"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -65,17 +66,22 @@ Executes a instance within a performance test suite to generate synthetic load a
 			log.Warn("both the \"instance-name\" and \"instance-index\" flags were provided, using \"instance-name\"")
 		}
 
-		instance, err := selectInstance(config)
+		instanceConfig, err := selectInstance(config)
 		if err != nil {
 			return err
 		}
 
-		runnerConfig, err := generateRunnerConfigFromInstance(instance, config)
+		runnerConfig, err := generateRunnerConfigFromInstance(instanceConfig, config)
 		if err != nil {
 			return err
 		}
 
-		perfRunner = perf.New(runnerConfig)
+		configYaml, err := yaml.Marshal(instanceConfig)
+		if err != nil {
+			return err
+		}
+
+		perfRunner = perf.New(runnerConfig, util.NewReportForTestInstance(string(configYaml), instanceName))
 		httpServer = server.NewHttpServer()
 
 		return nil

go.sum

@@ -173,12 +173,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hyperledger/firefly v1.2.0 h1:No82vzsur3TODU0giIECDcMnWQ/8BRGoYo7QK/avr4A=
 github.com/hyperledger/firefly v1.2.0/go.mod h1:tmpTfSjX/NIa7xHTtTb36S48X9+3nNutY7ZxLt3lgCU=
-github.com/hyperledger/firefly-common v1.2.13 h1:4pGL8LusXoijeoxM9J36fzBq4jvZpZbGjpQqgempXMk=
-github.com/hyperledger/firefly-common v1.2.13/go.mod h1:17lOH4YufiPy82LpKm8fPa/YXJ0pUyq01zK1CmklJwM=
-github.com/hyperledger/firefly-common v1.2.14 h1:HON9GJZXvrL0l2AG5DWHSGiBh05hElgFS5lm1OPR83M=
-github.com/hyperledger/firefly-common v1.2.14/go.mod h1:17lOH4YufiPy82LpKm8fPa/YXJ0pUyq01zK1CmklJwM=
-github.com/hyperledger/firefly-common v1.2.15 h1:WdNB65IJvIyiOhVW3nxB3sQKqtJbdJ7ie0PJIM11CSU=
-github.com/hyperledger/firefly-common v1.2.15/go.mod h1:17lOH4YufiPy82LpKm8fPa/YXJ0pUyq01zK1CmklJwM=
 github.com/hyperledger/firefly-common v1.2.16 h1:cVSaxKycOb+/oT2wExbrzxr68aVKQObeBOLaiJ0mTLg=
 github.com/hyperledger/firefly-common v1.2.16/go.mod h1:17lOH4YufiPy82LpKm8fPa/YXJ0pUyq01zK1CmklJwM=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=

internal/conf/conf.go

@@ -81,7 +81,7 @@ type InstanceConfig struct {
 	MaxTimePerAction          time.Duration    `json:"maxTimePerAction,omitempty" yaml:"maxTimePerAction,omitempty"`
 	MaxActions                int64            `json:"maxActions,omitempty" yaml:"maxActions,omitempty"`
 	RampLength                time.Duration    `json:"rampLength,omitempty" yaml:"rampLength,omitempty"`
-	SkipMintConfirmations     bool             `json:"skipMintConfirmations,omitempty" yaml:"skipMintConfirmations,omitempty"`
+	SkipMintConfirmations     bool             `json:"skipMintConfirmations" yaml:"skipMintConfirmations"`
 	DelinquentAction          string           `json:"delinquentAction,omitempty" yaml:"delinquentAction,omitempty"`
 	PerWorkerSigningKeyPrefix string           `json:"perWorkerSigningKeyPrefix,omitempty" yaml:"perWorkerSigningKeyPrefix,omitempty"`
 }
@@ -121,8 +121,8 @@ type TokenConfig struct {
 
 type ContractOptions struct {
 	Address   string `json:"address" yaml:"address"`
-	Channel   string `json:"channel" yaml:"channel"`
-	Chaincode string `json:"chaincode" yaml:"chaincode"`
+	Channel   string `json:"channel,omitempty" yaml:"channel,omitempty"`
+	Chaincode string `json:"chaincode,omitempty" yaml:"chaincode,omitempty"`
 }
 
 type FireFlyWsConfig struct {

internal/perf/perf.go

@@ -156,21 +156,23 @@ type inflightTest struct {
 var mintStartingBalance int
 
 type perfRunner struct {
-	bfr           chan int
-	cfg           *conf.RunnerConfig
-	client        *resty.Client
-	ctx           context.Context
-	shutdown      context.CancelFunc
-	stopping      bool
+	bfr      chan int
+	cfg      *conf.RunnerConfig
+	client   *resty.Client
+	ctx      context.Context
+	shutdown context.CancelFunc
+	stopping bool
+
 	startTime     int64
 	endSendTime   int64
 	endTime       int64
 	startRampTime int64
 	endRampTime   int64
 
-	sendTime    *util.Latency
-	receiveTime *util.Latency
-	totalTime   *util.Latency
+	reportBuilder *util.Report
+	sendTime      *util.Latency
+	receiveTime   *util.Latency
+	totalTime     *util.Latency
 
 	msgTimeMap              map[string]*inflightTest
 	rampSummary             int64
@@ -196,7 +198,7 @@ type SubscriptionInfo struct {
 	Job     fftypes.FFEnum
 }
 
-func New(config *conf.RunnerConfig) PerfRunner {
+func New(config *conf.RunnerConfig, reportBuilder *util.Report) PerfRunner {
 	if config.LogLevel != "" {
 		if level, err := log.ParseLevel(config.LogLevel); err == nil {
 			log.SetLevel(level)
@@ -234,6 +236,7 @@ func New(config *conf.RunnerConfig) PerfRunner {
 		startTime:         startTime,
 		endTime:           endTime,
 		poolName:          poolName,
+		reportBuilder:     reportBuilder,
 		sendTime:          &util.Latency{},
 		receiveTime:       &util.Latency{},
 		totalTime:         &util.Latency{},
@@ -513,9 +516,23 @@ perfLoop:
 
 	pr.stopping = true
 	measuredActions := pr.totalSummary
-	measuredTime := time.Since(time.Unix(pr.startTime, 0)).Seconds()
-	measuredTps := pr.calculateCurrentTps(true)
-	measuredSendTps := pr.calculateSendTps()
+	measuredTime := time.Since(time.Unix(pr.startTime, 0))
+
+	testNames := make([]string, len(pr.cfg.Tests))
+	for i, t := range pr.cfg.Tests {
+		testNames[i] = t.Name.String()
+	}
+	testNameString := testNames[0]
+	if len(testNames) > 1 {
+		testNameString = strings.Join(testNames[:], ",")
+	}
+	tps := util.GenerateTPS(measuredActions, pr.startTime, pr.endSendTime)
+	pr.reportBuilder.AddTestRunMetrics(testNameString, measuredActions, measuredTime, tps, pr.totalTime)
+	err = pr.reportBuilder.GenerateHTML()
+
+	if err != nil {
+		log.Errorf("failed to generate performance report: %+v", err)
+	}
 
 	// we sleep on shutdown / completion to allow for Prometheus metrics to be scraped one final time
 	// After 30 seconds workers should be completed, so we check for delinquent messages
@@ -536,10 +553,10 @@ perfLoop:
 	log.Infof(" - Prometheus metric incomplete_events_total = %f\n", getMetricVal(incompleteEventsCounter))
 	log.Infof(" - Prometheus metric delinquent_msgs_total    = %f\n", getMetricVal(delinquentMsgsCounter))
 	log.Infof(" - Prometheus metric actions_submitted_total = %f\n", getMetricVal(totalActionsCounter))
-	log.Infof(" - Test duration (secs): %2f", measuredTime)
+	log.Infof(" - Test duration: %s", measuredTime)
 	log.Infof(" - Measured actions: %d", measuredActions)
-	log.Infof(" - Measured send TPS: %2f", measuredSendTps)
-	log.Infof(" - Measured throughput: %2f", measuredTps)
+	log.Infof(" - Measured send TPS: %2f", tps.SendRate)
+	log.Infof(" - Measured throughput: %2f", tps.Throughput)
 	log.Infof(" - Measured send duration: %s", pr.sendTime)
 	log.Infof(" - Measured event receiving duration: %s", pr.receiveTime)
 	log.Infof(" - Measured total duration: %s", pr.totalTime)
@@ -1352,14 +1369,6 @@ func (pr *perfRunner) calculateCurrentTps(logValue bool) float64 {
 	}
 	return currentTps
 }
-func (pr *perfRunner) calculateSendTps() float64 {
-	measuredActions := pr.totalSummary
-	sendDuration := time.Duration((pr.endSendTime - pr.startTime) * int64(time.Second))
-	durationSec := sendDuration.Seconds()
-	sendTps := float64(measuredActions) / durationSec
-	log.Infof("Send TPS: %v Measured Actions: %v Duration: %v", sendTps, measuredActions, durationSec)
-	return sendTps
-}
 
 func (pr *perfRunner) ramping() bool {
 	if time.Now().Before(time.Unix(pr.endRampTime, 0)) {

internal/util/report_builder.go

@@ -2,27 +2,208 @@ package util
 
 import (
 	"fmt"
+	"html/template"
+	"os"
 	"sync"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
-type Summary struct {
-	Latency     *Latency `json:"latency"`
-	TPS         *TPS     `json:"tps"`
-	ResultCount *TPS     `json:"resultCount"`
+type TestRunMetrics struct {
+	Name         string
+	TotalActions string
+	Duration     string
+	SendRate     string
+	MinLatency   string
+	MaxLatency   string
+	AvgLatency   string
+	Throughput   string
+}
+type Report struct {
+	RunnerConfig     string
+	TestInstanceName string
+	TestRuns         []TestRunMetrics
 }
 
-type SystemUnderTest struct {
+func (r *Report) GenerateHTML() error {
+	htmlTemplate := `<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>HyperLedger Firefly Performance Report</title>
+    <style>
+        table {
+            font-size: 11px;
+            color: #333333;
+            border-width: 1px;
+            border-color: #666666;
+            border-collapse: collapse;
+            margin-bottom: 10px;
+        }
+
+
+        th {
+            border-width: 1px;
+            font-size: small;
+            padding: 8px;
+            border-style: solid;
+            border-color: #666666;
+            background-color: #f2f2f2;
+        }
+
+        td {
+            border-width: 1px;
+            font-size: small;
+            padding: 8px;
+            border-style: solid;
+            border-color: #666666;
+            background-color: #ffffff;
+            font-weight: 400;
+        }
+
+        body {
+            font-family: Arial, sans-serif;
+            max-width: 800px;
+            margin: 0 auto;
+            padding: 20px;
+        }
+
+        section {
+            margin-bottom: 30px;
+        }
+
+        h2 {
+            font-size: 1.2em;
+            margin-bottom: 10px;
+        }
+
+        h3 {
+            font-size: 1.1em;
+            margin-bottom: 5px;
+        }
+
+        code {
+            display: block;
+            padding: 10px;
+            background-color: #f4f4f4;
+            border: 1px solid #ccc;
+            font-size: 0.9em;
+        }
+    </style>
+</head>
+
+<body>
+    <div class="navbar">
+        <img src="https://www.hyperledger.org/hubfs/hyperledger-firefly_color.png" loading="lazy" alt="Firefly"
+            height="40" class="header-brand-image">
+    </div>
+    <section>
+        <h2>Test runner configuration</h2>
+        <code>
+            <pre>
+{{.RunnerConfig}}
+            </pre>
+        </code>
+    </section>
+
+    <section>
+        <h2>Test metrics</h2>
+        <p>
+            <b>Test instance:</b>{{.TestInstanceName}}
+        </p>
+        <div>
+            <table style="min-width: 100%;">
+                <tr>
+                    <th>Test name</th>
+                    <th>Test duration</th>
+                    <th>Actions</th>
+                    <th>Send TPS</th>
+                    <th>Min Latency</th>
+                    <th>Max Latency</th>
+                    <th>Avg Latency</th>
+                    <th>Throughput</th>
+                </tr>
+                {{range .TestRuns}}
+                <tr>
+                    <td>{{.Name}}</td>
+                    <td>{{.TotalActions}}</td>
+                    <td>{{.Duration}}</td>
+                    <td>{{.SendRate}}</td>
+                    <td>{{.MinLatency}}</td>
+                    <td>{{.MaxLatency}}</td>
+                    <td>{{.AvgLatency}}</td>
+                    <td>{{.Throughput}}</td>
+                </tr>
+                {{end}}
+            </table>
+        </div>
+    </section>
+</body>
+
+</html>`
+	// Execute the template
+	tmpl, err := template.New("template").Parse(htmlTemplate)
+	if err != nil {
+		return err
+	}
+
+	// Create or open the output file
+	outputFile, err := os.Create("ffperf-report.html")
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
+
+	// Write the HTML output to the file
+	err = tmpl.Execute(outputFile, r)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *Report) AddTestRunMetrics(name string, totalActions int64, duration time.Duration, tps *TPS, lt *Latency) {
+	r.TestRuns = append(r.TestRuns, TestRunMetrics{
+		Name:         name,
+		TotalActions: fmt.Sprintf("%d", totalActions),
+		Duration:     fmt.Sprintf("%s", duration),
+		SendRate:     fmt.Sprintf("%f", tps.SendRate),
+		Throughput:   fmt.Sprintf("%f", tps.Throughput),
+		MinLatency:   lt.Min().String(),
+		MaxLatency:   lt.Max().String(),
+		AvgLatency:   lt.Avg().String(),
+	})
+}
+
+func NewReportForTestInstance(runnerConfig string, instanceName string) *Report {
+	return &Report{
+		RunnerConfig:     runnerConfig,
+		TestInstanceName: instanceName,
+		TestRuns:         make([]TestRunMetrics, 0),
+	}
 }
 
 type TPS struct {
 	SendRate   float64 `json:"sendRate"`
 	Throughput float64 `json:"throughput"`
 }
 
-type ResultCount struct {
-	TotalCount int64 `json:"totalCount"`
-	RetryCount int64 `json:"retryCount"`
+func GenerateTPS(totalActions int64, startTime int64, endSendTime int64) *TPS {
+	sendDuration := time.Duration((endSendTime - startTime) * int64(time.Second))
+	sendDurationSec := sendDuration.Seconds()
+	sendRate := float64(totalActions) / sendDurationSec
+
+	totalDurationSec := time.Since(time.Unix(startTime, 0)).Seconds()
+	throughput := float64(totalActions) / totalDurationSec
+	log.Infof("Send rate: %f, Throughput: %f, Measured Actions: %v Duration: %v (Send duration: %v)", sendRate, throughput, totalActions, sendDurationSec, totalDurationSec)
+	return &TPS{
+		SendRate:   sendRate,
+		Throughput: throughput,
+	}
 }
 
 type Latency struct {