refactor(latencygen): make generator support other csv files

This allows to switch the latency matrix to another supported csv file; currently aws.csv and wonderproxy.csv. To support other csv files, they can be added to the latencies folder, and then you can run the generator like this: latencygen -file other_latencies.csv
relab · Dec 24, 2024 · edc0f39 · edc0f39
1 parent 4a0f249
commit edc0f39
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 10 deletions.
diff --git a/.vscode/dict.txt b/.vscode/dict.txt
@@ -108,6 +108,7 @@ Warnf
 wcfg
 Welford
 Welford's
+wonderproxy
 wrfs
 xlabel
 xyer

diff --git a/Makefile b/Makefile
@@ -39,7 +39,7 @@ clean:
 	@rm -fv $(binaries)
 
 latencies:
-	@go run cmd/latencygen/main.go -dest internal/latencies/latency_matrix.go
+	@go run cmd/latencygen/main.go -file wonderproxy.csv
 
 %.pb.go %_gorums.pb.go : %.proto
 	protoc -I=.:$(proto_include) \

diff --git a/cmd/latencygen/latencies/aws.csv b/cmd/latencygen/latencies/aws.csv
@@ -0,0 +1,22 @@
+,Africa (Cape Town) af-south-1,Asia Pacific (Hong Kong) ap-east-1,Asia Pacific (Tokyo) ap-northeast-1,Asia Pacific (Seoul) ap-northeast-2,Asia Pacific (Osaka) ap-northeast-3,Asia Pacific (Mumbai) ap-south-1,Asia Pacific (Singapore) ap-southeast-1,Asia Pacific (Sydney) ap-southeast-2,Canada (Central) ca-central-1,EU (Frankfurt) eu-central-1,EU (Stockholm) eu-north-1,EU (Milan) eu-south-1,EU (Ireland) eu-west-1,EU (London) eu-west-2,EU (Paris) eu-west-3,Middle East (Bahrain) me-south-1,SA (São Paulo) sa-east-1,US East (N. Virginia) us-east-1,US East (Ohio) us-east-2,US West (N. California) us-west-1,US West (Oregon) us-west-2
+Africa (Cape Town) af-south-1,7.13,302.56,379.96,409.66,381.06,208.57,258.71,433.38,248.23,197.88,195.75,212.04,165.2,161.44,173.32,204.43,358.47,228.98,235.84,309.93,278.49
+Asia Pacific (Hong Kong) ap-east-1,283.19,2.62,55.7,40.27,52.65,98.64,42.14,136.67,201.82,204.95,224.83,198.24,224.68,211.53,205.74,125.94,316.47,204.58,191.74,160.52,147.23
+Asia Pacific (Tokyo) ap-northeast-1,359.34,54.8,3.69,35.67,11.22,134.83,78.51,114.19,145.51,232.69,243.7,228.75,203.43,212.24,219.25,167.94,259.44,154.82,143.13,110.75,100.51
+Asia Pacific (Seoul) ap-northeast-2,391.29,39.82,35.53,2.66,31.9,130.01,75.21,147.75,176.82,237.39,265.18,233.87,235.02,243.28,249.11,163.02,290.52,183.3,171.23,138.41,126.77
+Asia Pacific (Osaka) ap-northeast-3,366.26,52.07,11.05,30.81,2.79,126.43,72.6,120.23,152.05,228.96,246.73,220.51,210.23,218.11,224.31,158.26,265.21,152.75,139.74,111.17,100.66
+Asia Pacific (Mumbai) ap-south-1,198.18,96.95,135,130.83,130.04,2.61,59.92,149.74,193.42,119.9,140.12,114.68,124.94,115.65,109.13,44.06,303.59,188.68,199.6,241.08,231.86
+Asia Pacific (Singapore) ap-southeast-1,244.65,40.35,78.33,75.21,72.72,59.63,2.47,94.41,224.84,163.65,183.27,157.58,181.07,171.24,164.63,88.73,334.15,225.64,217.51,184.73,176.35
+Asia Pacific (Sydney) ap-southeast-2,415.02,133.27,113.45,146.93,120.9,148.41,94.01,3.53,198.79,283.17,295.25,249.29,257.26,265.49,280.41,179.77,312.86,199.68,189.59,139.84,141.17
+Canada (Central) ca-central-1,229.3,198.5,145.11,176.28,151.67,192.36,224.4,198.97,3.83,94.86,110.24,104.71,75.38,81.24,87.77,202.98,129.63,18.02,28.59,80.58,63.33
+EU (Frankfurt) eu-central-1,187.02,200.83,232.23,236.81,226.51,118.5,163.74,283.19,94.49,3.05,23.14,12.07,26.13,16.07,10.56,119.97,205.5,90.98,101.26,151.74,143.31
+EU (Stockholm) eu-north-1,179.16,222.74,243.33,265.25,247.43,138.96,183.39,296.03,110.32,23.48,3.59,32.42,41.87,32.24,30.88,142.49,219.3,108.83,117.18,169.03,158.92
+EU (Milan) eu-south-1,193.49,196.1,227.59,229.58,223.04,112.66,156.83,248.86,104.69,12.7,32.39,3.27,36.43,26.6,20.06,113.77,214.84,100.26,110.48,161.46,153.77
+EU (Ireland) eu-west-1,160.73,224.03,204.18,235.55,210.94,124.9,182.15,259.03,76.32,27.05,42.97,37.46,2.66,13.59,20.41,136.04,181.73,69.44,85.16,140.23,122.92
+EU (London) eu-west-2,149.78,207.51,211.03,242.13,217.1,114.12,170.85,265.73,80.9,16.5,31.59,26.22,12.68,3,9.8,124.04,188.9,77.52,87.37,139.43,129.93
+EU (Paris) eu-west-3,156.22,201.77,217.89,247.85,223.52,107.37,164.01,280.41,87.53,10.94,30.46,19.55,19.22,9.92,2.38,118.91,198.47,83.42,93.64,144.2,136.14
+Middle East (Bahrain) me-south-1,188.39,134.28,169.24,169.12,169.62,42.34,90.11,190.38,214.82,122.61,149.05,123.85,137.45,132.88,127.79,2.38,324.66,198.78,210.27,273.61,252.22
+SA (São Paulo) sa-east-1,339.75,315.29,259.07,289.95,266.96,302.71,333.64,313.04,130.03,205.43,219.18,215.32,181.14,189.73,198.68,318.98,3.91,116.92,127.26,178.06,178.02
+US East (N. Virginia) us-east-1,225.68,204.91,156.97,184.59,155.42,190.43,227.86,202.13,19.73,93.01,111.35,102.27,70.45,79.25,85.53,200.33,118.54,5.06,15.32,64.61,72.67
+US East (Ohio) us-east-2,237.19,194.16,145.64,173.14,142.65,201.43,219.84,192.2,31.39,104.17,121.07,113.65,86.56,90.06,96.62,212.69,129.75,16.3,6.12,55.32,53.04
+US West (N. California) us-west-1,287.13,156.28,109.98,137.7,110.8,239.44,184.31,140.01,80.71,151.94,168.69,161.7,139.32,139.77,144.57,259.87,177.53,63.16,52.44,2.7,22.32
+US West (Oregon) us-west-2,277.98,145.94,101.12,127.82,100.45,233.15,177.28,142.13,64.14,144.1,159.87,154.53,123.52,131.07,137.66,253.38,178.83,71.58,51.68,23,3.08
diff --git a/cmd/latencygen/latencies.csv → cmd/latencygen/latencies/wonderproxy.csv b/cmd/latencygen/latencies.csv → cmd/latencygen/latencies/wonderproxy.csv
diff --git a/cmd/latencygen/main.go b/cmd/latencygen/main.go
@@ -2,35 +2,41 @@
 package main
 
 import (
-	_ "embed"
+	"embed"
 	"flag"
 	"log"
 	"os"
+	"path/filepath"
 
 	"github.com/relab/hotstuff/internal/cli"
+	"github.com/relab/hotstuff/internal/root"
 )
 
-//go:embed latencies.csv
-var csvLatencies string
+//go:embed latencies/*.csv
+var csvFiles embed.FS
 
 //go:generate go run .
 
 func main() {
-	dstFile := flag.String("dest", "../../internal/latency/latency_matrix.go", "file path to save latencies to.")
+	latencyFile := flag.String("file", "wonderproxy.csv", "csv file to use for latency matrix (default: wonderproxy)")
 	flag.Parse()
-	if *dstFile == "" {
-		flag.Usage()
-		os.Exit(1)
+
+	csvLatencies, err := csvFiles.ReadFile(filepath.Join("latencies", *latencyFile))
+	if err != nil {
+		log.Fatal(err)
 	}
-	allToAllMatrix, err := cli.ParseCSVLatencies(csvLatencies)
+	allToAllMatrix, err := cli.ParseCSVLatencies(string(csvLatencies))
 	if err != nil {
 		log.Fatal(err)
 	}
 	latenciesGoCode, err := cli.GenerateGoLatencyMatrix(allToAllMatrix)
 	if err != nil {
 		log.Fatal(err)
 	}
-	if err = os.WriteFile(*dstFile, latenciesGoCode, 0o600); err != nil {
+
+	// file path to save generated latencies to.
+	dstFile := filepath.Join(root.Dir, "internal", "latency", "latency_matrix.go")
+	if err = os.WriteFile(dstFile, latenciesGoCode, 0o600); err != nil {
 		log.Fatal(err)
 	}
 }
diff --git a/internal/root/dir.go b/internal/root/dir.go
@@ -0,0 +1,13 @@
+package root
+
+import (
+	"path/filepath"
+	"runtime"
+)
+
+var (
+	_, b, _, _ = runtime.Caller(0)
+
+	// Dir is the root directory of the project.
+	Dir = filepath.Join(filepath.Dir(b), "../..")
+)
-Original file line number
+Diff line change
@@ Expand Up / @@ -108,6 +108,7 @@ Warnf @@
     wcfg
     Welford
     Welford's
+    wonderproxy
     wrfs
     xlabel
     xyer
@@ Expand Down @@