ci: Add comparison benchmark [WIP]

.github/workflows/benchmark-comparision.yaml

@@ -0,0 +1,68 @@
+# - When a third-party action is added (i.e., `uses`), please also add it to `download-licenses` in Makefile.
+# - When a job is added/removed/renamed, please make corresponding changes in ci-docs.yaml.
+name: Comparison Benchmark
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+      - 'contrib/**'
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  benchmark:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          [ ubuntu-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          # We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail.
+          fetch-depth: 0
+          submodules: true
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Clean up previous files
+        run: |
+          sudo rm -rf /opt/finch
+          sudo rm -rf ~/.finch
+          sudo rm -rf ./_output
+          if pgrep '^qemu-system'; then
+            sudo pkill '^qemu-system'
+          fi
+          if pgrep '^socket_vmnet'; then
+            sudo pkill '^socket_vmnet'
+          fi
+      - name: Install Dependencies
+        run: | 
+          sudo apt-get install -y lz4 automake autoconf libtool
+          sudo snap install yq
+      - name: Build project
+        run: |
+          make
+      - name: Run benchmark
+        run: make test-benchmark-container | tee benchmark-container.txt
+      - name: Set OS info as env variable
+        run: |
+          echo "OS_VERSION=$(lsb_release -sr)" >> $GITHUB_ENV
+          echo "ARCH=$(uname -m)" >> $GITHUB_ENV
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 # v1.20.3
+        with:
+          name: Finch Benchmark
+          tool: 'go'
+          benchmark-data-dir-path: "dev/bench/comparison/${{ env.OS_VERSION }}/${{ env.ARCH }}"
+          output-file-path: benchmark-container.txt
+      - name: Push benchmark result
+        run: git push 'https://github.com/coderbirju/finch.git' gh-pages:gh-pages

Makefile

@@ -284,7 +284,7 @@ test-benchmark-vm:
 
 .PHONY: test-benchmark-container
 test-benchmark-container:
-	cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)"
+	cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem -benchtime=1x -timeout 2h --installed="$(INSTALLED)"
 
 .PHONY: gen-code
 # Since different projects may have different versions of tool binaries,

benchmark/all/all_test.go

@@ -31,11 +31,11 @@ func BenchmarkAll(b *testing.B) {
 	}
 
 	b.Run("BenchmarkContainerRun", func(b *testing.B) {
-		suite.BenchmarkContainerRun(b)
+		suite.BenchmarkContainerRun(b, "finch")
 	})
 
 	b.Run("BenchmarkImageBuild", func(b *testing.B) {
-		suite.BenchmarkImageBuild(b)
+		suite.BenchmarkImageBuild(b, "finch")
 	})
 
 	err = suite.StopVM()

benchmark/benchmark.go

@@ -57,6 +57,59 @@
 	return subject, nil
 }
 
+// GetDocker gets the path of docker installation
+func GetDocker() (string, error) {
+	subject := filepath.Join("/usr/bin/", "docker")
+	return subject, nil
+}
+
+// func CleanUpFunc() error {
+// cmd0 := exec.Command("sudo", "systemctl", "stop", "docker")
+// if err := cmd0.Run(); err != nil {
+// 	return fmt.Errorf("docker stop failed: %v", err)
+// }
+// cmd1 := exec.Command("sudo", "rm", "-rf", "/var/lib/docker")
+// if err := cmd1.Run(); err != nil {
+// 	return fmt.Errorf("docker cleanup failed: %v", err)
+// }
+// cmd2 := exec.Command("sudo", "rm", "-rf", "/var/run/docker.pid")
+// if err := cmd2.Run(); err != nil {
+// 	return fmt.Errorf("failed to stop docker process: %v", err)
+// }
+// cmd3 := exec.Command("sudo", "systemctl", "restart", "docker")
+// if err := cmd3.Run(); err != nil {
+// 	return fmt.Errorf("docker failed to restart: %v", err)
+// }
+
+// cmd4 := exec.Command("sudo", "rm", "-rf", "/var/lib/finch/buildkit/cache.db")
+// if err := cmd4.Run(); err != nil {
+// 	return fmt.Errorf("finch buildkit cleanup failed: %v", err)
+// }
+
+// cmd5 := exec.Command("sudo", "docker", "builder", "prune", "-a", "-f")
+// if err := cmd5.Run(); err != nil {
+// 	return fmt.Errorf("docker builder prune failed: %v", err)
+// }
+
+// cmd6 := exec.Command("sudo", "docker", "image", "prune", "-a", "-f")
+// if err := cmd6.Run(); err != nil {
+// 	return fmt.Errorf("docker image prune failed: %v", err)
+// }
+
+// cmd7 := exec.Command("sudo", "rm", "-rf", "/var/lib/containerd")
+// if err := cmd7.Run(); err != nil {
+// 	return fmt.Errorf("containerd cleanup failed: %v", err)
+// }
+
+// cmd8 := exec.Command("sudo", "systemctl", "restart", "containerd")
+// if err := cmd8.Run(); err != nil {
+// 	return fmt.Errorf("docker failed to restart: %v", err)
+// }
+
+// return nil
+
+// }
+
 // Wrapper reports the benchmarking metrics of targetFunc to testing.B.
 func Wrapper(b *testing.B, targetFunc func(), cleanupFunc func()) {
 	metricsSum := Metrics{}

benchmark/container/container_test.go

@@ -17,11 +17,36 @@
 		b.Fatal(err)
 	}
 
-	b.Run("BenchmarkContainerRun", func(b *testing.B) {
-		suite.BenchmarkContainerRun(b)
+	b.Run("BenchmarkContainerRun-docker", func(b *testing.B) {
+		suite.BenchmarkContainerRun(b, "docker")
 	})
 
-	b.Run("BenchmarkImageBuild", func(b *testing.B) {
-		suite.BenchmarkImageBuild(b)
+	b.Run("BenchmarkContainerRun-finch", func(b *testing.B) {
+		suite.BenchmarkContainerRun(b, "finch")
 	})
+
+	b.Run("BenchmarkContainerPull-docker", func(b *testing.B) {
+		suite.BenchmarkContainerPull(b, "docker")
+	})
+
+	b.Run("BenchmarkContainerPull-finch", func(b *testing.B) {
+		suite.BenchmarkContainerPull(b, "finch")
+	})
+
+	b.Run("BenchmarkImageBuild-docker", func(b *testing.B) {
+		suite.BenchmarkImageBuild(b, "docker")
+	})
+
+	b.Run("BenchmarkImageBuild-finch", func(b *testing.B) {
+		suite.BenchmarkImageBuild(b, "finch")
+	})
+
+	b.Run("BenchmarkImageDelete-docker", func(b *testing.B) {
+		suite.BenchmarkImageDelete(b, "docker")
+	})
+
+	b.Run("BenchmarkImageDelete-finch", func(b *testing.B) {
+		suite.BenchmarkImageDelete(b, "finch")
+	})
+
 }

benchmark/suite.go

@@ -16,14 +16,17 @@
 
 const (
 	virtualMachineRootCmd = "vm"
-	alpineImage           = "public.ecr.aws/docker/library/alpine:latest"
+	alpineImage           = "public.ecr.aws/y0o4y9o3/anaconda-pkg-build:latest"
 	testImageName         = "test:tag"
 	testContainerName     = "ctr-test"
+	ligthImage            = "public.ecr.aws/docker/library/amazonlinux:latest"
 )
 
+// public.ecr.aws/soci-workshop-examples/mongo:latest public.ecr.aws/soci-workshop-examples/redis:latest public.ecr.aws/docker/library/alpine:latest
 // Suite is a struct that groups benchmark functions and shared state.
 type Suite struct {
 	subject string
+	docker  string
 }
 
 // Setup initializes the Suite by getting the subject.
@@ -32,7 +35,13 @@
 	if err != nil {
 		return err
 	}
+
+	docker, err := GetDocker()
+	if err != nil {
+		return err
+	}
 	suite.subject = subject
+	suite.docker = docker
 	return nil
 }
 
@@ -80,18 +89,49 @@
 }
 
 // BenchmarkContainerRun measures the metrics to run a container.
-func (suite *Suite) BenchmarkContainerRun(b *testing.B) {
-	assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
-	Wrapper(b, func() {
-		assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
-	})
-	assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
+func (suite *Suite) BenchmarkContainerRun(b *testing.B, binaryName string) {
+	// assert.NoError(b, CleanUpFunc())
+	if binaryName == "finch" {
+		assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
+		})
+		assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
+	} else {
+		// assert.NoError(b, CleanUpFunc())
+		assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only
+		})
+		assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
+	}
+
+}
+
+func (suite *Suite) BenchmarkContainerPull(b *testing.B, binaryName string) {
+	if binaryName == "finch" {
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage, "--namespace=finch").Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
+		})
+	} else {
+		assert.NoError(b, exec.Command("sudo", suite.docker, "images", "prune", "-a").Run())
+		assert.NoError(b, exec.Command("sudo", suite.docker, "volume", "prune", "-a").Run())
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only
+		})
+	}
 }
 
 // BenchmarkImageBuild measures the metrics to build an image.
-func (suite *Suite) BenchmarkImageBuild(b *testing.B) {
+func (suite *Suite) BenchmarkImageBuild(b *testing.B, binaryName string) {
 	homeDir, err := os.UserHomeDir()
 	assert.NoError(b, err)
 	tempDir, err := os.MkdirTemp(homeDir, "finch-test")
@@ -103,9 +143,50 @@
 	assert.NoError(b, err)
 	buildContext := filepath.Dir(dockerFilePath)
 	defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
-	Wrapper(b, func() {
-		assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
-	}, func() {
-		assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
-	})
+	// assert.NoError(b, CleanUpFunc())
+	if binaryName == "finch" {
+		assert.NoError(b, exec.Command("sudo", suite.subject, "builder", "prune").Run())
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext, "--namespace=finch").Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+		})
+	} else {
+		// assert.NoError(b, CleanUpFunc())
+		assert.NoError(b, exec.Command("sudo", suite.docker, "builder", "prune").Run())
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext, "--no-cache").Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+		})
+	}
+}
+
+func (suite *Suite) BenchmarkImageDelete(b *testing.B, binaryImage string) {
+	homeDir, err := os.UserHomeDir()
+	assert.NoError(b, err)
+	tempDir, err := os.MkdirTemp(homeDir, "finch-test")
+	assert.NoError(b, err)
+	dockerFilePath := filepath.Join(tempDir, "Dockerfile")
+	err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s
+			CMD ["echo", "finch-test-dummy-output"]
+			`, alpineImage)), 0o600)
+	assert.NoError(b, err)
+	buildContext := filepath.Dir(dockerFilePath)
+	defer os.RemoveAll(buildContext) //nolint:errcheck // testing only
+	if binaryImage == "finch" {
+		assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--help").Run())
+		})
+	} else {
+		assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only
+		Wrapper(b, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only
+		}, func() {
+			assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--help").Run())
+		})
+	}
 }