diff --git a/.github/workflows/benchmark-comparision.yaml b/.github/workflows/benchmark-comparision.yaml new file mode 100644 index 000000000..f05a2fb72 --- /dev/null +++ b/.github/workflows/benchmark-comparision.yaml @@ -0,0 +1,68 @@ +# - When a third-party action is added (i.e., `uses`), please also add it to `download-licenses` in Makefile. +# - When a job is added/removed/renamed, please make corresponding changes in ci-docs.yaml. +name: Comparison Benchmark +on: + push: + branches: + - main + paths-ignore: + - '**.md' + - 'contrib/**' + +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + os: + [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + # We need to get all the git tags to make version injection work. See VERSION in Makefile for more detail. + fetch-depth: 0 + submodules: true + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + cache: true + - name: Clean up previous files + run: | + sudo rm -rf /opt/finch + sudo rm -rf ~/.finch + sudo rm -rf ./_output + if pgrep '^qemu-system'; then + sudo pkill '^qemu-system' + fi + if pgrep '^socket_vmnet'; then + sudo pkill '^socket_vmnet' + fi + - name: Install Dependencies + run: | + sudo apt-get install -y lz4 automake autoconf libtool + sudo snap install yq + - name: Build project + run: | + make + - name: Run benchmark + run: make test-benchmark-container | tee benchmark-container.txt + - name: Set OS info as env variable + run: | + echo "OS_VERSION=$(lsb_release -sr)" >> $GITHUB_ENV + echo "ARCH=$(uname -m)" >> $GITHUB_ENV + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 # v1.20.3 + with: + name: Finch Benchmark + tool: 'go' + benchmark-data-dir-path: "dev/bench/comparison/${{ env.OS_VERSION }}/${{ env.ARCH }}" + output-file-path: benchmark-container.txt + - name: Push benchmark result + run: git push 'https://github.com/coderbirju/finch.git' gh-pages:gh-pages diff --git a/Makefile b/Makefile index c5adf413f..5eeca3527 100644 --- a/Makefile +++ b/Makefile @@ -284,7 +284,7 @@ test-benchmark-vm: .PHONY: test-benchmark-container test-benchmark-container: - cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem --installed="$(INSTALLED)" + cd benchmark/container && go test -ldflags $(LDFLAGS) -bench=. -benchmem -benchtime=1x -timeout 2h --installed="$(INSTALLED)" .PHONY: gen-code # Since different projects may have different versions of tool binaries, diff --git a/benchmark/all/all_test.go b/benchmark/all/all_test.go index 1ba257d8d..ea548b9c1 100644 --- a/benchmark/all/all_test.go +++ b/benchmark/all/all_test.go @@ -31,11 +31,11 @@ func BenchmarkAll(b *testing.B) { } b.Run("BenchmarkContainerRun", func(b *testing.B) { - suite.BenchmarkContainerRun(b) + suite.BenchmarkContainerRun(b, "finch") }) b.Run("BenchmarkImageBuild", func(b *testing.B) { - suite.BenchmarkImageBuild(b) + suite.BenchmarkImageBuild(b, "finch") }) err = suite.StopVM() diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go index 941333b04..4f79b42cb 100644 --- a/benchmark/benchmark.go +++ b/benchmark/benchmark.go @@ -57,6 +57,59 @@ func GetSubject() (string, error) { return subject, nil } +// GetDocker gets the path of docker installation +func GetDocker() (string, error) { + subject := filepath.Join("/usr/bin/", "docker") + return subject, nil +} + +// func CleanUpFunc() error { +// cmd0 := exec.Command("sudo", "systemctl", "stop", "docker") +// if err := cmd0.Run(); err != nil { +// return fmt.Errorf("docker stop failed: %v", err) +// } +// cmd1 := exec.Command("sudo", "rm", "-rf", "/var/lib/docker") +// if err := cmd1.Run(); err != nil { +// return fmt.Errorf("docker cleanup failed: %v", err) +// } +// cmd2 := exec.Command("sudo", "rm", "-rf", "/var/run/docker.pid") +// if err := cmd2.Run(); err != nil { +// return fmt.Errorf("failed to stop docker process: %v", err) +// } +// cmd3 := exec.Command("sudo", "systemctl", "restart", "docker") +// if err := cmd3.Run(); err != nil { +// return fmt.Errorf("docker failed to restart: %v", err) +// } + +// cmd4 := exec.Command("sudo", "rm", "-rf", "/var/lib/finch/buildkit/cache.db") +// if err := cmd4.Run(); err != nil { +// return fmt.Errorf("finch buildkit cleanup failed: %v", err) +// } + +// cmd5 := exec.Command("sudo", "docker", "builder", "prune", "-a", "-f") +// if err := cmd5.Run(); err != nil { +// return fmt.Errorf("docker builder prune failed: %v", err) +// } + +// cmd6 := exec.Command("sudo", "docker", "image", "prune", "-a", "-f") +// if err := cmd6.Run(); err != nil { +// return fmt.Errorf("docker image prune failed: %v", err) +// } + +// cmd7 := exec.Command("sudo", "rm", "-rf", "/var/lib/containerd") +// if err := cmd7.Run(); err != nil { +// return fmt.Errorf("containerd cleanup failed: %v", err) +// } + +// cmd8 := exec.Command("sudo", "systemctl", "restart", "containerd") +// if err := cmd8.Run(); err != nil { +// return fmt.Errorf("docker failed to restart: %v", err) +// } + +// return nil + +// } + // Wrapper reports the benchmarking metrics of targetFunc to testing.B. func Wrapper(b *testing.B, targetFunc func(), cleanupFunc func()) { metricsSum := Metrics{} diff --git a/benchmark/container/container_test.go b/benchmark/container/container_test.go index cc84d7432..3970a1be5 100644 --- a/benchmark/container/container_test.go +++ b/benchmark/container/container_test.go @@ -17,11 +17,36 @@ func BenchmarkContainer(b *testing.B) { b.Fatal(err) } - b.Run("BenchmarkContainerRun", func(b *testing.B) { - suite.BenchmarkContainerRun(b) + b.Run("BenchmarkContainerRun-docker", func(b *testing.B) { + suite.BenchmarkContainerRun(b, "docker") }) - b.Run("BenchmarkImageBuild", func(b *testing.B) { - suite.BenchmarkImageBuild(b) + b.Run("BenchmarkContainerRun-finch", func(b *testing.B) { + suite.BenchmarkContainerRun(b, "finch") }) + + b.Run("BenchmarkContainerPull-docker", func(b *testing.B) { + suite.BenchmarkContainerPull(b, "docker") + }) + + b.Run("BenchmarkContainerPull-finch", func(b *testing.B) { + suite.BenchmarkContainerPull(b, "finch") + }) + + b.Run("BenchmarkImageBuild-docker", func(b *testing.B) { + suite.BenchmarkImageBuild(b, "docker") + }) + + b.Run("BenchmarkImageBuild-finch", func(b *testing.B) { + suite.BenchmarkImageBuild(b, "finch") + }) + + b.Run("BenchmarkImageDelete-docker", func(b *testing.B) { + suite.BenchmarkImageDelete(b, "docker") + }) + + b.Run("BenchmarkImageDelete-finch", func(b *testing.B) { + suite.BenchmarkImageDelete(b, "finch") + }) + } diff --git a/benchmark/suite.go b/benchmark/suite.go index ec9b0c7da..6138e92cb 100644 --- a/benchmark/suite.go +++ b/benchmark/suite.go @@ -16,14 +16,17 @@ import ( const ( virtualMachineRootCmd = "vm" - alpineImage = "public.ecr.aws/docker/library/alpine:latest" + alpineImage = "public.ecr.aws/y0o4y9o3/anaconda-pkg-build:latest" testImageName = "test:tag" testContainerName = "ctr-test" + ligthImage = "public.ecr.aws/docker/library/amazonlinux:latest" ) +// public.ecr.aws/soci-workshop-examples/mongo:latest public.ecr.aws/soci-workshop-examples/redis:latest public.ecr.aws/docker/library/alpine:latest // Suite is a struct that groups benchmark functions and shared state. type Suite struct { subject string + docker string } // Setup initializes the Suite by getting the subject. @@ -32,7 +35,13 @@ func (suite *Suite) Setup() error { if err != nil { return err } + + docker, err := GetDocker() + if err != nil { + return err + } suite.subject = subject + suite.docker = docker return nil } @@ -80,18 +89,49 @@ func (suite *Suite) BenchmarkVMStart(b *testing.B) { } // BenchmarkContainerRun measures the metrics to run a container. -func (suite *Suite) BenchmarkContainerRun(b *testing.B) { - assert.NoError(b, exec.Command(suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only - Wrapper(b, func() { - assert.NoError(b, exec.Command(suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only - }) - assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only +func (suite *Suite) BenchmarkContainerRun(b *testing.B, binaryName string) { + // assert.NoError(b, CleanUpFunc()) + if binaryName == "finch" { + assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only + }) + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + } else { + // assert.NoError(b, CleanUpFunc()) + assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "run", "--name", testContainerName, alpineImage).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rm", "--force", testContainerName).Run()) //nolint:gosec // testing only + }) + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + } + +} + +func (suite *Suite) BenchmarkContainerPull(b *testing.B, binaryName string) { + if binaryName == "finch" { + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "pull", alpineImage, "--namespace=finch").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + }) + } else { + assert.NoError(b, exec.Command("sudo", suite.docker, "images", "prune", "-a").Run()) + assert.NoError(b, exec.Command("sudo", suite.docker, "volume", "prune", "-a").Run()) + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "pull", alpineImage).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", alpineImage).Run()) //nolint:gosec // testing only + }) + } } // BenchmarkImageBuild measures the metrics to build an image. -func (suite *Suite) BenchmarkImageBuild(b *testing.B) { +func (suite *Suite) BenchmarkImageBuild(b *testing.B, binaryName string) { homeDir, err := os.UserHomeDir() assert.NoError(b, err) tempDir, err := os.MkdirTemp(homeDir, "finch-test") @@ -103,9 +143,50 @@ func (suite *Suite) BenchmarkImageBuild(b *testing.B) { assert.NoError(b, err) buildContext := filepath.Dir(dockerFilePath) defer os.RemoveAll(buildContext) //nolint:errcheck // testing only - Wrapper(b, func() { - assert.NoError(b, exec.Command(suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only - }, func() { - assert.NoError(b, exec.Command(suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only - }) + // assert.NoError(b, CleanUpFunc()) + if binaryName == "finch" { + assert.NoError(b, exec.Command("sudo", suite.subject, "builder", "prune").Run()) + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext, "--namespace=finch").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }) + } else { + // assert.NoError(b, CleanUpFunc()) + assert.NoError(b, exec.Command("sudo", suite.docker, "builder", "prune").Run()) + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext, "--no-cache").Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }) + } +} + +func (suite *Suite) BenchmarkImageDelete(b *testing.B, binaryImage string) { + homeDir, err := os.UserHomeDir() + assert.NoError(b, err) + tempDir, err := os.MkdirTemp(homeDir, "finch-test") + assert.NoError(b, err) + dockerFilePath := filepath.Join(tempDir, "Dockerfile") + err = os.WriteFile(dockerFilePath, []byte(fmt.Sprintf(`FROM %s + CMD ["echo", "finch-test-dummy-output"] + `, alpineImage)), 0o600) + assert.NoError(b, err) + buildContext := filepath.Dir(dockerFilePath) + defer os.RemoveAll(buildContext) //nolint:errcheck // testing only + if binaryImage == "finch" { + assert.NoError(b, exec.Command("sudo", suite.subject, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.subject, "rmi", "--help").Run()) + }) + } else { + assert.NoError(b, exec.Command("sudo", suite.docker, "build", "--tag", testImageName, buildContext).Run()) //nolint:gosec // testing only + Wrapper(b, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--force", testImageName).Run()) //nolint:gosec // testing only + }, func() { + assert.NoError(b, exec.Command("sudo", suite.docker, "rmi", "--help").Run()) + }) + } }