add benchmark CI

.github/workflows/benchmark.yaml

@@ -0,0 +1,98 @@
+# Copyright 2021 iLogtail Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Benchmark
+
+on:
+ pull_request:
+ paths-ignore:
+ - 'docs/**'
+ - 'example_config/**'
+ - 'docker/**'
+ - 'k8s_template/**'
+ - 'changes/**'
+ - 'licenses/**'
+ - 'CHANGELOG.md'
+ push:
+ branches:
+ - main
+ - 1.*
+jobs:
+ CI:
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 60
+ strategy:
+ matrix:
+ go-version: [ 1.19.10 ]
+ runner: [ ubuntu-latest ]
+ fail-fast: true
+ steps:
+ # Clean up space to prevent action from running out of disk space.
+ - name: clean
+ if: matrix.runner == 'ubuntu-latest'
+ run: |
+ sudo rm -rf /usr/share/dotnet
+ sudo rm -rf /opt/ghc
+ sudo rm -rf "/usr/local/share/boost"
+ sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+ sudo -E apt-get -qq autoremove --purge
+ sudo -E apt-get -qq clean
+
+ - name: Check disk space
+ run: |
+ df -hT $PWD
+
+ - name: Set up Go ${{ matrix.go-version }}
+ uses: actions/setup-go@v2
+ with:
+ go-version: ${{ matrix.go-version }}
+
+ - name: Check out code
+ uses: actions/checkout@v2
+ with:
+ submodules: true
+
+ - name: Update Docker-compose to v2
+ run: |
+ sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
+ sudo chmod +x /usr/local/bin/docker-compose
+
+ - name: System environment
+ run: |
+ uname -r
+ docker --version
+ go version
+
+ - name: Run benchmark
+ env:
+ BUILD_LOGTAIL_UT: OFF
+ WITHOUTGDB: ON
+ run: make benchmark
+
+ - name: Store benchmark result
+ uses: benchmark-action/github-action-benchmark@v1
+ with:
+ name: benchmark
+ tool: "customSmallerIsBetter"
+ output-file-path: "test/benchmark/report/combined_benchmark.json"
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ auto-push: true
+
+ result:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ needs: [ CI ]
+ steps:
+ - name: Build Result
+ run: echo "Just to make the GitHub merge button green"

Makefile

@@ -185,7 +185,7 @@ e2e-performance: clean docker gocdocker
 
 .PHONY: unittest_e2e_engine
 unittest_e2e_engine: clean gocdocker
- cd test && go test $$(go list ./... | grep -Ev "engine|e2e") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
+ cd test && go test $$(go list ./... | grep -Ev "engine|e2e|benchmark") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
 
 .PHONY: unittest_plugin
 unittest_plugin: clean import_plugins
@@ -209,6 +209,13 @@ unittest_pluginmanager: clean import_plugins
  go test $$(go list ./...|grep -Ev "telegraf|external|envconfig"| grep -E "plugin_main|pluginmanager") -coverprofile .coretestCoverage.txt
  mv ./plugins/input/prometheus/input_prometheus.go.bak ./plugins/input/prometheus/input_prometheus.go
 
+# benchmark
+.PHONY: benchmark
+benchmark: clean gocdocker e2edocker
+ ./scripts/e2e.sh benchmark performance
+ ./scripts/benchmark_collect_result.sh
+
+
 .PHONY: all
 all: clean import_plugins
  ./scripts/gen_build_scripts.sh all $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)

scripts/benchmark_collect_result.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Define the input files and the output file
+input_files=($(find test/benchmark/report -type f -name '*benchmark.json'))
+output_file="test/benchmark/report/combined_benchmark.json"
+
+# Start the output file with an opening square bracket
+rm -f "$output_file"
+touch "$output_file"
+echo '[' > "$output_file"
+
+# Loop through each input file
+for i in "${!input_files[@]}"; do
+ # Read the file, remove the first and last line (the square brackets), and append to the output file
+ cat "${input_files[$i]}" | sed '1d;$d' >> "$output_file"
+
+ # If this is not the last file, append a comma to separate the arrays
+ if [ $i -lt $((${#input_files[@]} - 1)) ]; then
+ echo ',' >> "$output_file"
+ fi
+done
+
+# Finish the output file with a closing square bracket
+echo ']' >> "$output_file"

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml

@@ -1,32 +1,10 @@
 version: '3.8'
 
 services:
- elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:8.14.2
- environment:
- - ELASTIC_PASSWORD=elastic
- - discovery.type=single-node
- - xpack.security.http.ssl.enabled=false
- - xpack.license.self_generated.type=trial
- ports:
- - 9200:9200
- healthcheck:
- test: ["CMD-SHELL", "curl -u elastic:elastic -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
- interval: 10s
- timeout: 5s
- retries: 3
- restart: always
-
  filebeat:
  image: docker.elastic.co/beats/filebeat:8.14.2
  user: root
  volumes:
  - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
  - ./a.log:/home/filebeat/a.log:ro
  command: filebeat -e --strict.perms=false
- environment:
- - OUTPUT_ELASTICSEARCH_HOSTS=["elasticsearch:9200"]
- depends_on:
- elasticsearch:
- condition: service_healthy
- restart: always

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml

@@ -7,11 +7,12 @@ filebeat.inputs:
 processors:
  - decode_json_fields:
  fields: ["message"]
- process_array: false
- max_depth: 1
- target: ""
- overwrite_keys: false
- add_error_key: true
+ target: "json"
+ - drop_event:
+ when:
+ not:
+ equals:
+ json.user-agent: "no-agent"
 
-output.discard:
- enabled: true
+output.console:
+ pretty: true

test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/main.conf

@@ -1,10 +1,16 @@
 [SERVICE]
- Parsers+File /tmp/parsers.conf
+ parsers_file /tmp/parsers.conf
 
 [INPUT]
- Name tail
- Path /home/fluentbit/a.log
+ name tail
+ path /home/fluentbit/a.log
+ parser json
+
+[FILTER]
+ name grep
+ match *
+ regex user-agent no-agent
 
 [OUTPUT]
- Name stdout
- Match non
+ name stdout
+ match *

test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/parsers.conf

@@ -1,3 +1,3 @@
 [PARSER]
- Name docker
+ Name json
  Format json

test/benchmark/test_cases/performance_file_to_blackhole_vector/docker-compose.yaml

@@ -5,5 +5,4 @@ services:
  image: timberio/vector:0.39.0-debian
  volumes:
  - ./vector.yaml:/etc/vector/vector.yaml
- - ./a.log:/home/vector-log/a.log
- restart: always
+ - ./a.log:/home/vector-log/a.log

test/benchmark/test_cases/performance_file_to_blackhole_vector/vector.yaml

@@ -4,8 +4,25 @@ sources:
  include:
  - /home/vector-log/*.log
 
+transforms:
+ transform_json:
+ type: remap
+ inputs:
+ - input_file
+ source: |-
+ . = parse_json!(string!(.message))
+ filter_agent:
+ type: filter
+ inputs:
+ - transform_json
+ condition: 
+ type: "vrl"
+ source: ."usr-agent" == "no-agent"
+
 sinks:
- my_sink_id:
- type: blackhole
+ output_std:
+ type: console
  inputs:
- - input_file
+ - filter_agent
+ encoding:
+ codec: json

test/engine/setup/monitor/monitor.go

@@ -3,10 +3,8 @@ package monitor
 import (
  "context"
  "fmt"
- "log"
  "os"
  "path/filepath"
- "strings"
  "sync/atomic"
  "time"
 
@@ -53,38 +51,21 @@ func StopMonitor(ctx context.Context) (context.Context, error) {
 
 func monitoring(client *client.Client, containerName string) {
  // create csv file
- reportDir := filepath.Join(config.CaseHome, "report")
- reportDir, err := filepath.Abs(reportDir)
- if err != nil {
- log.Fatalf("Failed to get absolute path: %s", err)
- }
- if _, err = os.Stat(reportDir); os.IsNotExist(err) {
- // 文件夹不存在，创建文件夹
- err = os.MkdirAll(reportDir, 0750)
- if err != nil {
- log.Fatalf("Failed to create folder: %s", err)
- }
- }
+ root, _ := filepath.Abs(".")
+ reportDir := root + "/report/"
+ benchmarkFile := reportDir + config.CaseName + "_benchmark.json"
  // new ticker
  ticker := time.NewTicker(interval * time.Second)
  defer ticker.Stop()
  // read from cadvisor per interval seconds
  request := &v1.ContainerInfoRequest{NumStats: 10}
- monitorStatistic := NewMonitorStatistic()
+ monitorStatistic := NewMonitorStatistic(config.CaseName)
  for {
  select {
  case <-stopCh:
  isMonitoring.Store(false)
- var builder strings.Builder
- builder.WriteString("Metric,Value\n")
- builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Max(%)", monitorStatistic.cpu.maxVal))
- builder.WriteString(fmt.Sprintf("%s,%f\n", "CPU Usage Avg(%)", monitorStatistic.cpu.avgVal))
- builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Max(MB)", monitorStatistic.mem.maxVal))
- builder.WriteString(fmt.Sprintf("%s,%f\n", "Memory Usage Avg(MB)", monitorStatistic.mem.avgVal))
- err = os.WriteFile(filepath.Join(reportDir, "monitor.csv"), []byte(builder.String()), 0600)
- if err != nil {
- log.Default().Printf("Failed to write monitor result: %s", err)
- }
+ bytes, _ := monitorStatistic.MarshalJSON()
+ _ = os.WriteFile(benchmarkFile, bytes, 0600)
  return
  case <-ticker.C:
  // 获取容器信息

test/engine/setup/monitor/statistic.go

@@ -1,6 +1,9 @@
 package monitor
 
 import (
+ "encoding/json"
+ "fmt"
+
  v1 "github.com/google/cadvisor/info/v1"
 )
 
@@ -25,13 +28,15 @@ func (s *Info) Add(val float64) {
 }
 
 type Statistic struct {
+ name string
  cpu Info
  mem Info
  lastStat *v1.ContainerStats
 }
 
-func NewMonitorStatistic() *Statistic {
+func NewMonitorStatistic(name string) *Statistic {
  return &Statistic{
+ name: name,
  cpu: Info{},
  mem: Info{},
  lastStat: nil,
@@ -57,3 +62,28 @@ func calculateCPUUsageRate(lastStat, stat *v1.ContainerStats) float64 {
  cpuUsageRateTotal := float64(cpuUsageTotal) * 100 / float64(stat.Timestamp.Sub(lastStat.Timestamp).Nanoseconds())
  return cpuUsageRateTotal
 }
+
+type StatisticItem struct {
+ Name string `json:"name"`
+ Value float64 `json:"value"`
+ Unit string `json:"unit"`
+}
+
+func (m *Statistic) MarshalJSON() ([]byte, error) {
+ items := []StatisticItem{
+ {"CPU_Usage_Max-" + m.name, m.cpu.maxVal, "%"},
+ {"CPU_Usage_Avg-" + m.name, m.cpu.avgVal, "%"},
+ {"Memory_Usage_Max-" + m.name, m.mem.maxVal, "MB"},
+ {"Memory_Usage_Avg-" + m.name, m.mem.avgVal, "MB"},
+ }
+
+ // Serialize the slice to JSON
+ jsonData, err := json.MarshalIndent(items, "", " ")
+ if err != nil {
+ fmt.Println("Error serializing statistics:", err)
+ return nil, err
+ }
+
+ // Output the JSON string
+ return jsonData, nil
+}