Merge branch 'main' into e2e-performance-test

.devcontainer/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:2.0.2
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail-build-linux:2.0.3
 
 ARG USERNAME=admin
 USER root
@@ -28,6 +28,18 @@ RUN wget http://mirrors.ustc.edu.cn/gnu/libc/glibc-2.18.tar.gz && \
  cd ../ && \
  rm -fr glibc-2.18*
 
+# install python3.8
+RUN cd /opt && curl -O https://cdn.npmmirror.com/binaries/python/3.8.12/Python-3.8.12.tgz && \
+ tar -zxvf Python-3.8.12.tgz && cd Python-3.8.12 && \
+ mkdir /usr/local/python3 && \
+ ./configure --prefix=/usr/local/python3 && \
+ make clean && make && make install && \
+ cp /usr/local/python3/bin/python3.8 /usr/bin/python3
+# install gcovr
+RUN python3 -m pip install --upgrade pip
+RUN cp /usr/local/python3/bin/pip3 /usr/bin/pip3 && pip3 install gcovr==7.0
+RUN cp /usr/local/python3/bin/gcovr /usr/bin/gcovr
+
 # Create the user
 COPY .env /tmp/.env
 RUN source /tmp/.env && rm /tmp/.env; \

.github/workflows/build-core-ut.yaml

@@ -17,13 +17,13 @@ name: Build Core Unit Test
 on:
  pull_request:
  paths-ignore:
- - 'docs/**'
- - 'example_config/**'
- - 'docker/**'
- - 'k8s_template/**'
- - 'changes/**'
- - 'licenses/**'
- - 'CHANGELOG.md'
+ - "docs/**"
+ - "example_config/**"
+ - "docker/**"
+ - "k8s_template/**"
+ - "changes/**"
+ - "licenses/**"
+ - "CHANGELOG.md"
  push:
  branches:
  - main
@@ -51,7 +51,7 @@ jobs:
  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
  sudo -E apt-get -qq autoremove --purge
  sudo -E apt-get -qq clean
- 
+
  - name: Check disk space
  run: |
  df -hT $PWD
@@ -65,21 +65,33 @@ jobs:
  uses: actions/checkout@v4
  with:
  submodules: true
+ fetch-depth: 2
 
  - name: Build Unit Test
  env:
  BUILD_LOGTAIL: OFF
  BUILD_LOGTAIL_UT: ON
- ENABLE_COMPATIBLE_MODE: ON
+ ENABLE_COMPATIBLE_MODE: OFF
  ENABLE_STATIC_LINK_CRT: ON
  WITHOUTGDB: ON
- MAKE_JOBS: 8
+ MAKE_JOBS: 16
  # BUILD_TYPE: Debug # TODO: Uncomment when memory management is refined
- run: make core
+ run: CURRENT_DIR=$(pwd) && sed -i "s|/src|$CURRENT_DIR|g" docker/Dockerfile_build && make core PATH_IN_DOCKER=$(pwd)
 
  - name: Unit Test
  run: make unittest_core
 
+ - name: Unit Test Coverage
+ run: docker build -t unittest_coverage -f ./docker/Dockerfile_coverage . && docker run -v $(pwd):$(pwd) unittest_coverage bash -c "cd $(pwd)/core && gcovr --root . --lcov coverage.lcov --txt coverage.txt -e \".*sdk.*\" -e \".*observer.*\" -e \".*log_pb.*\" -e \".*unittest.*\" -e \".*config_server.*\" -e \".*fuse.*\" -e \".*go_pipeline.*\""
+
+ - name: Setup Python3.10
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.10"
+
+ - name: Report code coverage
+ run: python3 tools/coverage-diff/main.py core/coverage.txt
+
  result:
  runs-on: arc-runner-set-ilogtail
  timeout-minutes: 90

.github/workflows/build-core.yaml

@@ -17,13 +17,13 @@ name: Build Core
 on:
  pull_request:
  paths-ignore:
- - 'docs/**'
- - 'example_config/**'
- - 'docker/**'
- - 'k8s_template/**'
- - 'changes/**'
- - 'licenses/**'
- - 'CHANGELOG.md'
+ - "docs/**"
+ - "example_config/**"
+ - "docker/**"
+ - "k8s_template/**"
+ - "changes/**"
+ - "licenses/**"
+ - "CHANGELOG.md"
  push:
  branches:
  - main
@@ -37,7 +37,7 @@ jobs:
  matrix:
  go-version: [1.19]
  # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
- runner: [ arc-runner-set-ilogtail ]
+ runner: [arc-runner-set-ilogtail]
  fail-fast: true
  steps:
  - name: prepare ubuntu environment
@@ -60,7 +60,7 @@ jobs:
  uses: actions/setup-go@v4
  with:
  go-version: ${{ matrix.go-version }}
- 
+
  - name: Check out code
  uses: actions/checkout@v4
  with:

.github/workflows/build.yaml

@@ -17,13 +17,13 @@ name: Build
 on:
  pull_request:
  paths-ignore:
- - 'docs/**'
- - 'example_config/**'
- - 'docker/**'
- - 'k8s_template/**'
- - 'changes/**'
- - 'licenses/**'
- - 'CHANGELOG.md'
+ - "docs/**"
+ - "example_config/**"
+ - "docker/**"
+ - "k8s_template/**"
+ - "changes/**"
+ - "licenses/**"
+ - "CHANGELOG.md"
  push:
  branches:
  - main
@@ -37,7 +37,7 @@ jobs:
  matrix:
  go-version: [1.19]
  # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idruns-on
- runner: [ arc-runner-set-ilogtail ]
+ runner: [arc-runner-set-ilogtail]
  fail-fast: true
  steps:
  - name: prepare ubuntu environment
@@ -51,7 +51,7 @@ jobs:
  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
  sudo -E apt-get -qq autoremove --purge
  sudo -E apt-get -qq clean
- 
+
  - name: Check disk space
  run: |
  df -hT $PWD
@@ -60,7 +60,7 @@ jobs:
  uses: actions/setup-go@v4
  with:
  go-version: ${{ matrix.go-version }}
- 
+
  - name: Check out code
  uses: actions/checkout@v4
  with:

.github/workflows/e2e-framework.yaml

@@ -17,13 +17,13 @@ name: E2E Core Test
 on:
  pull_request:
  paths-ignore:
- - 'docs/**'
- - 'example_config/**'
- - 'docker/**'
- - 'k8s_template/**'
- - 'changes/**'
- - 'licenses/**'
- - 'CHANGELOG.md'
+ - "docs/**"
+ - "example_config/**"
+ - "docker/**"
+ - "k8s_template/**"
+ - "changes/**"
+ - "licenses/**"
+ - "CHANGELOG.md"
  push:
  branches:
  - main
@@ -34,8 +34,8 @@ jobs:
  timeout-minutes: 60
  strategy:
  matrix:
- go-version: [ 1.19.10 ]
- runner: [ ubuntu-latest ]
+ go-version: [1.19.10]
+ runner: [ubuntu-latest]
  fail-fast: true
  steps:
  # Clean up space to prevent action from running out of disk space.
@@ -81,7 +81,7 @@ jobs:
  result:
  runs-on: ubuntu-latest
  timeout-minutes: 60
- needs: [ CI ]
+ needs: [CI]
  steps:
  - name: Build Result
  run: echo "Just to make the GitHub merge button green"

.github/workflows/e2e.yaml

@@ -17,13 +17,13 @@ name: E2E Test
 on:
  pull_request:
  paths-ignore:
- - 'docs/**'
- - 'example_config/**'
- - 'docker/**'
- - 'k8s_template/**'
- - 'changes/**'
- - 'licenses/**'
- - 'CHANGELOG.md'
+ - "docs/**"
+ - "example_config/**"
+ - "docker/**"
+ - "k8s_template/**"
+ - "changes/**"
+ - "licenses/**"
+ - "CHANGELOG.md"
  push:
  branches:
  - main
@@ -34,8 +34,8 @@ jobs:
  timeout-minutes: 60
  strategy:
  matrix:
- go-version: [ 1.19.10 ]
- runner: [ ubuntu-latest ]
+ go-version: [1.19.10]
+ runner: [ubuntu-latest]
  fail-fast: true
  steps:
  # Clean up space to prevent action from running out of disk space.
@@ -83,7 +83,7 @@ jobs:
  result:
  runs-on: ubuntu-latest
  timeout-minutes: 60
- needs: [ CI ]
+ needs: [CI]
  steps:
  - name: Build Result
  run: echo "Just to make the GitHub merge button green"

Makefile

@@ -18,8 +18,9 @@ DOCKER_PUSH ?= false
 DOCKER_REPOSITORY ?= aliyun/ilogtail
 BUILD_REPOSITORY ?= aliyun/ilogtail_build
 GENERATED_HOME ?= generated_files
-PLUGINS_CONFIG_FILE ?= "plugins.yml,external_plugins.yml"
+PLUGINS_CONFIG_FILE ?= plugins.yml,external_plugins.yml
 GO_MOD_FILE ?= go.mod
+PATH_IN_DOCER ?= /src
 DOCKER_BUILD_EXPORT_GO_ENVS ?= true
 DOCKER_BUILD_COPY_GIT_CONFIGS ?= true
 DOCKER_BUILD_USE_BUILDKIT ?=
@@ -78,14 +79,14 @@ tools:
 
 .PHONY: clean
 clean:
- rm -rf $(LICENSE_COVERAGE_FILE)
- rm -rf $(OUT_DIR) $(DIST_DIR)
+ rm -rf "$(LICENSE_COVERAGE_FILE)"
+ rm -rf "$(OUT_DIR)" "$(DIST_DIR)"
  rm -rf behavior-test
  rm -rf performance-test
  rm -rf core-test
  rm -rf e2e-engine-coverage.txt
  rm -rf find_licenses
- rm -rf $(GENERATED_HOME)
+ rm -rf "$(GENERATED_HOME)"
  rm -rf .testCoverage.txt
  rm -rf .coretestCoverage.txt
  rm -rf core/build
@@ -95,7 +96,7 @@ clean:
  rm -rf plugins/all/all_debug.go
  rm -rf plugins/all/all_windows.go
  rm -rf plugins/all/all_linux.go
- go mod tidy -modfile $(GO_MOD_FILE) || true
+ go mod tidy -modfile "$(GO_MOD_FILE)" || true
 
 .PHONY: license
 license: clean tools import_plugins
@@ -119,14 +120,14 @@ lint-e2e: clean tools
 
 .PHONY: core
 core: clean import_plugins
- ./scripts/gen_build_scripts.sh core $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
- ./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
+ ./scripts/gen_build_scripts.sh core "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" "$(OUT_DIR)" "$(DOCKER_BUILD_EXPORT_GO_ENVS)" "$(DOCKER_BUILD_COPY_GIT_CONFIGS)" "$(PLUGINS_CONFIG_FILE)" "$(GO_MOD_FILE)" "$(PATH_IN_DOCKER)"
+ ./scripts/docker_build.sh build "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" false "$(DOCKER_BUILD_USE_BUILDKIT)"
  ./$(GENERATED_HOME)/gen_copy_docker.sh
 
 .PHONY: plugin
 plugin: clean import_plugins
- ./scripts/gen_build_scripts.sh plugin $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
- ./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
+ ./scripts/gen_build_scripts.sh plugin "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" "$(OUT_DIR)" "$(DOCKER_BUILD_EXPORT_GO_ENVS)" "$(DOCKER_BUILD_COPY_GIT_CONFIGS)" "$(PLUGINS_CONFIG_FILE)" "$(GO_MOD_FILE)"
+ ./scripts/docker_build.sh build "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" false "$(DOCKER_BUILD_USE_BUILDKIT)"
  ./$(GENERATED_HOME)/gen_copy_docker.sh
 
 .PHONY: upgrade_adapter_lib
@@ -149,13 +150,13 @@ import_plugins:
 
 .PHONY: e2edocker
 e2edocker: clean import_plugins
- ./scripts/gen_build_scripts.sh e2e $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
- ./scripts/docker_build.sh development $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
+ ./scripts/gen_build_scripts.sh e2e "$(GENERATED_HOME)" "$(VERSION)" "$(DOCKER_REPOSITORY)" "$(OUT_DIR)" "$(DOCKER_BUILD_EXPORT_GO_ENVS)" "$(DOCKER_BUILD_COPY_GIT_CONFIGS)" "$(PLUGINS_CONFIG_FILE)" "$(GO_MOD_FILE)"
+ ./scripts/docker_build.sh development "$(GENERATED_HOME)" "$(VERSION)" "$(DOCKER_REPOSITORY)" false "$(DOCKER_BUILD_USE_BUILDKIT)"
 
 # provide a goc server for e2e testing
 .PHONY: gocdocker
 gocdocker: clean
- ./scripts/docker_build.sh goc $(GENERATED_HOME) latest goc-server false $(DOCKER_BUILD_USE_BUILDKIT)
+ ./scripts/docker_build.sh goc "$(GENERATED_HOME)" latest goc-server false "$(DOCKER_BUILD_USE_BUILDKIT)"
 
 .PHONY: vendor
 vendor: clean import_plugins
@@ -218,23 +219,23 @@ benchmark: clean gocdocker e2edocker
 
 .PHONY: all
 all: clean import_plugins
- ./scripts/gen_build_scripts.sh all $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) $(OUT_DIR) $(DOCKER_BUILD_EXPORT_GO_ENVS) $(DOCKER_BUILD_COPY_GIT_CONFIGS) $(PLUGINS_CONFIG_FILE) $(GO_MOD_FILE)
- ./scripts/docker_build.sh build $(GENERATED_HOME) $(VERSION) $(BUILD_REPOSITORY) false $(DOCKER_BUILD_USE_BUILDKIT)
+ ./scripts/gen_build_scripts.sh all "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" "$(OUT_DIR)" "$(DOCKER_BUILD_EXPORT_GO_ENVS)" "$(DOCKER_BUILD_COPY_GIT_CONFIGS)" "$(PLUGINS_CONFIG_FILE)" "$(GO_MOD_FILE)"
+ ./scripts/docker_build.sh build "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" false "$(DOCKER_BUILD_USE_BUILDKIT)"
  ./$(GENERATED_HOME)/gen_copy_docker.sh
 
 .PHONY: dist
 dist: all
- ./scripts/dist.sh $(OUT_DIR) $(DIST_DIR) $(PACKAGE_DIR)
+ ./scripts/dist.sh "$(OUT_DIR)" "$(DIST_DIR)" "$(PACKAGE_DIR)"
 
 $(DIST_FILE):
  @echo 'ilogtail-$(VERSION) dist does not exist! Please download or run `make dist` first!'
  @false
 
 .PHONY: docker
 docker: $(DIST_FILE)
- ./scripts/docker_build.sh production $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH) $(DOCKER_BUILD_USE_BUILDKIT)
+ ./scripts/docker_build.sh production "$(GENERATED_HOME)" "$(VERSION)" "$(DOCKER_REPOSITORY)" "$(DOCKER_PUSH)" "$(DOCKER_BUILD_USE_BUILDKIT)"
 
 .PHONY: multi-arch-docker
 multi-arch-docker: $(DIST_FILE)
  @echo "will push to $(DOCKER_REPOSITORY):edge. Make sure this tag does not exist or push will fail."
- ./scripts/docker_build.sh multi-arch-production $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH) $(DOCKER_BUILD_USE_BUILDKIT)
+ ./scripts/docker_build.sh multi-arch-production "$(GENERATED_HOME)" "$(VERSION)" "$(DOCKER_REPOSITORY)" "$(DOCKER_PUSH)" "$(DOCKER_BUILD_USE_BUILDKIT)"