From d029974a5ac2290ad306a051a2ae4fd87fc22b21 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 18 Oct 2023 22:00:11 +0200
Subject: [PATCH 01/31] buildkite: run opentelemetry benchmark

---
 .buildkite/README.md                          |  9 +++
 .buildkite/hooks/pre-command                  | 61 ++++---------------
 .buildkite/hooks/prepare-benchmark.sh         |  7 +++
 .buildkite/hooks/prepare-release.sh           | 52 ++++++++++++++++
 .buildkite/opentelemetry-benchmark.yml        |  5 ++
 .buildkite/scripts/opentelemetry-benchmark.sh | 27 ++++++++
 catalog-info.yaml                             | 37 +++++++++++
 7 files changed, 149 insertions(+), 49 deletions(-)
 create mode 100644 .buildkite/hooks/prepare-benchmark.sh
 create mode 100644 .buildkite/hooks/prepare-release.sh
 create mode 100644 .buildkite/opentelemetry-benchmark.yml
 create mode 100755 .buildkite/scripts/opentelemetry-benchmark.sh

diff --git a/.buildkite/README.md b/.buildkite/README.md
index cac080f834..1fcff227d8 100644
--- a/.buildkite/README.md
+++ b/.buildkite/README.md
@@ -19,3 +19,12 @@ This is the Buildkite pipeline for the APM Agent java in charge of the snaposhot
 
 To view the pipeline and its configuration, click [here](https://buildkite.com/elastic/apm-agent-java-snapshot) or
 go to the definition in the `elastic/ci` repository.
+
+## opentelemetry-benchmark pipeline
+
+This is the Buildkite pipeline for the Opentelemetry Benchmark.
+
+### Pipeline Configuration
+
+To view the pipeline and its configuration, click [here](https://buildkite.com/elastic/apm-agent-java-opentelemetry-benchmark) or
+go to the definition in `opentelemetry-benchmark.yml`.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index e63e82b085..346200dd44 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -8,55 +8,18 @@
 
 set -eo pipefail
 
-echo "--- Prepare vault context :vault:"
-VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-apm-agent-java/internal-ci-approle)
-export VAULT_ROLE_ID_SECRET
-
-VAULT_SECRET_ID_SECRET=$(vault read -field=secret-id secret/ci/elastic-apm-agent-java/internal-ci-approle)
-export VAULT_SECRET_ID_SECRET
-
-VAULT_ADDR=$(vault read -field=vault-url secret/ci/elastic-apm-agent-java/internal-ci-approle)
-export VAULT_ADDR
-
-# Delete the vault specific accessing the ci vault
-PREVIOUS_VAULT_TOKEN=$VAULT_TOKEN
-export PREVIOUS_VAULT_TOKEN
-unset VAULT_TOKEN
-
-echo "--- Prepare a secure temp :closed_lock_with_key:"
-# Prepare a secure temp folder not shared between other jobs to store the key ring
-export TMP_WORKSPACE=/tmp/secured
-export KEY_FILE=$TMP_WORKSPACE"/private.key"
-
-# Secure home for our keyring
-export GNUPGHOME=$TMP_WORKSPACE"/keyring"
-mkdir -p $GNUPGHOME
-chmod -R 700 $TMP_WORKSPACE
-
-echo "--- Prepare keys context :key:"
-VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID_SECRET" secret_id="$VAULT_SECRET_ID_SECRET")
-export VAULT_TOKEN
-
-# Nexus credentials
-SERVER_USERNAME=$(vault read -field username secret/release/nexus)
-export SERVER_USERNAME
-SERVER_PASSWORD=$(vault read -field password secret/release/nexus)
-export SERVER_PASSWORD
-
-# Signing keys
-vault read -field=key secret/release/signing >$KEY_FILE
-KEYPASS_SECRET=$(vault read -field=passphrase secret/release/signing)
-export KEYPASS_SECRET
-export KEY_ID_SECRET=D88E42B4
-
-# Import the key into the keyring
-echo "$KEYPASS_SECRET" | gpg --batch --import "$KEY_FILE"
-
-echo "--- Configure git context :git:"
-# Configure the committer since the maven release requires to push changes to GitHub
-# This will help with the SLSA requirements.
-git config --global user.email "infra-root+apmmachine@elastic.co"
-git config --global user.name "apmmachine"
+## TODO: change name for the opentelemetry-benchmark
+if [ "$BUILDKITE_PIPELINE_SLUG" == "apm-agent-java-load-testing" ]; then
+    source .buildkite/hooks/prepare-benchmark.sh
+fi
+
+if [ "$BUILDKITE_PIPELINE_SLUG" == "apm-agent-java-snapshot" ]; then
+    source .buildkite/hooks/prepare-release.sh
+fi
+
+if [ "$BUILDKITE_PIPELINE_SLUG" == "apm-agent-java-release" ]; then
+    source .buildkite/hooks/prepare-release.sh
+fi
 
 echo "--- Install JDK17 :java:"
 # JDK version is defined in two different locations, here and .github/workflows/maven-goal/action.yml
diff --git a/.buildkite/hooks/prepare-benchmark.sh b/.buildkite/hooks/prepare-benchmark.sh
new file mode 100644
index 0000000000..4f3b184df8
--- /dev/null
+++ b/.buildkite/hooks/prepare-benchmark.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+GITHUB_SECRET=$(retry 5 vault kv get -field token "kv/ci-shared/observability-ci/github-apmmachine")
+export GITHUB_SECRET
+GITHUB_USERNAME=apmmachine
+export GITHUB_USERNAME
diff --git a/.buildkite/hooks/prepare-release.sh b/.buildkite/hooks/prepare-release.sh
new file mode 100644
index 0000000000..83520fd544
--- /dev/null
+++ b/.buildkite/hooks/prepare-release.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "--- Prepare vault context :vault:"
+VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-apm-agent-java/internal-ci-approle)
+export VAULT_ROLE_ID_SECRET
+
+VAULT_SECRET_ID_SECRET=$(vault read -field=secret-id secret/ci/elastic-apm-agent-java/internal-ci-approle)
+export VAULT_SECRET_ID_SECRET
+
+VAULT_ADDR=$(vault read -field=vault-url secret/ci/elastic-apm-agent-java/internal-ci-approle)
+export VAULT_ADDR
+
+# Delete the vault specific accessing the ci vault
+PREVIOUS_VAULT_TOKEN=$VAULT_TOKEN
+export PREVIOUS_VAULT_TOKEN
+unset VAULT_TOKEN
+
+echo "--- Prepare a secure temp :closed_lock_with_key:"
+# Prepare a secure temp folder not shared between other jobs to store the key ring
+export TMP_WORKSPACE=/tmp/secured
+export KEY_FILE=$TMP_WORKSPACE"/private.key"
+
+# Secure home for our keyring
+export GNUPGHOME=$TMP_WORKSPACE"/keyring"
+mkdir -p $GNUPGHOME
+chmod -R 700 $TMP_WORKSPACE
+
+echo "--- Prepare keys context :key:"
+VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID_SECRET" secret_id="$VAULT_SECRET_ID_SECRET")
+export VAULT_TOKEN
+
+# Nexus credentials
+SERVER_USERNAME=$(vault read -field username secret/release/nexus)
+export SERVER_USERNAME
+SERVER_PASSWORD=$(vault read -field password secret/release/nexus)
+export SERVER_PASSWORD
+
+# Signing keys
+vault read -field=key secret/release/signing >$KEY_FILE
+KEYPASS_SECRET=$(vault read -field=passphrase secret/release/signing)
+export KEYPASS_SECRET
+export KEY_ID_SECRET=D88E42B4
+
+# Import the key into the keyring
+echo "$KEYPASS_SECRET" | gpg --batch --import "$KEY_FILE"
+
+echo "--- Configure git context :git:"
+# Configure the committer since the maven release requires to push changes to GitHub
+# This will help with the SLSA requirements.
+git config --global user.email "infra-root+apmmachine@elastic.co"
+git config --global user.name "apmmachine"
diff --git a/.buildkite/opentelemetry-benchmark.yml b/.buildkite/opentelemetry-benchmark.yml
new file mode 100644
index 0000000000..85766bfae7
--- /dev/null
+++ b/.buildkite/opentelemetry-benchmark.yml
@@ -0,0 +1,5 @@
+steps:
+  - label: "Run the opentelemetry-benchmark"
+    commands: .buildkite/scripts/opentelemetry-benchmark.sh
+    agents:
+      queue: observability-microbenchmarks
\ No newline at end of file
diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
new file mode 100755
index 0000000000..1ac8c5a24e
--- /dev/null
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+echo "--- Build opentelemetry-java-instrumentation"
+git clone https://github.com/open-telemetry/opentelemetry-java-instrumentation.git
+cd opentelemetry-java-instrumentation/
+./gradlew assemble
+
+echo "--- Download the latest elastic-agent.zip"
+cd benchmark-overhead
+ELASTIC_SNAPSHOT_URL=$(curl -s -u $GITHUB_USERNAME:$GITHUB_SECRET "https://api.github.com/repos/elastic/apm-agent-java/actions/workflows/49838992/runs?branch=main" | jq -c '.workflow_runs[] | {conclusion, updated_at, display_title, url}' | grep -v null  | grep -v pending | grep -v cancelled | grep success | head -1 | awk -F'":"' '{print $5}' | tr -d '"}')
+ELASTIC_SNAPSHOT_ARTIFACTS=$(curl -s -u $GITHUB_USERNAME:$GITHUB_SECRET "$ELASTIC_SNAPSHOT_URL" | grep artifacts_url | awk -F'":' '{print $2}' | tr -d '"} ,')
+ELASTIC_SNAPSHOT_ZIPFILE=$(curl -s -u $GITHUB_USERNAME:$GITHUB_SECRET "$ELASTIC_SNAPSHOT_ARTIFACTS" | jq -c ".artifacts[] | {name,archive_download_url}" | grep '"elastic-apm-agent"' | awk -F'":' '{print $3}' | tr -d '"}')
+curl -s --output "elastic-agent.zip" -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_SECRET" -H "X-GitHub-Api-Version: 2022-11-28" -u $GITHUB_USERNAME:$GITHUB_SECRET "$ELASTIC_SNAPSHOT_ZIPFILE"
+unzip elastic-agent.zip
+ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
+ELASTIC_SNAPSHOT_ENTRY="new Agent(\\\"elastic-snapshot\\\",\\\"latest available snapshot version from elastic main\\\",\\\"file://$PWD/$ELASTIC_SNAPSHOT_JAR\\\")"
+ELASTIC_LATEST_VERSION=$(curl -s https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/ | perl -ne 's/<.*?>//g; if(s/^([\d\.]+).*$/$1/){print}' | sort -V | tail -1)
+ELASTIC_LATEST_ENTRY="new Agent(\\\"elastic-latest\\\",\\\"latest available released version from elastic main\\\",\\\"https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_LATEST_VERSION/elastic-apm-agent-$ELASTIC_LATEST_VERSION.jar\\\")"
+ELASTIC_LATEST_ENTRY2="new Agent(\\\"elastic-async\\\",\\\"latest available released version from elastic main\\\",\\\"https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_LATEST_VERSION/elastic-apm-agent-$ELASTIC_LATEST_VERSION.jar\\\", java.util.List.of(\\\"-Delastic.apm.delay_agent_premain_ms=15000\\\"))"
+NEW_LINE="              .withAgents(Agent.NONE, Agent.LATEST_RELEASE, Agent.LATEST_SNAPSHOT, $ELASTIC_LATEST_ENTRY, $ELASTIC_LATEST_ENTRY2, $ELASTIC_SNAPSHOT_ENTRY)"
+echo $NEW_LINE
+perl -i -ne "if (/withAgents/) {print \"$NEW_LINE\n\"}else{print}" src/test/java/io/opentelemetry/config/Configs.java
+
+echo "--- Run tests with sudo access"
+sudo ./gradlew test
+perl -ne '/Standard output/ && $on++; /\<\/pre\>/ && ($on=0);$on && s/\<.*\>//;$on && !/^\s*$/ && print' build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html
diff --git a/catalog-info.yaml b/catalog-info.yaml
index 17564e5920..4fb2838cb0 100644
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@@ -91,3 +91,40 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-apm-agent-java-opentelemetry-benchmark
+  description: Buildkite Opentelemetry Benchmark for apm-agent-java
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/apm-agent-java-opentelemetry-benchmark
+spec:
+  type: buildkite-pipeline
+  owner: group:apm-agent-java
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: apm-agent-java-opentelemetry-benchmark
+    spec:
+      repository: elastic/apm-agent-java
+      pipeline_file: ".buildkite/opentelemetry-benchmark.yml"
+      default_branch: main
+      provider_settings:
+        publish_commit_status: false
+      teams:
+        apm-agent-java:
+          access_level: MANAGE_BUILD_AND_READ
+        observablt-robots:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      schedules:
+        Weekly Benchmark on main Branch:
+          cronline: "@weekly"
+          message: "Run the quick benchmark weekly."

From fc095ed3eb86fbd75be219fcfdfafef9494b811a Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 18 Oct 2023 22:00:23 +0200
Subject: [PATCH 02/31] chore: for testing purposes

---
 .buildkite/load-testing.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.buildkite/load-testing.yml b/.buildkite/load-testing.yml
index dc9f232059..22b803d409 100644
--- a/.buildkite/load-testing.yml
+++ b/.buildkite/load-testing.yml
@@ -1,4 +1,5 @@
-# @reakaleek: This is a place holder to create the pipeline in Buildkite. I will work on it, in a follow-up.
 steps:
-  - label: ":wave: Greetings"            # Label (with rich emojis https://ela.st/bk-emoji).
-    command: "echo 'My first pipeline!'" # Command to run (evaluated by Bash).
+  - label: "Run the microbenchmark"
+    commands: .buildkite/scripts/opentelemetry-benchmark.sh
+    agents:
+      queue: observability-microbenchmarks
\ No newline at end of file

From 218b8630e2d4152cc8945fdb3d8e7b09a1d7d98f Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 18 Oct 2023 22:01:23 +0200
Subject: [PATCH 03/31] no retry

---
 .buildkite/hooks/prepare-benchmark.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/prepare-benchmark.sh b/.buildkite/hooks/prepare-benchmark.sh
index 4f3b184df8..1768475b81 100644
--- a/.buildkite/hooks/prepare-benchmark.sh
+++ b/.buildkite/hooks/prepare-benchmark.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-GITHUB_SECRET=$(retry 5 vault kv get -field token "kv/ci-shared/observability-ci/github-apmmachine")
+GITHUB_SECRET=$(vault kv get -field token "kv/ci-shared/observability-ci/github-apmmachine")
 export GITHUB_SECRET
 GITHUB_USERNAME=apmmachine
 export GITHUB_USERNAME

From e2828df9517e827a7a9b8c0369bba5f2c773d99c Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Wed, 18 Oct 2023 22:42:39 +0200
Subject: [PATCH 04/31] Update .buildkite/scripts/opentelemetry-benchmark.sh

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 1ac8c5a24e..f918e95690 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -23,5 +23,5 @@ echo $NEW_LINE
 perl -i -ne "if (/withAgents/) {print \"$NEW_LINE\n\"}else{print}" src/test/java/io/opentelemetry/config/Configs.java
 
 echo "--- Run tests with sudo access"
-sudo ./gradlew test
+./gradlew test
 perl -ne '/Standard output/ && $on++; /\<\/pre\>/ && ($on=0);$on && s/\<.*\>//;$on && !/^\s*$/ && print' build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html

From 38b42a4e145f4b27d10307b2a97e2bd4a2c75d67 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 18 Oct 2023 23:02:52 +0200
Subject: [PATCH 05/31] improve: use gh and clone main with shallow cloning

---
 .buildkite/hooks/prepare-benchmark.sh         | 18 +++++++++++++++++-
 .buildkite/scripts/opentelemetry-benchmark.sh |  9 +++------
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/.buildkite/hooks/prepare-benchmark.sh b/.buildkite/hooks/prepare-benchmark.sh
index 1768475b81..eca3276fb8 100644
--- a/.buildkite/hooks/prepare-benchmark.sh
+++ b/.buildkite/hooks/prepare-benchmark.sh
@@ -1,7 +1,23 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+echo "--- Prepare github secrets :vault:"
 GITHUB_SECRET=$(vault kv get -field token "kv/ci-shared/observability-ci/github-apmmachine")
-export GITHUB_SECRET
+GH_TOKEN=$GITHUB_SECRET
+export GITHUB_SECRET GH_TOKEN
 GITHUB_USERNAME=apmmachine
 export GITHUB_USERNAME
+
+echo "--- Install gh :github:"
+GH_URL=https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.tar.gz
+GH_HOME=$(pwd)/.gh
+curl -L --output /tmp/gh.tar.gz "$GH_URL"
+mkdir -p "$GH_HOME"
+tar --extract --file /tmp/gh.tar.gz --directory "$GH_HOME" --strip-components 1
+
+PATH=$GH_HOME/bin:$PATH
+export PATH
+
+echo "--- Configure gh :github:"
+echo "$GH_TOKEN" | gh auth login --with-token
+gh auth setup-git
diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index f918e95690..6c19c727aa 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -2,17 +2,14 @@
 set -eo pipefail
 
 echo "--- Build opentelemetry-java-instrumentation"
-git clone https://github.com/open-telemetry/opentelemetry-java-instrumentation.git
+git clone https://github.com/open-telemetry/opentelemetry-java-instrumentation.git --depth 1 --branch main
 cd opentelemetry-java-instrumentation/
 ./gradlew assemble
 
 echo "--- Download the latest elastic-agent.zip"
 cd benchmark-overhead
-ELASTIC_SNAPSHOT_URL=$(curl -s -u $GITHUB_USERNAME:$GITHUB_SECRET "https://api.github.com/repos/elastic/apm-agent-java/actions/workflows/49838992/runs?branch=main" | jq -c '.workflow_runs[] | {conclusion, updated_at, display_title, url}' | grep -v null  | grep -v pending | grep -v cancelled | grep success | head -1 | awk -F'":"' '{print $5}' | tr -d '"}')
-ELASTIC_SNAPSHOT_ARTIFACTS=$(curl -s -u $GITHUB_USERNAME:$GITHUB_SECRET "$ELASTIC_SNAPSHOT_URL" | grep artifacts_url | awk -F'":' '{print $2}' | tr -d '"} ,')
-ELASTIC_SNAPSHOT_ZIPFILE=$(curl -s -u $GITHUB_USERNAME:$GITHUB_SECRET "$ELASTIC_SNAPSHOT_ARTIFACTS" | jq -c ".artifacts[] | {name,archive_download_url}" | grep '"elastic-apm-agent"' | awk -F'":' '{print $3}' | tr -d '"}')
-curl -s --output "elastic-agent.zip" -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_SECRET" -H "X-GitHub-Api-Version: 2022-11-28" -u $GITHUB_USERNAME:$GITHUB_SECRET "$ELASTIC_SNAPSHOT_ZIPFILE"
-unzip elastic-agent.zip
+run_id=$(gh run list --branch main --status success --workflow main.yml -L 1 --json databaseId --jq '.[].databaseId')
+gh run download "$run_id" -n elastic-apm-agent
 ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
 ELASTIC_SNAPSHOT_ENTRY="new Agent(\\\"elastic-snapshot\\\",\\\"latest available snapshot version from elastic main\\\",\\\"file://$PWD/$ELASTIC_SNAPSHOT_JAR\\\")"
 ELASTIC_LATEST_VERSION=$(curl -s https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/ | perl -ne 's/<.*?>//g; if(s/^([\d\.]+).*$/$1/){print}' | sort -V | tail -1)

From acc74fe21e5ec6cfac33de45a6794c2720e3cbcb Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 18 Oct 2023 23:05:05 +0200
Subject: [PATCH 06/31] fixes: To have GitHub CLI store credentials instead,
 first clear the value from the environment.

---
 .buildkite/hooks/prepare-benchmark.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/hooks/prepare-benchmark.sh b/.buildkite/hooks/prepare-benchmark.sh
index eca3276fb8..3d0f0339ab 100644
--- a/.buildkite/hooks/prepare-benchmark.sh
+++ b/.buildkite/hooks/prepare-benchmark.sh
@@ -20,4 +20,3 @@ export PATH
 
 echo "--- Configure gh :github:"
 echo "$GH_TOKEN" | gh auth login --with-token
-gh auth setup-git

From b8f8b39dc3de717e45af9fe4ad6a56596ace1b2f Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 18 Oct 2023 23:07:50 +0200
Subject: [PATCH 07/31] avoid load pipeline using pre-command

---
 .buildkite/hooks/pre-command | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 346200dd44..74a5a879a6 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -8,6 +8,12 @@
 
 set -eo pipefail
 
+# Upload should not do much with the pre-command.
+if [[ "$BUILDKITE_COMMAND" =~ .*"upload".* ]]; then
+  echo "Skipped pre-command when running the Upload pipeline"
+  exit 0
+fi
+
 ## TODO: change name for the opentelemetry-benchmark
 if [ "$BUILDKITE_PIPELINE_SLUG" == "apm-agent-java-load-testing" ]; then
     source .buildkite/hooks/prepare-benchmark.sh

From 1efccaf84b95c639bc332155ca97ac73014f2c9e Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 18 Oct 2023 23:09:54 +0200
Subject: [PATCH 08/31] avoid login

---
 .buildkite/hooks/prepare-benchmark.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.buildkite/hooks/prepare-benchmark.sh b/.buildkite/hooks/prepare-benchmark.sh
index 3d0f0339ab..d3fe0b549b 100644
--- a/.buildkite/hooks/prepare-benchmark.sh
+++ b/.buildkite/hooks/prepare-benchmark.sh
@@ -17,6 +17,3 @@ tar --extract --file /tmp/gh.tar.gz --directory "$GH_HOME" --strip-components 1
 
 PATH=$GH_HOME/bin:$PATH
 export PATH
-
-echo "--- Configure gh :github:"
-echo "$GH_TOKEN" | gh auth login --with-token

From eff8b4e2c20bb0c0768a677263bb9a8373f63261 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 09:20:59 +0200
Subject: [PATCH 09/31] run gh within the apm-agent-java context

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 6c19c727aa..2e2131e7b6 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -1,16 +1,21 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
+echo "--- Download the latest elastic-agent.zip"
+# run earlier so gh can use the current github repository.
+run_id=$(gh run list --branch main --status success --workflow main.yml -L 1 --json databaseId --jq '.[].databaseId')
+gh run download "$run_id" -n elastic-apm-agent
+ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
+ELASTIC_SNAPSHOT_JAR_FILE="$(pwd)/$ELASTIC_SNAPSHOT_JAR"
+
 echo "--- Build opentelemetry-java-instrumentation"
 git clone https://github.com/open-telemetry/opentelemetry-java-instrumentation.git --depth 1 --branch main
 cd opentelemetry-java-instrumentation/
 ./gradlew assemble
 
-echo "--- Download the latest elastic-agent.zip"
+echo "--- Customise the elastic opentelemetry java instrumentation"
 cd benchmark-overhead
-run_id=$(gh run list --branch main --status success --workflow main.yml -L 1 --json databaseId --jq '.[].databaseId')
-gh run download "$run_id" -n elastic-apm-agent
-ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
+cp "$ELASTIC_SNAPSHOT_JAR_FILE" .
 ELASTIC_SNAPSHOT_ENTRY="new Agent(\\\"elastic-snapshot\\\",\\\"latest available snapshot version from elastic main\\\",\\\"file://$PWD/$ELASTIC_SNAPSHOT_JAR\\\")"
 ELASTIC_LATEST_VERSION=$(curl -s https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/ | perl -ne 's/<.*?>//g; if(s/^([\d\.]+).*$/$1/){print}' | sort -V | tail -1)
 ELASTIC_LATEST_ENTRY="new Agent(\\\"elastic-latest\\\",\\\"latest available released version from elastic main\\\",\\\"https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_LATEST_VERSION/elastic-apm-agent-$ELASTIC_LATEST_VERSION.jar\\\")"

From e76c73b88b79dd7ea96b603364fea4ddacf337ad Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 09:21:12 +0200
Subject: [PATCH 10/31] buildkite: notify in the channel

---
 .buildkite/opentelemetry-benchmark.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.buildkite/opentelemetry-benchmark.yml b/.buildkite/opentelemetry-benchmark.yml
index 85766bfae7..a0f2e64b43 100644
--- a/.buildkite/opentelemetry-benchmark.yml
+++ b/.buildkite/opentelemetry-benchmark.yml
@@ -2,4 +2,8 @@ steps:
   - label: "Run the opentelemetry-benchmark"
     commands: .buildkite/scripts/opentelemetry-benchmark.sh
     agents:
-      queue: observability-microbenchmarks
\ No newline at end of file
+      queue: observability-microbenchmarks
+
+notify:
+  - slack: "#apm-agent-java"
+    if: 'build.state != "passed"'
\ No newline at end of file

From 703824f0ec6c6cd173189f008606b97b6ea74518 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 09:24:20 +0200
Subject: [PATCH 11/31] common steps

---
 .buildkite/hooks/pre-command       | 16 ++--------------
 .buildkite/hooks/prepare-common.sh | 17 +++++++++++++++++
 2 files changed, 19 insertions(+), 14 deletions(-)
 create mode 100644 .buildkite/hooks/prepare-common.sh

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 74a5a879a6..efef2f25ec 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -27,17 +27,5 @@ if [ "$BUILDKITE_PIPELINE_SLUG" == "apm-agent-java-release" ]; then
     source .buildkite/hooks/prepare-release.sh
 fi
 
-echo "--- Install JDK17 :java:"
-# JDK version is defined in two different locations, here and .github/workflows/maven-goal/action.yml
-JAVA_URL=https://jvm-catalog.elastic.co/jdk
-JAVA_HOME=$(pwd)/.openjdk17
-JAVA_PKG="$JAVA_URL/latest_openjdk_17_linux.tar.gz"
-curl -L --output /tmp/jdk.tar.gz "$JAVA_PKG"
-mkdir -p "$JAVA_HOME"
-tar --extract --file /tmp/jdk.tar.gz --directory "$JAVA_HOME" --strip-components 1
-
-export JAVA_HOME
-PATH=$JAVA_HOME/bin:$PATH
-export PATH
-
-java -version || true
+# Run always
+source .buildkite/hooks/prepare-common.sh
diff --git a/.buildkite/hooks/prepare-common.sh b/.buildkite/hooks/prepare-common.sh
new file mode 100644
index 0000000000..68cf0d4ad4
--- /dev/null
+++ b/.buildkite/hooks/prepare-common.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "--- Install JDK17 :java:"
+# JDK version is defined in two different locations, here and .github/workflows/maven-goal/action.yml
+JAVA_URL=https://jvm-catalog.elastic.co/jdk
+JAVA_HOME=$(pwd)/.openjdk17
+JAVA_PKG="$JAVA_URL/latest_openjdk_17_linux.tar.gz"
+curl -L --output /tmp/jdk.tar.gz "$JAVA_PKG"
+mkdir -p "$JAVA_HOME"
+tar --extract --file /tmp/jdk.tar.gz --directory "$JAVA_HOME" --strip-components 1
+
+export JAVA_HOME
+PATH=$JAVA_HOME/bin:$PATH
+export PATH
+
+java -version || true

From fe3d9a555f2c9cedb982112d4f2924dc3ca9949a Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 09:33:14 +0200
Subject: [PATCH 12/31] chore: more log groups to help with printing output

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 2e2131e7b6..e3b6577e32 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -4,9 +4,11 @@ set -eo pipefail
 echo "--- Download the latest elastic-agent.zip"
 # run earlier so gh can use the current github repository.
 run_id=$(gh run list --branch main --status success --workflow main.yml -L 1 --json databaseId --jq '.[].databaseId')
+echo "downloading the latest artifact 'elastic-apm-agent' (using the workflow run '$run_id')"
 gh run download "$run_id" -n elastic-apm-agent
 ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
 ELASTIC_SNAPSHOT_JAR_FILE="$(pwd)/$ELASTIC_SNAPSHOT_JAR"
+echo "$ELASTIC_SNAPSHOT_JAR_FILE has been downloaded."
 
 echo "--- Build opentelemetry-java-instrumentation"
 git clone https://github.com/open-telemetry/opentelemetry-java-instrumentation.git --depth 1 --branch main
@@ -24,6 +26,8 @@ NEW_LINE="              .withAgents(Agent.NONE, Agent.LATEST_RELEASE, Agent.LATE
 echo $NEW_LINE
 perl -i -ne "if (/withAgents/) {print \"$NEW_LINE\n\"}else{print}" src/test/java/io/opentelemetry/config/Configs.java
 
-echo "--- Run tests with sudo access"
+echo "--- Run tests of benchmark-overhead"
 ./gradlew test
+
+echo "--- Report"
 perl -ne '/Standard output/ && $on++; /\<\/pre\>/ && ($on=0);$on && s/\<.*\>//;$on && !/^\s*$/ && print' build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html

From e811189d79dbedd8a25bae04bc6eb8138ccbf6cb Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 09:49:11 +0200
Subject: [PATCH 13/31] buildkite: use annotations to visualise report

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index e3b6577e32..f138688760 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -30,4 +30,14 @@ echo "--- Run tests of benchmark-overhead"
 ./gradlew test
 
 echo "--- Report"
-perl -ne '/Standard output/ && $on++; /\<\/pre\>/ && ($on=0);$on && s/\<.*\>//;$on && !/^\s*$/ && print' build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html
+perl -ne '/Standard output/ && $on++; /\<\/pre\>/ && ($on=0);$on && s/\<.*\>//;$on && !/^\s*$/ && print' build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html | tee report.txt
+
+# Buildkite annotation
+if [ -n "$BUILDKITE" ]; then
+  REPORT=$(cat report.txt)
+  cat << EOF | buildkite-agent annotate --style "info" --context report
+  ### OverheadTests Report
+
+  ${REPORT}
+EOF
+fi

From e3d2757cbc3f59a284acc7538068cfcae443e466 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 10:04:50 +0200
Subject: [PATCH 14/31] buildkite: archive report

---
 .buildkite/load-testing.yml            | 4 +++-
 .buildkite/opentelemetry-benchmark.yml | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.buildkite/load-testing.yml b/.buildkite/load-testing.yml
index 22b803d409..380613fa13 100644
--- a/.buildkite/load-testing.yml
+++ b/.buildkite/load-testing.yml
@@ -2,4 +2,6 @@ steps:
   - label: "Run the microbenchmark"
     commands: .buildkite/scripts/opentelemetry-benchmark.sh
     agents:
-      queue: observability-microbenchmarks
\ No newline at end of file
+      queue: observability-microbenchmarks
+    artifact_paths:
+      - "**/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html"
\ No newline at end of file
diff --git a/.buildkite/opentelemetry-benchmark.yml b/.buildkite/opentelemetry-benchmark.yml
index a0f2e64b43..cf24dc691e 100644
--- a/.buildkite/opentelemetry-benchmark.yml
+++ b/.buildkite/opentelemetry-benchmark.yml
@@ -3,7 +3,9 @@ steps:
     commands: .buildkite/scripts/opentelemetry-benchmark.sh
     agents:
       queue: observability-microbenchmarks
+    artifact_paths:
+      - "**/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html"
 
 notify:
   - slack: "#apm-agent-java"
-    if: 'build.state != "passed"'
\ No newline at end of file
+    if: 'build.state != "passed"'

From 9d5ccc72a10bfe4bd6516c3f71d0a60ec0a29c14 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 10:19:05 +0200
Subject: [PATCH 15/31] use markdown format and add entry for reporting output

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index f138688760..caa605a53a 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -38,6 +38,11 @@ if [ -n "$BUILDKITE" ]; then
   cat << EOF | buildkite-agent annotate --style "info" --context report
   ### OverheadTests Report
 
+  \`\`\`
   ${REPORT}
+  \`\`\`
+
 EOF
 fi
+
+echo "--- Send Report (TBC)"

From 60733ece323e2a90a60764ce1b15b79aaf66b6ec Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 19 Oct 2023 10:23:46 +0200
Subject: [PATCH 16/31] buildkite: prepare ES credentials

---
 .buildkite/hooks/prepare-benchmark.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.buildkite/hooks/prepare-benchmark.sh b/.buildkite/hooks/prepare-benchmark.sh
index d3fe0b549b..7dd44d5bca 100644
--- a/.buildkite/hooks/prepare-benchmark.sh
+++ b/.buildkite/hooks/prepare-benchmark.sh
@@ -1,6 +1,12 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+echo "--- Prepare elasticsearch secrets :vault:"
+ES_URL_SECRET=$(vault read -field=es_url secret/ci/elastic-apm-agent-java/opentelemetry-benchmark)
+ES_USER_SECRET=$(vault read -field=es_user secret/ci/elastic-apm-agent-java/opentelemetry-benchmark)
+ES_PASS_SECRET=$(vault read -field=es_pass secret/ci/elastic-apm-agent-java/opentelemetry-benchmark)
+export ES_URL_SECRET ES_USER_SECRET ES_PASS_SECRET
+
 echo "--- Prepare github secrets :vault:"
 GITHUB_SECRET=$(vault kv get -field token "kv/ci-shared/observability-ci/github-apmmachine")
 GH_TOKEN=$GITHUB_SECRET

From 88fda88fb18af63b83d1ffc625971180a4b09ffa Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 09:29:24 +0200
Subject: [PATCH 17/31] add apm-server mock and tear down

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index caa605a53a..9bf914f11e 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -1,6 +1,17 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
+CONTAINER_NAME=mock-apm-server
+
+function cleanup {
+  echo "--- Tear down the environment"
+  MOCK_APM_SERVER=$(docker ps | grep $CONTAINER_NAME | awk '{print $1}')
+  docker stop $MOCK_APM_SERVER
+  docker rm $MOCK_APM_SERVER
+}
+
+trap cleanup EXIT
+
 echo "--- Download the latest elastic-agent.zip"
 # run earlier so gh can use the current github repository.
 run_id=$(gh run list --branch main --status success --workflow main.yml -L 1 --json databaseId --jq '.[].databaseId')
@@ -10,6 +21,13 @@ ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
 ELASTIC_SNAPSHOT_JAR_FILE="$(pwd)/$ELASTIC_SNAPSHOT_JAR"
 echo "$ELASTIC_SNAPSHOT_JAR_FILE has been downloaded."
 
+echo "--- Download the latest elastic-agent.zip"
+git clone https://github.com/elastic/apm-mutating-webhook.git
+pushd apm-mutating-webhook/test/mock
+docker build -t $CONTAINER_NAME .
+docker run -dp 127.0.0.1:8027:8027 $CONTAINER_NAME
+popd
+
 echo "--- Build opentelemetry-java-instrumentation"
 git clone https://github.com/open-telemetry/opentelemetry-java-instrumentation.git --depth 1 --branch main
 cd opentelemetry-java-instrumentation/

From 5c35ae1d53e7313516e15d802de58c79bf9f2a5a Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 13:06:38 +0200
Subject: [PATCH 18/31] chore: change logs

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 9bf914f11e..5491da4443 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -21,7 +21,7 @@ ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
 ELASTIC_SNAPSHOT_JAR_FILE="$(pwd)/$ELASTIC_SNAPSHOT_JAR"
 echo "$ELASTIC_SNAPSHOT_JAR_FILE has been downloaded."
 
-echo "--- Download the latest elastic-agent.zip"
+echo "--- Start APM Server mock"
 git clone https://github.com/elastic/apm-mutating-webhook.git
 pushd apm-mutating-webhook/test/mock
 docker build -t $CONTAINER_NAME .

From 7ea0636f6d1f2d0f23cdb8c50b70c47c262abf14 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 13:44:35 +0200
Subject: [PATCH 19/31] support generate ES docs

---
 .buildkite/load-testing.yml                   |  3 ++-
 .buildkite/opentelemetry-benchmark.yml        |  1 +
 .buildkite/scripts/opentelemetry-benchmark.sh | 25 ++++++++++++++++---
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/.buildkite/load-testing.yml b/.buildkite/load-testing.yml
index 380613fa13..9beba8d952 100644
--- a/.buildkite/load-testing.yml
+++ b/.buildkite/load-testing.yml
@@ -4,4 +4,5 @@ steps:
     agents:
       queue: observability-microbenchmarks
     artifact_paths:
-      - "**/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html"
\ No newline at end of file
+      - "**/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html"
+      - "output.json"
\ No newline at end of file
diff --git a/.buildkite/opentelemetry-benchmark.yml b/.buildkite/opentelemetry-benchmark.yml
index cf24dc691e..6d26e30b14 100644
--- a/.buildkite/opentelemetry-benchmark.yml
+++ b/.buildkite/opentelemetry-benchmark.yml
@@ -5,6 +5,7 @@ steps:
       queue: observability-microbenchmarks
     artifact_paths:
       - "**/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html"
+      - "output.json"
 
 notify:
   - slack: "#apm-agent-java"
diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 5491da4443..27d8d4b885 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -30,11 +30,11 @@ popd
 
 echo "--- Build opentelemetry-java-instrumentation"
 git clone https://github.com/open-telemetry/opentelemetry-java-instrumentation.git --depth 1 --branch main
-cd opentelemetry-java-instrumentation/
+pushd opentelemetry-java-instrumentation/
 ./gradlew assemble
 
 echo "--- Customise the elastic opentelemetry java instrumentation"
-cd benchmark-overhead
+pushd benchmark-overhead
 cp "$ELASTIC_SNAPSHOT_JAR_FILE" .
 ELASTIC_SNAPSHOT_ENTRY="new Agent(\\\"elastic-snapshot\\\",\\\"latest available snapshot version from elastic main\\\",\\\"file://$PWD/$ELASTIC_SNAPSHOT_JAR\\\")"
 ELASTIC_LATEST_VERSION=$(curl -s https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/ | perl -ne 's/<.*?>//g; if(s/^([\d\.]+).*$/$1/){print}' | sort -V | tail -1)
@@ -47,8 +47,10 @@ perl -i -ne "if (/withAgents/) {print \"$NEW_LINE\n\"}else{print}" src/test/java
 echo "--- Run tests of benchmark-overhead"
 ./gradlew test
 
-echo "--- Report"
-perl -ne '/Standard output/ && $on++; /\<\/pre\>/ && ($on=0);$on && s/\<.*\>//;$on && !/^\s*$/ && print' build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html | tee report.txt
+echo "--- Report in Buildkite"
+
+REPORT_FILE=$(pwd)/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html
+perl -ne '/Standard output/ && $on++; /\<\/pre\>/ && ($on=0);$on && s/\<.*\>//;$on && !/^\s*$/ && print' $REPORT_FILE | tee report.txt
 
 # Buildkite annotation
 if [ -n "$BUILDKITE" ]; then
@@ -63,4 +65,19 @@ if [ -n "$BUILDKITE" ]; then
 EOF
 fi
 
+echo "--- Setup Report"
+popd
+popd
+find . -name "opentelemetry-javaagent.jar" -ls
+./mvnw clean package -DskipTests=true -Dmaven.javadoc.skip=true
+java -cp apm-agent-benchmarks/target/benchmarks.jar \
+  co.elastic.apm.agent.benchmark.ProcessOtelBenchmarkResults \
+  ${RESULT_FILE} output.json $ELASTIC_LATEST_VERSION opentelemetry-javaagent.jar
+
 echo "--- Send Report (TBC)"
+exit 0
+curl -X POST \
+  --user "${ES_USER_SECRET}:${ES_PASS_SECRET}" \
+  "${ES_URL_SECRET}/_bulk?pretty" \
+  -H "Content-Type: application/x-ndjson" \
+  --data-binary @output.json

From 315214241674060150a478f214726d0a1fc79e5f Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 14:29:40 +0200
Subject: [PATCH 20/31] fix: env variable and use the relative path to the jar
 file

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 27d8d4b885..5e1e3d0f73 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -68,11 +68,11 @@ fi
 echo "--- Setup Report"
 popd
 popd
-find . -name "opentelemetry-javaagent.jar" -ls
+JSON_FILE="$(pwd)/output.json"
 ./mvnw clean package -DskipTests=true -Dmaven.javadoc.skip=true
 java -cp apm-agent-benchmarks/target/benchmarks.jar \
   co.elastic.apm.agent.benchmark.ProcessOtelBenchmarkResults \
-  ${RESULT_FILE} output.json $ELASTIC_LATEST_VERSION opentelemetry-javaagent.jar
+  "$REPORT_FILE" "$JSON_FILE" "$ELASTIC_LATEST_VERSION" ./opentelemetry-java-instrumentation/benchmark-overhead/opentelemetry-javaagent.jar
 
 echo "--- Send Report (TBC)"
 exit 0

From e846d1582dde327c2ee1de25e0fcb09ae3064321 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 14:30:18 +0200
Subject: [PATCH 21/31] reduce log levels for the maven thing

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 5e1e3d0f73..37cb362386 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -69,7 +69,7 @@ echo "--- Setup Report"
 popd
 popd
 JSON_FILE="$(pwd)/output.json"
-./mvnw clean package -DskipTests=true -Dmaven.javadoc.skip=true
+./mvnw -q clean package -DskipTests=true -Dmaven.javadoc.skip=true
 java -cp apm-agent-benchmarks/target/benchmarks.jar \
   co.elastic.apm.agent.benchmark.ProcessOtelBenchmarkResults \
   "$REPORT_FILE" "$JSON_FILE" "$ELASTIC_LATEST_VERSION" ./opentelemetry-java-instrumentation/benchmark-overhead/opentelemetry-javaagent.jar

From d0c4fd683def6ba5cda2d050d68c3a82d191c143 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 14:35:04 +0200
Subject: [PATCH 22/31] chore: log

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 37cb362386..8d3836af2c 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -12,7 +12,7 @@ function cleanup {
 
 trap cleanup EXIT
 
-echo "--- Download the latest elastic-agent.zip"
+echo "--- Download the latest elastic-apm-agent artifact"
 # run earlier so gh can use the current github repository.
 run_id=$(gh run list --branch main --status success --workflow main.yml -L 1 --json databaseId --jq '.[].databaseId')
 echo "downloading the latest artifact 'elastic-apm-agent' (using the workflow run '$run_id')"

From 00d630305b17c5f8906e2dd9d46cced63d87ebbc Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 14:35:34 +0200
Subject: [PATCH 23/31] chore

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 8d3836af2c..94371ea488 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -66,6 +66,7 @@ EOF
 fi
 
 echo "--- Setup Report"
+## TODO: use the existing generated benchmarks.jar file
 popd
 popd
 JSON_FILE="$(pwd)/output.json"

From 5c0e65188cd59976304e8d7e1d666e122a41ffae Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 14:56:19 +0200
Subject: [PATCH 24/31] feat: change index name

---
 .../apm/agent/benchmark/ProcessOtelBenchmarkResults.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java b/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java
index bfef48cbbc..fd2886a82f 100644
--- a/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java
+++ b/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java
@@ -122,7 +122,7 @@ private void writeBulkFile(String resultFilePath) throws IOException {
         final File file = new File(resultFilePath);
         final FileWriter fileWriter = new FileWriter(file);
         for (JsonNode benchmark : bechmarkResultJson) {
-            fileWriter.append("{ \"index\" : { \"_index\" : \"microbenchmarks\" } }\n");
+            fileWriter.append("{ \"index\" : { \"_index\" : \"otel-microbenchmarks\" } }\n");
             fileWriter.append(objectMapper.writer().writeValueAsString(benchmark));
             fileWriter.append("\n");
         }

From 313354c06ab3cb2b32dab9c01a11fdcfa76b41f2 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 14:56:28 +0200
Subject: [PATCH 25/31] enable send benchmark

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 94371ea488..404b48c555 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -76,7 +76,6 @@ java -cp apm-agent-benchmarks/target/benchmarks.jar \
   "$REPORT_FILE" "$JSON_FILE" "$ELASTIC_LATEST_VERSION" ./opentelemetry-java-instrumentation/benchmark-overhead/opentelemetry-javaagent.jar
 
 echo "--- Send Report (TBC)"
-exit 0
 curl -X POST \
   --user "${ES_USER_SECRET}:${ES_PASS_SECRET}" \
   "${ES_URL_SECRET}/_bulk?pretty" \

From 84962979b16969be194acf7b8665639ccad1a1c6 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 15:42:08 +0200
Subject: [PATCH 26/31] consume github action artifact

so there is no need to built from sources but use the latest one
---
 .buildkite/scripts/opentelemetry-benchmark.sh | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 404b48c555..d003ea2cf8 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -2,6 +2,7 @@
 set -eo pipefail
 
 CONTAINER_NAME=mock-apm-server
+JSON_FILE="$(pwd)/output.json"
 
 function cleanup {
   echo "--- Tear down the environment"
@@ -20,6 +21,10 @@ gh run download "$run_id" -n elastic-apm-agent
 ELASTIC_SNAPSHOT_JAR=$(ls -1 elastic-apm-agent-*.jar)
 ELASTIC_SNAPSHOT_JAR_FILE="$(pwd)/$ELASTIC_SNAPSHOT_JAR"
 echo "$ELASTIC_SNAPSHOT_JAR_FILE has been downloaded."
+gh run download "$run_id" -n apm-agent-benchmarks
+BENCHMARKS_JAR=$(ls -1 benchmarks*.jar)
+BENCHMARKS_JAR_FILE="$(pwd)/$BENCHMARKS_JAR"
+echo "$BENCHMARKS_JAR_FILE has been downloaded."
 
 echo "--- Start APM Server mock"
 git clone https://github.com/elastic/apm-mutating-webhook.git
@@ -65,19 +70,15 @@ if [ -n "$BUILDKITE" ]; then
 EOF
 fi
 
-echo "--- Setup Report"
-## TODO: use the existing generated benchmarks.jar file
-popd
-popd
+echo "--- Generate ES docs"
 JSON_FILE="$(pwd)/output.json"
-./mvnw -q clean package -DskipTests=true -Dmaven.javadoc.skip=true
-java -cp apm-agent-benchmarks/target/benchmarks.jar \
+java -cp $BENCHMARKS_JAR_FILE \
   co.elastic.apm.agent.benchmark.ProcessOtelBenchmarkResults \
-  "$REPORT_FILE" "$JSON_FILE" "$ELASTIC_LATEST_VERSION" ./opentelemetry-java-instrumentation/benchmark-overhead/opentelemetry-javaagent.jar
+  "$REPORT_FILE" "$JSON_FILE" "$ELASTIC_LATEST_VERSION" ./opentelemetry-javaagent.jar
 
-echo "--- Send Report (TBC)"
+echo "--- Send Report"
 curl -X POST \
   --user "${ES_USER_SECRET}:${ES_PASS_SECRET}" \
   "${ES_URL_SECRET}/_bulk?pretty" \
   -H "Content-Type: application/x-ndjson" \
-  --data-binary @output.json
+  --data-binary @"$JSON_FILE"

From f8a3cc8f4a40572bcbd90c9aa11cd4385e589dc0 Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Thu, 26 Oct 2023 17:28:40 +0200
Subject: [PATCH 27/31] Apply suggestions from code review

Co-authored-by: jackshirazi <jack.shirazi@elastic.co>
---
 .buildkite/scripts/opentelemetry-benchmark.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index d003ea2cf8..0d8963b79c 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -41,10 +41,10 @@ pushd opentelemetry-java-instrumentation/
 echo "--- Customise the elastic opentelemetry java instrumentation"
 pushd benchmark-overhead
 cp "$ELASTIC_SNAPSHOT_JAR_FILE" .
-ELASTIC_SNAPSHOT_ENTRY="new Agent(\\\"elastic-snapshot\\\",\\\"latest available snapshot version from elastic main\\\",\\\"file://$PWD/$ELASTIC_SNAPSHOT_JAR\\\")"
+ELASTIC_SNAPSHOT_ENTRY="new Agent(\\\"elastic-snapshot\\\",\\\"latest available snapshot version from elastic main\\\",\\\"file://$PWD/$ELASTIC_SNAPSHOT_JAR\\\", java.util.List.of(\\\"-Delastic.apm.server_url=http://host.docker.internal:8027/\\\"))"
 ELASTIC_LATEST_VERSION=$(curl -s https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/ | perl -ne 's/<.*?>//g; if(s/^([\d\.]+).*$/$1/){print}' | sort -V | tail -1)
-ELASTIC_LATEST_ENTRY="new Agent(\\\"elastic-latest\\\",\\\"latest available released version from elastic main\\\",\\\"https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_LATEST_VERSION/elastic-apm-agent-$ELASTIC_LATEST_VERSION.jar\\\")"
-ELASTIC_LATEST_ENTRY2="new Agent(\\\"elastic-async\\\",\\\"latest available released version from elastic main\\\",\\\"https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_LATEST_VERSION/elastic-apm-agent-$ELASTIC_LATEST_VERSION.jar\\\", java.util.List.of(\\\"-Delastic.apm.delay_agent_premain_ms=15000\\\"))"
+ELASTIC_LATEST_ENTRY="new Agent(\\\"elastic-latest\\\",\\\"latest available released version from elastic main\\\",\\\"https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_LATEST_VERSION/elastic-apm-agent-$ELASTIC_LATEST_VERSION.jar\\\", java.util.List.of(\\\"-Delastic.apm.server_url=http://host.docker.internal:8027/\\\"))"
+ELASTIC_LATEST_ENTRY2="new Agent(\\\"elastic-async\\\",\\\"latest available released version from elastic main\\\",\\\"https://repo1.maven.org/maven2/co/elastic/apm/elastic-apm-agent/$ELASTIC_LATEST_VERSION/elastic-apm-agent-$ELASTIC_LATEST_VERSION.jar\\\", java.util.List.of(\\\"-Delastic.apm.delay_agent_premain_ms=15000\\\",\\\"-Delastic.apm.server_url=http://host.docker.internal:8027/\\\"))"
 NEW_LINE="              .withAgents(Agent.NONE, Agent.LATEST_RELEASE, Agent.LATEST_SNAPSHOT, $ELASTIC_LATEST_ENTRY, $ELASTIC_LATEST_ENTRY2, $ELASTIC_SNAPSHOT_ENTRY)"
 echo $NEW_LINE
 perl -i -ne "if (/withAgents/) {print \"$NEW_LINE\n\"}else{print}" src/test/java/io/opentelemetry/config/Configs.java

From 8c39208c2ec6e1c3298c9f5594797cad23a48c9d Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 17:51:55 +0200
Subject: [PATCH 28/31] feat: change index name

---
 .../apm/agent/benchmark/ProcessOtelBenchmarkResults.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java b/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java
index 7af55e86ad..073b61574b 100644
--- a/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java
+++ b/apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/ProcessOtelBenchmarkResults.java
@@ -122,7 +122,7 @@ private void writeBulkFile(String resultFilePath) throws IOException {
         final File file = new File(resultFilePath);
         final FileWriter fileWriter = new FileWriter(file);
         for (JsonNode benchmark : bechmarkResultJson) {
-            fileWriter.append("{ \"index\" : { \"_index\" : \"microbenchmarks\" } }\n");
+            fileWriter.append("{ \"index\" : { \"_index\" : \"otel-microbenchmarks\" } }\n");
             fileWriter.append(objectMapper.writer().writeValueAsString(benchmark));
             fileWriter.append("\n");
         }

From ac57ca09953a447fa4f566f67fdfe9f4b29bafc2 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 17:56:39 +0200
Subject: [PATCH 29/31] fix location of the file

---
 .buildkite/scripts/opentelemetry-benchmark.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.buildkite/scripts/opentelemetry-benchmark.sh b/.buildkite/scripts/opentelemetry-benchmark.sh
index 0d8963b79c..8753247883 100755
--- a/.buildkite/scripts/opentelemetry-benchmark.sh
+++ b/.buildkite/scripts/opentelemetry-benchmark.sh
@@ -71,7 +71,6 @@ EOF
 fi
 
 echo "--- Generate ES docs"
-JSON_FILE="$(pwd)/output.json"
 java -cp $BENCHMARKS_JAR_FILE \
   co.elastic.apm.agent.benchmark.ProcessOtelBenchmarkResults \
   "$REPORT_FILE" "$JSON_FILE" "$ELASTIC_LATEST_VERSION" ./opentelemetry-javaagent.jar

From 7d02e6db70f99bbdb916ea179e92f9eb937854b4 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 17:58:09 +0200
Subject: [PATCH 30/31] revert

---
 .buildkite/load-testing.yml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/.buildkite/load-testing.yml b/.buildkite/load-testing.yml
index 9beba8d952..09cf887956 100644
--- a/.buildkite/load-testing.yml
+++ b/.buildkite/load-testing.yml
@@ -1,8 +1,4 @@
+# @reakaleek: This is a place holder to create the pipeline in Buildkite. I will work on it, in a follow-up.
 steps:
-  - label: "Run the microbenchmark"
-    commands: .buildkite/scripts/opentelemetry-benchmark.sh
-    agents:
-      queue: observability-microbenchmarks
-    artifact_paths:
-      - "**/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html"
-      - "output.json"
\ No newline at end of file
+  - label: ":wave: Greetings"            # Label (with rich emojis https://ela.st/bk-emoji).
+    command: "echo 'My first pipeline!'" # Command to run (evaluated by Bash).
\ No newline at end of file

From ac40f0877aa1616159079f8376cc5fce88e5d404 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 26 Oct 2023 18:08:29 +0200
Subject: [PATCH 31/31] chore: archive test results to help with debugging if
 any test failures

---
 .buildkite/opentelemetry-benchmark.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/opentelemetry-benchmark.yml b/.buildkite/opentelemetry-benchmark.yml
index 6d26e30b14..9deb6395bc 100644
--- a/.buildkite/opentelemetry-benchmark.yml
+++ b/.buildkite/opentelemetry-benchmark.yml
@@ -5,6 +5,7 @@ steps:
       queue: observability-microbenchmarks
     artifact_paths:
       - "**/build/reports/tests/test/classes/io.opentelemetry.OverheadTests.html"
+      - "**/build/reports/tests/test/index.html"
       - "output.json"
 
 notify: