build/teamcity: add changes to enable openmetrics in nightly roachtests

There was a regression in #135239.
The change was reverted and this PR aims to fix the regressions

Epic: https://cockroachlabs.atlassian.net/browse/CRDB-41852

Release note: None

build/github/cockroach-microbench-ci.sh

@@ -48,7 +48,6 @@ bazel test //pkg/sql/tests:tests_test \
   --test_arg=-test.run=- \
   --test_arg='-test.bench=^BenchmarkKV$' \
   --test_sharding_strategy=disabled \
-  --test_arg=-test.cpu --test_arg=1 \
   --test_arg=-test.v \
   --test_arg=-test.count=10 \
   --test_arg=-test.benchmem \

build/teamcity/cockroach/nightlies/roachtest_nightly_aws.sh

@@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"  # For $root
 source "$dir/teamcity-bazel-support.sh"  # For run_bazel
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
 			       run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

build/teamcity/cockroach/nightlies/roachtest_nightly_azure.sh

@@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"  # For $root
 source "$dir/teamcity-bazel-support.sh"  # For run_bazel
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AZURE_CLIENT_ID -e AZURE_CLIENT_SECRET -e AZURE_SUBSCRIPTION_ID -e AZURE_TENANT_ID -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e CLEAR_CLUSTER_CACHE -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
 			       run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

build/teamcity/cockroach/nightlies/roachtest_nightly_gce.sh

@@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"  # For $root
 source "$dir/teamcity-bazel-support.sh"  # For run_bazel
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY" \
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e GOOGLE_KMS_KEY_A -e GOOGLE_KMS_KEY_B -e GOOGLE_CREDENTIALS_ASSUME_ROLE -e GOOGLE_SERVICE_ACCOUNT -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e SELECT_PROBABILITY -e COCKROACH_RANDOM_SEED -e ROACHTEST_ASSERTIONS_ENABLED_SEED -e ROACHTEST_FORCE_RUN_INVALID_RELEASE_BRANCH -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SELECTIVE_TESTS -e SFUSER -e SFPASSWORD -e SIDE_EYE_API_TOKEN -e COCKROACH_EA_PROBABILITY -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
 			       run_bazel build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

@@ -84,4 +84,5 @@ build/teamcity-roachtest-invoke.sh \
   --suite nightly \
   --selective-tests="${SELECTIVE_TESTS:-true}" \
   --side-eye-token="${SIDE_EYE_API_TOKEN}" \
+  --export-openmetrics="${EXPORT_OPENMETRICS:-false}" \
   "${TESTS}"

build/teamcity/cockroach/nightlies/roachtest_weekly.sh

@@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"  # For $root
 source "$dir/teamcity-bazel-support.sh"  # For run_bazel
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN" \
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e GRAFANA_SERVICE_ACCOUNT_JSON -e GRAFANA_SERVICE_ACCOUNT_AUDIENCE -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
 			       run_bazel build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh

build/teamcity/cockroach/nightlies/roachtest_weekly_aws.sh

@@ -13,5 +13,5 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"  # For $root
 source "$dir/teamcity-bazel-support.sh"  # For run_bazel
 
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN" \
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e AWS_ACCESS_KEY_ID -e AWS_ACCESS_KEY_ID_ASSUME_ROLE -e AWS_KMS_KEY_ARN_A -e AWS_KMS_KEY_ARN_B -e AWS_KMS_REGION_A -e AWS_KMS_REGION_B -e AWS_ROLE_ARN -e AWS_SECRET_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY_ASSUME_ROLE -e BUILD_VCS_NUMBER -e CLOUD -e COCKROACH_DEV_LICENSE -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e ARM_PROBABILITY -e USE_SPOT -e SIDE_EYE_API_TOKEN -e EXPORT_OPENMETRICS -e ROACHPERF_OPENMETRICS_CREDENTIALS" \
 			       run_bazel build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh

build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh

@@ -47,4 +47,5 @@ timeout -s INT $((7800*60)) build/teamcity-roachtest-invoke.sh \
   --metamorphic-arm64-probability="${ARM_PROBABILITY:-0.5}" \
   --use-spot="${USE_SPOT:-auto}" \
   --slack-token="${SLACK_TOKEN}" \
-  --side-eye-token="${SIDE_EYE_API_TOKEN}"
+  --side-eye-token="${SIDE_EYE_API_TOKEN}" \
+  --export-openmetrics="${EXPORT_OPENMETRICS:-false}"

build/teamcity/util/roachtest_util.sh

@@ -23,50 +23,70 @@ source  $root/build/teamcity/util/roachtest_arch_util.sh
 # date at the time of the start of the run (which identifies the version of the
 # code run best).
 stats_dir="$(date +"%Y%m%d")-${TC_BUILD_ID}"
+stats_file_name="stats.json"
+
+# Provide a default value for EXPORT_OPENMETRICS if it is not set
+EXPORT_OPENMETRICS="${EXPORT_OPENMETRICS:-false}"
+
+if [[ "${EXPORT_OPENMETRICS}" == "true" ]]; then
+  stats_file_name="stats.om"
+fi
 
 # Set up a function we'll invoke at the end.
 function upload_stats {
   if tc_release_branch; then
-      bucket="${ROACHTEST_BUCKET:-cockroach-nightly-${CLOUD}}"
-      if [[ "${CLOUD}" == "gce" ]]; then
-          # GCE, having been there first, gets an exemption.
-          bucket="cockroach-nightly"
-      fi
+    bucket="${ROACHTEST_BUCKET:-cockroach-nightly-${CLOUD}}"
+    if [[ "${EXPORT_OPENMETRICS}" == "true" ]]; then
 
-      branch=$(tc_build_branch)
-      remote_artifacts_dir="artifacts-${branch}"
-      if [[ "${branch}" == "master" ]]; then
-        # The master branch is special, as roachperf hard-codes
-        # the location.
-        remote_artifacts_dir="artifacts"
-      fi
-      # TODO: FIPS_ENABLED is deprecated, use roachtest --metamorphic-fips-probability, instead.
-      # In FIPS-mode, keep artifacts separate by using the 'fips' suffix.
-      if [[ ${FIPS_ENABLED:-0} == 1 ]]; then
-        remote_artifacts_dir="${remote_artifacts_dir}-fips"
-      fi
-
-      # The stats.json files need some path translation:
-      #     ${artifacts}/path/to/test/stats.json
-      # to
-      #     gs://${bucket}/artifacts/${stats_dir}/path/to/test/stats.json
-      #
-      # `find` below will expand "{}" as ./path/to/test/stats.json. We need
-      # to bend over backwards to remove the `./` prefix or gsutil will have
-      # a `.` folder in ${stats_dir}, which we don't want.
-      (cd "${artifacts}" && \
-        while IFS= read -r f; do
-          if [[ -n "${f}" ]]; then
-            artifacts_dir="${remote_artifacts_dir}"
-            # If 'cpu_arch=xxx' is encoded in the path, use it as suffix to separate artifacts by cpu_arch.
-            if [[ "${f}" == *"/cpu_arch=arm64/"* ]]; then
-              artifacts_dir="${artifacts_dir}-arm64"
-            elif [[ "${f}" == *"/cpu_arch=fips/"* ]]; then
-              artifacts_dir="${artifacts_dir}-fips"
-            fi
-            gsutil cp "${f}" "gs://${bucket}/${artifacts_dir}/${stats_dir}/${f}"
+        # TODO(sambhav-jain-16): Change the bucket after new buckets are created
+        bucket="${ROACHTEST_BUCKET:-cockroach-testeng-metrics/omloader/incoming/${CLOUD}}"
+    fi
+
+    if [[ "${CLOUD}" == "gce" && "${EXPORT_OPENMETRICS}" == "false" ]]; then
+        # GCE, having been there first, gets an exemption.
+        bucket="cockroach-nightly"
+    fi
+
+    branch=$(tc_build_branch)
+    remote_artifacts_dir="artifacts-${branch}"
+    if [[ "${branch}" == "master" ]]; then
+      # The master branch is special, as roachperf hard-codes
+      # the location.
+      remote_artifacts_dir="artifacts"
+    fi
+    # TODO: FIPS_ENABLED is deprecated, use roachtest --metamorphic-fips-probability, instead.
+    # In FIPS-mode, keep artifacts separate by using the 'fips' suffix.
+    if [[ ${FIPS_ENABLED:-0} == 1 ]]; then
+      remote_artifacts_dir="${remote_artifacts_dir}-fips"
+    fi
+
+    # If using openmetrics, activate new service account for uploading to openmetrics bucket
+    if [[ "${EXPORT_OPENMETRICS}" == "true" && "$ROACHPERF_OPENMETRICS_CREDENTIALS" ]]; then
+      echo "$ROACHPERF_OPENMETRICS_CREDENTIALS" > roachperf.json
+      gcloud auth activate-service-account --key-file=roachperf.json
+    fi
+
+    # The ${stats_file_name} files need some path translation:
+    #     ${artifacts}/path/to/test/${stats_file_name}
+    # to
+    #     gs://${bucket}/artifacts/${stats_dir}/path/to/test/${stats_file_name}
+    #
+    # `find` below will expand "{}" as ./path/to/test/${stats_file_name}. We need
+    # to bend over backwards to remove the `./` prefix or gsutil will have
+    # a `.` folder in ${stats_dir}, which we don't want.
+    (cd "${artifacts}" && \
+      while IFS= read -r f; do
+        if [[ -n "${f}" ]]; then
+          artifacts_dir="${remote_artifacts_dir}"
+          # If 'cpu_arch=xxx' is encoded in the path, use it as suffix to separate artifacts by cpu_arch.
+          if [[ "${f}" == *"/cpu_arch=arm64/"* ]]; then
+            artifacts_dir="${artifacts_dir}-arm64"
+          elif [[ "${f}" == *"/cpu_arch=fips/"* ]]; then
+            artifacts_dir="${artifacts_dir}-fips"
           fi
-        done <<< "$(find . -name stats.json | sed 's/^\.\///')")
+          gsutil cp "${f}" "gs://${bucket}/${artifacts_dir}/${stats_dir}/${f}"
+        fi
+      done <<< "$(find . -name ${stats_file_name} | sed 's/^\.\///')")
   fi
 }
 
@@ -92,7 +112,7 @@ function upload_all {
   upload_binaries
 }
 
-# Upload any stats.json we can find, and some binaries, no matter what happens.
+# Upload any ${stats_file_name} we can find, and some binaries, no matter what happens.
 trap upload_all EXIT
 
 # Set up the parameters for the roachtest invocation.