Add option to enable TBS in benchmark terraform (#14985)

Update testing/benchmark terraform and github workflow to add an option to enable TBS.

.github/workflows/benchmarks.yml

@@ -8,6 +8,16 @@ on:
         required: false
         type: boolean
         default: false
+      enableTailSampling:
+        description: 'Enable tail-based sampling on the APM server'
+        required: false
+        type: boolean
+        default: false
+      tailSamplingStorageLimit:
+        description: 'Storage size limit of tail-based sampling on the APM server, defaults to 10GB'
+        required: false
+        type: string
+        default: "10GB"
       profile:
         description: 'The system profile used to run the benchmarks'
         required: false
@@ -52,12 +62,14 @@ jobs:
       TF_VAR_private_key: ./id_rsa_terraform
       TF_VAR_public_key: ./id_rsa_terraform.pub
       TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
+      TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling }}
+      TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit }}
       RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
       TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile
       TF_VAR_BUILD_ID: ${{ github.run_id }}
       TF_VAR_ENVIRONMENT: ci
       TF_VAR_REPO: ${{ github.repository }}
-      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }}
+      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }},enable_tail_sampling=${{ inputs.enableTailSampling }}
       GOBENCH_PASSWORD: ${{ secrets.GOBENCH_PASSWORD }}
       GOBENCH_USERNAME: ${{ secrets.GOBENCH_USERNAME }}
       GOBENCH_HOST: ${{ secrets.GOBENCH_HOST }}

testing/benchmark/main.tf

@@ -97,12 +97,14 @@ module "ec_deployment" {
   deployment_template    = var.deployment_template
   deployment_name_prefix = local.name_prefix
 
-  apm_server_size       = var.apm_server_size
-  apm_server_zone_count = var.apm_server_zone_count
-  apm_index_shards      = var.apm_shards
-  drop_pipeline         = var.drop_pipeline
-  apm_server_expvar     = true
-  apm_server_pprof      = true
+  apm_server_size                        = var.apm_server_size
+  apm_server_zone_count                  = var.apm_server_zone_count
+  apm_index_shards                       = var.apm_shards
+  drop_pipeline                          = var.drop_pipeline
+  apm_server_expvar                      = true
+  apm_server_pprof                       = true
+  apm_server_tail_sampling               = var.apm_server_tail_sampling
+  apm_server_tail_sampling_storage_limit = var.apm_server_tail_sampling_storage_limit
 
   elasticsearch_size              = var.elasticsearch_size
   elasticsearch_zone_count        = var.elasticsearch_zone_count

testing/benchmark/variables.tf

@@ -92,6 +92,18 @@ variable "drop_pipeline" {
   type        = bool
 }
 
+variable "apm_server_tail_sampling" {
+  default     = false
+  description = "Whether or not to enable APM Server tail-based sampling. Defaults to false"
+  type        = bool
+}
+
+variable "apm_server_tail_sampling_storage_limit" {
+  default     = "10GB"
+  description = "Storage size limit of APM Server tail-based sampling. Defaults to 10GB"
+  type        = string
+}
+
 # Standalone
 
 variable "apm_server_bin_path" {

testing/infra/terraform/modules/ec_deployment/README.md

@@ -28,12 +28,12 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | [ec_deployment.deployment](https://registry.terraform.io/providers/elastic/ec/0.5.1/docs/resources/deployment) | resource |
 | [local_file.custom_apm_integration_pkg](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.drop_pipeline](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
-| [local_file.enable_expvar](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.enable_features](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.secret_token](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.shard_settings](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [null_resource.custom_apm_integration_pkg](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.drop_pipeline](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
-| [null_resource.enable_expvar](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
+| [null_resource.enable_features](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.secret_token](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [null_resource.shard_settings](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [ec_stack.deployment_version](https://registry.terraform.io/providers/elastic/ec/0.5.1/docs/data-sources/stack) | data source |
@@ -47,12 +47,14 @@ used to configure the module, please refer to the [EC Provider docs](https://reg
 | <a name="input_apm_server_expvar"></a> [apm\_server\_expvar](#input\_apm\_server\_expvar) | Whether or not to enable APM Server's expvar endpoint. Defaults to false | `bool` | `false` | no |
 | <a name="input_apm_server_pprof"></a> [apm\_server\_pprof](#input\_apm\_server\_pprof) | Whether or not to enable APM Server's pprof endpoint. Defaults to false | `bool` | `false` | no |
 | <a name="input_apm_server_size"></a> [apm\_server\_size](#input\_apm\_server\_size) | Optional apm server instance size | `string` | `"1g"` | no |
+| <a name="input_apm_server_tail_sampling"></a> [apm\_server\_tail\_sampling](#input\_apm\_server\_tail\_sampling) | Whether or not to enable APM Server tail-based sampling. Defaults to false | `bool` | `false` | no |
+| <a name="input_apm_server_tail_sampling_storage_limit"></a> [apm\_server\_tail\_sampling\_storage\_limit](#input\_apm\_server\_tail\_sampling\_storage\_limit) | Storage size limit of APM Server tail-based sampling. Defaults to 10GB | `string` | `"10GB"` | no |
 | <a name="input_apm_server_zone_count"></a> [apm\_server\_zone\_count](#input\_apm\_server\_zone\_count) | Optional apm server zone count | `number` | `1` | no |
 | <a name="input_custom_apm_integration_pkg_path"></a> [custom\_apm\_integration\_pkg\_path](#input\_custom\_apm\_integration\_pkg\_path) | Path to the zipped custom APM integration package, if empty custom apm integration pkg is not installed | `string` | `""` | no |
 | <a name="input_deployment_name_prefix"></a> [deployment\_name\_prefix](#input\_deployment\_name\_prefix) | Optional ESS or ECE region. Defaults to GCP US West 2 (Los Angeles) | `string` | `"apmserver"` | no |
 | <a name="input_deployment_template"></a> [deployment\_template](#input\_deployment\_template) | Optional deployment template. Defaults to the CPU optimized template for GCP | `string` | `"gcp-cpu-optimized"` | no |
-| <a name="input_docker_image"></a> [docker\_image](#input\_docker\_image) | Optional docker image overrides. The full map needs to be specified | `map(string)` | <pre>{<br>  "apm": "docker.elastic.co/cloud-release/elastic-agent-cloud",<br>  "elasticsearch": "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",<br>  "kibana": "docker.elastic.co/cloud-release/kibana-cloud"<br>}</pre> | no |
-| <a name="input_docker_image_tag_override"></a> [docker\_image\_tag\_override](#input\_docker\_image\_tag\_override) | Optional docker image tag overrides, The full map needs to be specified | `map(string)` | <pre>{<br>  "apm": "",<br>  "elasticsearch": "",<br>  "kibana": ""<br>}</pre> | no |
+| <a name="input_docker_image"></a> [docker\_image](#input\_docker\_image) | Optional docker image overrides. The full map needs to be specified | `map(string)` | <pre>{<br/>  "apm": "docker.elastic.co/cloud-release/elastic-agent-cloud",<br/>  "elasticsearch": "docker.elastic.co/cloud-release/elasticsearch-cloud-ess",<br/>  "kibana": "docker.elastic.co/cloud-release/kibana-cloud"<br/>}</pre> | no |
+| <a name="input_docker_image_tag_override"></a> [docker\_image\_tag\_override](#input\_docker\_image\_tag\_override) | Optional docker image tag overrides, The full map needs to be specified | `map(string)` | <pre>{<br/>  "apm": "",<br/>  "elasticsearch": "",<br/>  "kibana": ""<br/>}</pre> | no |
 | <a name="input_drop_pipeline"></a> [drop\_pipeline](#input\_drop\_pipeline) | Whether or not to install an Elasticsearch ingest pipeline to drop all incoming APM documents. Defaults to false | `bool` | `false` | no |
 | <a name="input_elasticsearch_autoscale"></a> [elasticsearch\_autoscale](#input\_elasticsearch\_autoscale) | Optional autoscale the Elasticsearch cluster | `bool` | `false` | no |
 | <a name="input_elasticsearch_dedicated_masters"></a> [elasticsearch\_dedicated\_masters](#input\_elasticsearch\_dedicated\_masters) | Optionally use dedicated masters for the Elasticsearch cluster | `bool` | `false` | no |

testing/infra/terraform/modules/ec_deployment/deployment.tf

@@ -113,14 +113,16 @@ resource "ec_deployment" "dedicated_observability_deployment" {
   kibana {}
 }
 
-resource "local_file" "enable_expvar" {
-  content = templatefile("${path.module}/scripts/enable_expvar.tftpl", {
-    kibana_url       = ec_deployment.deployment.kibana.0.https_endpoint,
-    elastic_password = ec_deployment.deployment.elasticsearch_password,
-    enable_expvar    = var.apm_server_expvar
-    enable_pprof     = var.apm_server_pprof
+resource "local_file" "enable_features" {
+  content = templatefile("${path.module}/scripts/enable_features.tftpl", {
+    kibana_url                  = ec_deployment.deployment.kibana.0.https_endpoint,
+    elastic_password            = ec_deployment.deployment.elasticsearch_password,
+    enable_expvar               = var.apm_server_expvar
+    enable_pprof                = var.apm_server_pprof
+    enable_tail_sampling        = var.apm_server_tail_sampling
+    tail_sampling_storage_limit = var.apm_server_tail_sampling_storage_limit
   })
-  filename = "${path.module}/scripts/enable_expvar.sh"
+  filename = "${path.module}/scripts/enable_features.sh"
 }
 
 resource "local_file" "secret_token" {
@@ -153,13 +155,13 @@ resource "local_file" "custom_apm_integration_pkg" {
   filename = "${path.module}/scripts/custom-apm-integration-pkg.sh"
 }
 
-resource "null_resource" "enable_expvar" {
+resource "null_resource" "enable_features" {
   triggers = {
-    shell_hash          = local_file.enable_expvar.id
+    shell_hash          = local_file.enable_features.id
     integrations_server = var.integrations_server
   }
   provisioner "local-exec" {
-    command     = "scripts/enable_expvar.sh"
+    command     = "scripts/enable_features.sh"
     interpreter = ["/bin/bash", "-c"]
     working_dir = path.module
   }

testing/infra/terraform/modules/ec_deployment/scripts/enable_expvar.tftpl → testing/infra/terraform/modules/ec_deployment/scripts/enable_features.tftpl

@@ -14,19 +14,14 @@ if [[ "$${POLICY}" == *"$${NOT_FOUND_MSG}"* ]]; then
     exit 0
 fi
 
-echo $${POLICY} | grep '"expvar_enabled":{"type":"bool","value":true}' > /dev/null 2>&1 && EXPVAR_ENABLED=true
-echo $${POLICY} | grep '"pprof_enabled":{"type":"bool","value":true}' > /dev/null 2>&1 && PPROF_ENABLED=true
-if [[ $${EXPVAR_ENABLED} || $${PPROF_ENABLED} ]] ; then
-    echo "expvar or pprof already enabled"
-    exit 0
-fi
-
 # Download and modify the APM policy
 echo $${POLICY} | jq '.item' | \
     jq 'del(.id)' | jq 'del(.elasticsearch)'| jq 'del(.inputs[].compiled_input)' | jq 'del(.revision)' |\
     jq 'del(.created_at)' | jq 'del(.created_by)' | jq 'del(.updated_at)' | jq 'del(.updated_by)' |\
     jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.expvar_enabled = {type: "bool", value: ${enable_expvar}}' |\
-    jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.pprof_enabled = {type: "bool", value: ${enable_pprof}}' > policy.json
+    jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.pprof_enabled = {type: "bool", value: ${enable_pprof}}' |\
+    jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.tail_sampling_storage_limit = {"value":"${tail_sampling_storage_limit}","type":"text"}' |\
+    jq 'select(.inputs[].policy_template == "apmserver").inputs[].vars.tail_sampling_enabled = {type: "bool", value: ${enable_tail_sampling}}' > policy.json
 
 # Update the policy
 curl -s -H 'content-type: application/json' -H 'kbn-xsrf: true' -X PUT -k [email protected] -u $${KIBANA_AUTH} $${KIBANA_ENDPOINT}

testing/infra/terraform/modules/ec_deployment/variables.tf

@@ -108,7 +108,7 @@ variable "docker_image" {
   description = "Optional docker image overrides. The full map needs to be specified"
 }
 
-# Enable APM Server's expvar
+# Enable APM Server's features
 
 variable "apm_server_expvar" {
   default     = false
@@ -122,6 +122,18 @@ variable "apm_server_pprof" {
   type        = bool
 }
 
+variable "apm_server_tail_sampling" {
+  default     = false
+  description = "Whether or not to enable APM Server tail-based sampling. Defaults to false"
+  type        = bool
+}
+
+variable "apm_server_tail_sampling_storage_limit" {
+  default     = "10GB"
+  description = "Storage size limit of APM Server tail-based sampling. Defaults to 10GB"
+  type        = string
+}
+
 variable "apm_index_shards" {
   default     = 0
   description = "The number of shards to set for APM Indices"