add functionaltests framework

Add utility packages to implement functional tests and Terraform
code to bootstrap a Elastic Cloud deployment.

Implement first test verifying upgrade from 8.15.4 to 8.16.0 for a
fresh Elastic Cloud deployment.

.github/workflows/functional-tests.yml

@@ -34,7 +34,7 @@ jobs:
           secrets: |-
             EC_API_KEY:elastic-observability/elastic-cloud-observability-team-${{ matrix.environment }}-api-key
 
-      - run: cd functionaltests && go test -v ./
+      - run: cd functionaltests && go test -v -timeout=20m -target "${{ matrix.environment }}" ./
 
   notify:
     if: always()

functionaltests/8_15_test.go

@@ -0,0 +1,125 @@
+package functionaltests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/elastic/apm-server/functionaltests/internal/esclient"
+	"github.com/elastic/apm-server/functionaltests/internal/terraform"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpgrade_8_15_4_to_8_16_0(t *testing.T) {
+	require.NoError(t, ecAPICheck(t))
+
+	start := time.Now()
+	ctx := context.Background()
+
+	t.Log("creating deploment with terraform")
+	tf, err := terraform.New(t, t.Name())
+	require.NoError(t, err)
+	ecTarget := terraform.Var("ec_target", *target)
+	version := terraform.Var("stack_version", "8.15.4")
+	name := terraform.Var("name", t.Name())
+	require.NoError(t, tf.Apply(ctx, ecTarget, version, name))
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Cleanup(func() {
+		if !t.Failed() || (t.Failed() && *cleanupOnFailure) {
+			t.Log("cleanup terraform resources")
+			require.NoError(t, tf.Destroy(ctx, ecTarget, name, version))
+		} else {
+			t.Log("test failed and cleanup-on-failure is false, skipping cleanup")
+		}
+	})
+
+	var deploymentID string
+	var escfg esclient.Config
+	tf.Output("deployment_id", &deploymentID)
+	tf.Output("apm_url", &escfg.APMServerURL)
+	tf.Output("es_url", &escfg.ElasticsearchURL)
+	tf.Output("username", &escfg.Username)
+	tf.Output("password", &escfg.Password)
+	tf.Output("kb_url", &escfg.KibanaURL)
+
+	t.Logf("created deployment %s", deploymentID)
+
+	ac, err := esclient.New(escfg)
+	require.NoError(t, err)
+
+	t.Log("creating APM API key")
+	apikey, err := ac.CreateAPMAPIKey(ctx, t.Name())
+	require.NoError(t, err)
+
+	ingest(t, escfg.APMServerURL, apikey)
+
+	// Wait few seconds before proceeding to ensure ES indexed all our documents.
+	// Manual tests had failures due to only 4 data streams being reported
+	// when no delay was used. Manual inspection always revealed the correct
+	// number of data streams.
+	time.Sleep(1 * time.Minute)
+
+	oldCount, err := ac.ApmDocCount(ctx)
+	require.NoError(t, err)
+
+	t.Log("check data streams")
+	var dss []types.DataStream
+	dss, err = ac.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     1,
+		IndicesManagedBy: []string{"Data stream lifecycle"},
+	}, dss)
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Log("upgrade to 8.16.0")
+	require.NoError(t, tf.Apply(ctx, ecTarget, name, terraform.Var("stack_version", "8.16.0")))
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	t.Log("check number of documents")
+	newCount, err := ac.ApmDocCount(ctx)
+	require.NoError(t, err)
+	assertDocCountEqual(t, oldCount, newCount)
+
+	t.Log("check data streams after upgrade, no rollover expected")
+	dss, err = ac.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     1,
+		IndicesManagedBy: []string{"Data stream lifecycle"},
+	}, dss)
+
+	ingest(t, escfg.APMServerURL, apikey)
+	time.Sleep(1 * time.Minute)
+
+	t.Log("check number of documents")
+	newCount2, err := ac.ApmDocCount(ctx)
+	require.NoError(t, err)
+	assertDocCountGreaterThan(t, oldCount, newCount2)
+
+	// Confirm datastreams are
+	// v managed by DSL if created after 8.15.0
+	// x managed by ILM if created before 8.15.0
+	t.Log("check data streams and verify lazy rollover happened")
+	dss2, err := ac.GetDataStream(ctx, "*apm*")
+	require.NoError(t, err)
+	assertDatastreams(t, checkDatastreamWant{
+		Quantity:         8,
+		PreferIlm:        false,
+		DSManagedBy:      "Data stream lifecycle",
+		IndicesPerDs:     2,
+		IndicesManagedBy: []string{"Data stream lifecycle", "Data stream lifecycle"},
+	}, dss2)
+	t.Logf("time elapsed: %s", time.Now().Sub(start))
+
+	// check ES logs, there should be no errors
+	// TODO: how to get these from Elastic Cloud? Is it possible?
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/main.tf

@@ -0,0 +1,27 @@
+resource "ec_deployment" "example_minimal" {
+  name = var.name
+
+  region                 = "aws-eu-west-1"
+  version                = var.stack_version
+  deployment_template_id = "aws-storage-optimized"
+
+  elasticsearch = {
+    hot = {
+      autoscaling = {}
+    }
+
+    ml = {
+      autoscaling = {
+        autoscale = true
+      }
+    }
+  }
+
+  kibana = {
+    topology = {}
+  }
+
+  integrations_server = {}
+
+  tags = merge(local.ci_tags, module.tags.tags)
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/outputs.tf

@@ -0,0 +1,23 @@
+output "deployment_id" {
+  value = ec_deployment.example_minimal.id
+}
+
+output "apm_url" {
+  value = ec_deployment.example_minimal.integrations_server.endpoints.apm
+}
+
+output "es_url" {
+  value = ec_deployment.example_minimal.elasticsearch.https_endpoint
+}
+
+output "username" {
+  value = ec_deployment.example_minimal.elasticsearch_username
+}
+output "password" {
+  value     = ec_deployment.example_minimal.elasticsearch_password
+  sensitive = true
+}
+
+output "kb_url" {
+  value = ec_deployment.example_minimal.kibana.https_endpoint
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/tags.tf

@@ -0,0 +1,20 @@
+resource "time_static" "created_date" {}
+
+locals {
+  ci_tags = {
+    environment  = var.ENVIRONMENT
+    repo         = var.REPO
+    branch       = var.BRANCH
+    build        = var.BUILD_ID
+    created_date = coalesce(var.CREATED_DATE, time_static.created_date.unix)
+  }
+  project = "apm-server-functionaltest"
+}
+
+module "tags" {
+  source = "../../testing/infra/terraform/modules/tags"
+  # use the convention for team/shared owned resources if we are running in CI. 
+  # assume this is an individually owned resource otherwise. 
+  project = local.project
+}
+

functionaltests/TestUpgrade_8_15_4_to_8_16_0/terraform.tf

@@ -0,0 +1,21 @@
+terraform {
+  required_version = ">= 0.12.29"
+
+  required_providers {
+    ec = {
+      source  = "elastic/ec"
+      version = "0.12.2"
+    }
+  }
+}
+
+locals {
+  api_endpoints = {
+    qa  = "https://public-api.qa.cld.elstc.co"
+    pro = "https://api.elastic-cloud.com"
+  }
+}
+
+provider "ec" {
+  endpoint = local.api_endpoints[var.ec_target]
+}

functionaltests/TestUpgrade_8_15_4_to_8_16_0/vars.tf

@@ -0,0 +1,42 @@
+variable "ec_target" {
+  type        = string
+  description = "The Elastic Cloud environment to target"
+  validation {
+    condition     = contains(["qa", "pro"], var.ec_target)
+    error_message = "Valid values are (qa, pro)."
+  }
+}
+
+variable "name" {
+  type        = string
+  description = "The deployment name"
+}
+
+variable "stack_version" {
+  type        = string
+  description = "The Elasticsearch version to bootstrap"
+}
+
+# CI variables
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default     = "unknown-branch"
+}
+
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default     = "unknown-build"
+}
+
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default     = ""
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo-name"
+}

functionaltests/go.mod

@@ -0,0 +1,42 @@
+module github.com/elastic/apm-server/functionaltests
+
+go 1.23.2
+
+// FIXME: use apm-perf from main after https://github.com/elastic/apm-perf/pull/197 is merged
+require (
+	github.com/elastic/apm-perf v0.0.0-20241213094810-d1d7602614f5
+	github.com/elastic/go-elasticsearch/v8 v8.16.0
+	github.com/hashicorp/terraform-exec v0.21.0
+	github.com/stretchr/testify v1.10.0
+	go.uber.org/zap v1.27.0
+)
+
+require (
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/hashicorp/terraform-json v0.22.1 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
+	github.com/zclconf/go-cty v1.14.4 // indirect
+	go.elastic.co/apm/v2 v2.6.2 // indirect
+	go.elastic.co/fastjson v1.4.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)