Merge branch 'main' into test-agentcfg-scroll-id

elastic · Sep 3, 2024 · 0d120e1 · 0d120e1
2 parents 9f4ce6e + ecffa8e
commit 0d120e1
Show file tree

Hide file tree

Showing 18 changed files with 115 additions and 91 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -170,7 +170,7 @@ jobs:
 
       # Notify result to Slack only on schedule (nightly run)
       - if: github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
+        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         with:

diff --git a/.github/workflows/bump-elastic-stack.yml b/.github/workflows/bump-elastic-stack.yml
@@ -17,7 +17,7 @@ jobs:
       matrix: ${{ steps.generator.outputs.matrix }}
     steps:
       - id: generator
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
 
   bump-elastic-stack:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml
@@ -19,7 +19,7 @@ jobs:
     steps:
       - id: generate
         name: Generate matrix
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
         with:
           exclude-branches: '7.17,main'
       - uses: actions/github-script@v7
@@ -70,9 +70,9 @@ jobs:
     if: always()
     steps:
       - id: check
-        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
         with:
-          needs: ${{ toJSON(needs) }}
+          jobs: ${{ toJSON(needs) }}
       - if: ${{ steps.check.outputs.isSuccess == 'false' }}
         uses: elastic/oblt-actions/slack/send@v1
         with:

diff --git a/.github/workflows/check-docker-compose.yml b/.github/workflows/check-docker-compose.yml
@@ -17,7 +17,7 @@ jobs:
       matrix: ${{ steps.generator.outputs.matrix }}
     steps:
       - id: generator
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
 
   check-docker-compose:
     needs:
@@ -47,9 +47,9 @@ jobs:
       - check-docker-compose
     steps:
       - id: check
-        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
         with:
-          needs: ${{ toJSON(needs) }}
+          jobs: ${{ toJSON(needs) }}
       - run: ${{ steps.check.outputs.isSuccess }}
       - if: failure()
         uses: elastic/oblt-actions/slack/notify-result@v1

diff --git a/.github/workflows/smoke-tests-schedule.yml b/.github/workflows/smoke-tests-schedule.yml
@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v4
       - id: generate
         name: Generate matrix
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
         with:
           exclude-branches: '7.17'
 
@@ -57,9 +57,9 @@ jobs:
       - smoke-tests-ess
     steps:
       - id: check
-        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
         with:
-          needs: ${{ toJSON(needs) }}
+          jobs: ${{ toJSON(needs) }}
       - uses: elastic/oblt-actions/slack/[email protected]
         with:
           bot-token: ${{ secrets.SLACK_BOT_TOKEN }}

diff --git a/.github/workflows/update-beats.yml b/.github/workflows/update-beats.yml
@@ -16,7 +16,7 @@ jobs:
       matrix: ${{ steps.generator.outputs.matrix }}
     steps:
       - id: generator
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
   bump:
     needs:
       - filter

diff --git a/.github/workflows/update-compose.yml b/.github/workflows/update-compose.yml
@@ -18,7 +18,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+      - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -3836,11 +3836,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/module/apmelasticsearch/v2
-Version: v2.6.0
+Version: v2.6.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasticsearch/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasticsearch/[email protected].2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -4047,11 +4047,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasti
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/module/apmgorilla/v2
-Version: v2.6.0
+Version: v2.6.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmgorilla/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmgorilla/[email protected].2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -4258,11 +4258,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmgorill
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/module/apmgrpc/v2
-Version: v2.6.0
+Version: v2.6.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmgrpc/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmgrpc/[email protected].2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -4469,11 +4469,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmgrpc/v
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/module/apmhttp/v2
-Version: v2.6.0
+Version: v2.6.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/[email protected].2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -4680,11 +4680,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/v
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/module/apmotel/v2
-Version: v2.6.0
+Version: v2.6.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmotel/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmotel/[email protected].2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -4891,11 +4891,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmotel/v
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/v2
-Version: v2.6.0
+Version: v2.6.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/[email protected].2/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/changelogs/8.15.asciidoc b/changelogs/8.15.asciidoc
@@ -1,7 +1,33 @@
 [[apm-release-notes-8.15]]
 == APM version 8.15
+
+* <<apm-release-notes-8.15.1>>
 * <<apm-release-notes-8.15.0>>
 
+[float]
+[[apm-release-notes-8.15.1]]
+=== APM version 8.15.1
+
+https://github.com/elastic/apm-server/compare/v8.15.0\...v8.15.1[View commits]
+
+[float]
+==== Bug fixes
+
+- Fix fallback memory limit check conversion from Bytes to Gigabytes {pull}13838[13838]
+- Fix Elasticsearch apm-data plugin to fallback to ILM policy for data streams using default APM integration and not explicitly migrated to data stream lifecycle. More details are available in the https://www.elastic.co/guide/en/observability/current/apm-known-issues.html[APM known issues page] {pull}13918[13918]
+
+[float]
+==== Breaking Changes
+
+[float]
+==== Deprecations
+
+[float]
+==== Intake API Changes
+
+[float]
+==== Added
+
 [float]
 [[apm-release-notes-8.15.0]]
 === APM version 8.15.0

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,7 +10,7 @@ x-logging: &default-logging
     max-size: "1g"
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-32bc94b7-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0-a3caaa65-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -41,7 +41,7 @@ services:
     logging: *default-logging
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.16.0-32bc94b7-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.16.0-a3caaa65-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -60,7 +60,7 @@ services:
     logging: *default-logging
 
   metricbeat:
-    image: docker.elastic.co/beats/metricbeat:8.16.0-32bc94b7-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:8.16.0-a3caaa65-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

diff --git a/go.mod b/go.mod
@@ -34,12 +34,12 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
-	go.elastic.co/apm/module/apmelasticsearch/v2 v2.6.0
-	go.elastic.co/apm/module/apmgorilla/v2 v2.6.0
-	go.elastic.co/apm/module/apmgrpc/v2 v2.6.0
-	go.elastic.co/apm/module/apmhttp/v2 v2.6.0
-	go.elastic.co/apm/module/apmotel/v2 v2.6.0
-	go.elastic.co/apm/v2 v2.6.0
+	go.elastic.co/apm/module/apmelasticsearch/v2 v2.6.2
+	go.elastic.co/apm/module/apmgorilla/v2 v2.6.2
+	go.elastic.co/apm/module/apmgrpc/v2 v2.6.2
+	go.elastic.co/apm/module/apmhttp/v2 v2.6.2
+	go.elastic.co/apm/module/apmotel/v2 v2.6.2
+	go.elastic.co/apm/v2 v2.6.2
 	go.elastic.co/fastjson v1.4.0
 	go.opentelemetry.io/collector/consumer v0.108.1
 	go.opentelemetry.io/collector/pdata v1.14.1

diff --git a/go.sum b/go.sum
@@ -430,20 +430,20 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.elastic.co/apm/module/apmelasticsearch/v2 v2.6.0 h1:ukMcwyMaDXsS1dRK2qRYXT2AsfwaUy74TOOYCqkWJow=
-go.elastic.co/apm/module/apmelasticsearch/v2 v2.6.0/go.mod h1:YpfiTTrqX5LB/CKBwX89oDCBAxuLJTFv40gcfxJyehM=
-go.elastic.co/apm/module/apmgorilla/v2 v2.6.0 h1:vXMZPNFss4a9146sIuT4zxQemVpQjNUDIIS9RMOr9PE=
-go.elastic.co/apm/module/apmgorilla/v2 v2.6.0/go.mod h1:w3/W4oDZPWBye0McuIpM7d5xSQ1WoFnMmgmEIzCsxIk=
-go.elastic.co/apm/module/apmgrpc/v2 v2.6.0 h1:0HPYGtjjS6wUE/f7Pw7L4R7IvgptGlXY8qMKFifFZj0=
-go.elastic.co/apm/module/apmgrpc/v2 v2.6.0/go.mod h1:96P3OJkfJe3ZzkCHmFofu5VOWa4CAPRWyUiCt4NgWkk=
-go.elastic.co/apm/module/apmhttp/v2 v2.6.0 h1:s8UeNFQmVBCNd4eoz7KDD9rEFhQC0HeUFXz3z9gpAmQ=
-go.elastic.co/apm/module/apmhttp/v2 v2.6.0/go.mod h1:D0GLppLuI0Ddwvtl595GUxRgn6Z8L5KaDFVMv2H3GK0=
-go.elastic.co/apm/module/apmotel/v2 v2.6.0 h1:5z1/kH2FD/K8Yacl04plBy2YVW6cDPTCMNmM6zG4FJk=
-go.elastic.co/apm/module/apmotel/v2 v2.6.0/go.mod h1:jU0+sy3GUHvM70Pf99BXbxgSZyHBFMaBiMWxiNwaxng=
+go.elastic.co/apm/module/apmelasticsearch/v2 v2.6.2 h1:x5LIkBAIo2XT8nBqoAjPeJKUJC94le7D9aclBYP9uCw=
+go.elastic.co/apm/module/apmelasticsearch/v2 v2.6.2/go.mod h1:LkYiSaKsGns6yRqXKAMbFyz7Nk8ikPCD/5CGkgQOY/A=
+go.elastic.co/apm/module/apmgorilla/v2 v2.6.2 h1:/myBx0D/JiwTUjFkVFG3zXmDfGPfQjP/cg27qcBbdfU=
+go.elastic.co/apm/module/apmgorilla/v2 v2.6.2/go.mod h1:uONZzSIh/cKjQ2rZmINR1VXVOJDq5eWOzKrCY+bu00w=
+go.elastic.co/apm/module/apmgrpc/v2 v2.6.2 h1:jaQeR7uZx9kF4JpMMwAxxyMu6xqWErt4poyqUq8TUrg=
+go.elastic.co/apm/module/apmgrpc/v2 v2.6.2/go.mod h1:AXDoIt5+0FWl8w2ev/M+dSU4J//SMS6V20wdiiUWPLE=
+go.elastic.co/apm/module/apmhttp/v2 v2.6.2 h1:+aYtP1Lnrsm+XtEs87RWG2PAyU6LHDDnYnJl3Lth0Qk=
+go.elastic.co/apm/module/apmhttp/v2 v2.6.2/go.mod h1:vlH+vXHaEijKK4pk605LOK+lbLDKwcByhlq4J24PeXw=
+go.elastic.co/apm/module/apmotel/v2 v2.6.2 h1:48hxUEPRwwcHpsDSde5P5a05ut6gTkq3oO1c6HrTjMI=
+go.elastic.co/apm/module/apmotel/v2 v2.6.2/go.mod h1:ZvyYDHJi49W42XhxmAYyDVtJhiEIaeB+RBgoHgzigM4=
 go.elastic.co/apm/module/apmzap/v2 v2.6.0 h1:R/iVORzGu3F9uM43iEVHD0nwiRo59O0bIXdayKsgayQ=
 go.elastic.co/apm/module/apmzap/v2 v2.6.0/go.mod h1:B3i/8xRkqLgi6zNuV+Bp7Pt4cutaOObvrVSa7wUTAPw=
-go.elastic.co/apm/v2 v2.6.0 h1:VieBMLQFtXua2YxpYxaSdYGnmmxhLT46gosI5yErJgY=
-go.elastic.co/apm/v2 v2.6.0/go.mod h1:33rOXgtHwbgZcDgi6I/GtCSMZQqgxkHC0IQT3gudKvo=
+go.elastic.co/apm/v2 v2.6.2 h1:VBplAxgbOgTv+Giw/FS91xJpHYw/q8fz/XKPvqC+7/o=
+go.elastic.co/apm/v2 v2.6.2/go.mod h1:33rOXgtHwbgZcDgi6I/GtCSMZQqgxkHC0IQT3gudKvo=
 go.elastic.co/ecszap v1.0.2 h1:iW5OGx8IiokiUzx/shD4AJCPFMC9uUtr7ycaiEIU++I=
 go.elastic.co/ecszap v1.0.2/go.mod h1:dJkSlK3BTiwG/qXhCwe50Mz/jwu854vSip8sIeQhNZg=
 go.elastic.co/fastjson v1.4.0 h1:a4BXUKXZHAzjVOPrqtEx2FDsIRBCMek01vCnrtyutWs=

diff --git a/internal/agentcfg/elasticsearch.go b/internal/agentcfg/elasticsearch.go
@@ -251,40 +251,38 @@ func (f *ElasticsearchFetcher) refreshCache(ctx context.Context) (err error) {
 
 func (f *ElasticsearchFetcher) singlePageRefresh(ctx context.Context, scrollID string) (cacheResult, error) {
 	var result cacheResult
+	var err error
+	var resp *esapi.Response
 
-	if scrollID == "" {
-		resp, err := esapi.SearchRequest{
+	switch scrollID {
+	case "":
+		resp, err = esapi.SearchRequest{
 			Index:  []string{ElasticsearchIndexName},
 			Size:   &f.searchSize,
 			Scroll: f.cacheDuration,
 		}.Do(ctx, f.client)
-		if err != nil {
-			return result, err
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode >= http.StatusBadRequest {
-			// Elasticsearch returns 401 on unauthorized requests and 403 on insufficient permission
-			if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
-				f.invalidESCfg.Store(true)
-			}
-			bodyBytes, err := io.ReadAll(resp.Body)
-			if err == nil {
-				f.logger.Debugf("refresh cache elasticsearch returned status %d: %s", resp.StatusCode, string(bodyBytes))
-			}
-			return result, fmt.Errorf("refresh cache elasticsearch returned status %d", resp.StatusCode)
-		}
-		return result, json.NewDecoder(resp.Body).Decode(&result)
+	default:
+		resp, err = esapi.ScrollRequest{
+			ScrollID: scrollID,
+			Scroll:   f.cacheDuration,
+		}.Do(ctx, f.client)
 	}
-
-	resp, err := esapi.ScrollRequest{
-		ScrollID: result.ScrollID,
-		Scroll:   f.cacheDuration,
-	}.Do(ctx, f.client)
 	if err != nil {
 		return result, err
 	}
 	defer resp.Body.Close()
+
+	if resp.StatusCode >= http.StatusBadRequest {
+		// Elasticsearch returns 401 on unauthorized requests and 403 on insufficient permission
+		if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+			f.invalidESCfg.Store(true)
+		}
+		bodyBytes, err := io.ReadAll(resp.Body)
+		if err == nil {
+			f.logger.Debugf("refresh cache elasticsearch returned status %d: %s", resp.StatusCode, string(bodyBytes))
+		}
+		return result, fmt.Errorf("refresh cache elasticsearch returned status %d", resp.StatusCode)
+	}
 	return result, json.NewDecoder(resp.Body).Decode(&result)
 }
 

diff --git a/systemtest/go.mod b/systemtest/go.mod
@@ -3,22 +3,22 @@ module github.com/elastic/apm-server/systemtest
 go 1.23.0
 
 require (
-	github.com/docker/docker v27.1.2+incompatible
+	github.com/docker/docker v27.2.0+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/elastic/apm-perf v0.0.0-20230608162138-29920c01cfd6
 	github.com/elastic/apm-tools v0.0.0-20240607105915-a4f490dc6959
-	github.com/elastic/go-elasticsearch/v8 v8.14.0
+	github.com/elastic/go-elasticsearch/v8 v8.15.0
 	github.com/elastic/go-lumber v0.1.1
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/jaegertracing/jaeger v1.60.0
 	github.com/stretchr/testify v1.9.0
 	github.com/testcontainers/testcontainers-go v0.33.0
 	github.com/tidwall/gjson v1.17.3
-	go.elastic.co/apm/v2 v2.6.0
+	go.elastic.co/apm/v2 v2.6.2
 	go.elastic.co/fastjson v1.4.0
-	go.opentelemetry.io/collector/pdata v1.13.0
-	go.opentelemetry.io/collector/semconv v0.107.0
+	go.opentelemetry.io/collector/pdata v1.14.1
+	go.opentelemetry.io/collector/semconv v0.108.1
 	go.opentelemetry.io/otel v1.29.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.29.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.29.0
@@ -33,7 +33,7 @@ require (
 	golang.org/x/sync v0.8.0
 	golang.org/x/sys v0.24.0
 	golang.org/x/time v0.6.0
-	google.golang.org/grpc v1.65.0
+	google.golang.org/grpc v1.66.0
 	gopkg.in/yaml.v3 v3.0.1
 )