From ecffa8e88d1d76831489e49e6bcd542749851eae Mon Sep 17 00:00:00 2001 From: up2neck <163534172+up2neck@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:50:33 +0300 Subject: [PATCH 1/3] Fix agentcfg cache renewal scroll (#13958) APM Agents configuration cache appears to be broken, resulting frequent invalid requests made by APM Server to Elasticsearch cluster. Fix ScrollID used for retrieval of APM Agents configuration. --- internal/agentcfg/elasticsearch.go | 46 ++++++++++++++---------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/internal/agentcfg/elasticsearch.go b/internal/agentcfg/elasticsearch.go index b217882bfe7..066b9374817 100644 --- a/internal/agentcfg/elasticsearch.go +++ b/internal/agentcfg/elasticsearch.go @@ -251,40 +251,38 @@ func (f *ElasticsearchFetcher) refreshCache(ctx context.Context) (err error) { func (f *ElasticsearchFetcher) singlePageRefresh(ctx context.Context, scrollID string) (cacheResult, error) { var result cacheResult + var err error + var resp *esapi.Response - if scrollID == "" { - resp, err := esapi.SearchRequest{ + switch scrollID { + case "": + resp, err = esapi.SearchRequest{ Index: []string{ElasticsearchIndexName}, Size: &f.searchSize, Scroll: f.cacheDuration, }.Do(ctx, f.client) - if err != nil { - return result, err - } - defer resp.Body.Close() - - if resp.StatusCode >= http.StatusBadRequest { - // Elasticsearch returns 401 on unauthorized requests and 403 on insufficient permission - if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { - f.invalidESCfg.Store(true) - } - bodyBytes, err := io.ReadAll(resp.Body) - if err == nil { - f.logger.Debugf("refresh cache elasticsearch returned status %d: %s", resp.StatusCode, string(bodyBytes)) - } - return result, fmt.Errorf("refresh cache elasticsearch returned status %d", resp.StatusCode) - } - return result, json.NewDecoder(resp.Body).Decode(&result) + default: + resp, err = esapi.ScrollRequest{ + ScrollID: scrollID, + Scroll: f.cacheDuration, + }.Do(ctx, f.client) } - - resp, err := esapi.ScrollRequest{ - ScrollID: result.ScrollID, - Scroll: f.cacheDuration, - }.Do(ctx, f.client) if err != nil { return result, err } defer resp.Body.Close() + + if resp.StatusCode >= http.StatusBadRequest { + // Elasticsearch returns 401 on unauthorized requests and 403 on insufficient permission + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + f.invalidESCfg.Store(true) + } + bodyBytes, err := io.ReadAll(resp.Body) + if err == nil { + f.logger.Debugf("refresh cache elasticsearch returned status %d: %s", resp.StatusCode, string(bodyBytes)) + } + return result, fmt.Errorf("refresh cache elasticsearch returned status %d", resp.StatusCode) + } return result, json.NewDecoder(resp.Body).Decode(&result) } From 2aec4d1a2096f066e6a423b1469570bc1d092174 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Tue, 3 Sep 2024 16:54:20 +0100 Subject: [PATCH 2/3] Test agentcfg es fetcher scroll id (#13959) Follow up on #13958 to add test --- internal/agentcfg/elasticsearch_test.go | 29 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/internal/agentcfg/elasticsearch_test.go b/internal/agentcfg/elasticsearch_test.go index 51432f805e6..4671934c847 100644 --- a/internal/agentcfg/elasticsearch_test.go +++ b/internal/agentcfg/elasticsearch_test.go @@ -20,7 +20,6 @@ package agentcfg import ( "context" "encoding/json" - "io" "net/http" "net/http/httptest" "testing" @@ -39,11 +38,10 @@ var sampleHits = []map[string]interface{}{ {"_id": "hvKmzYQBfJ4l0GgqXgJt", "_index": ".apm-agent-configuration", "_score": 1, "_source": map[string]interface{}{"@timestamp": 1.669897543277e+12, "applied_by_agent": false, "etag": "2da2f86251165ccced5c5e41100a216b0c880db4", "service": map[string]interface{}{"name": "second"}, "settings": map[string]interface{}{"sanitize_field_names": "foo,bar,baz", "transaction_sample_rate": "0.1"}}}, } -func newMockElasticsearchClient(t testing.TB, statusCode int, responseFunc func(io.Writer)) *elasticsearch.Client { +func newMockElasticsearchClient(t testing.TB, handler func(http.ResponseWriter, *http.Request)) *elasticsearch.Client { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Elastic-Product", "Elasticsearch") - w.WriteHeader(statusCode) - responseFunc(w) + handler(w, r) })) t.Cleanup(srv.Close) config := elasticsearch.DefaultConfig() @@ -80,7 +78,15 @@ func newElasticsearchFetcher( i := 0 - fetcher := NewElasticsearchFetcher(newMockElasticsearchClient(t, 200, func(w io.Writer) { + fetcher := NewElasticsearchFetcher(newMockElasticsearchClient(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/_search/scroll": + scrollID := r.URL.Query().Get("scroll_id") + assert.Equal(t, respTmpl["_scroll_id"], scrollID) + case "/.apm-agent-configuration/_search": + default: + assert.Failf(t, "unexpected path", "path: %s", r.URL.Path) + } if i < len(hits) { respTmpl["hits"].(map[string]interface{})["hits"] = hits[i : i+searchSize] } else { @@ -89,6 +95,7 @@ func newElasticsearchFetcher( b, err := json.Marshal(respTmpl) require.NoError(t, err) + w.WriteHeader(200) w.Write(b) i += searchSize }), time.Second, nil, rt) @@ -173,7 +180,9 @@ func TestFetchUseFallback(t *testing.T) { return Result{}, nil }) fetcher := NewElasticsearchFetcher( - newMockElasticsearchClient(t, 404, func(w io.Writer) {}), + newMockElasticsearchClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + }), time.Second, fallbackFetcher, apmtest.NewRecordingTracer().Tracer, @@ -186,7 +195,9 @@ func TestFetchUseFallback(t *testing.T) { func TestFetchNoFallbackInvalidESCfg(t *testing.T) { fetcher := NewElasticsearchFetcher( - newMockElasticsearchClient(t, 401, func(w io.Writer) {}), + newMockElasticsearchClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(401) + }), time.Second, nil, apmtest.NewRecordingTracer().Tracer, @@ -200,7 +211,9 @@ func TestFetchNoFallbackInvalidESCfg(t *testing.T) { func TestFetchNoFallback(t *testing.T) { fetcher := NewElasticsearchFetcher( - newMockElasticsearchClient(t, 500, func(w io.Writer) {}), + newMockElasticsearchClient(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + }), time.Second, nil, apmtest.NewRecordingTracer().Tracer, From 583c26c0c64080254674cc2bd402cdcdf3819ab5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 19:12:13 +0000 Subject: [PATCH 3/3] build(deps): bump github.com/elastic/apm-data from 1.11.0 to 1.12.0 (#13988) * build(deps): bump github.com/elastic/apm-data from 1.11.0 to 1.12.0 Bumps [github.com/elastic/apm-data](https://github.com/elastic/apm-data) from 1.11.0 to 1.12.0. - [Release notes](https://github.com/elastic/apm-data/releases) - [Commits](https://github.com/elastic/apm-data/compare/v1.11.0...v1.12.0) --- updated-dependencies: - dependency-name: github.com/elastic/apm-data dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index d0dc6c90ee1..27a85c9e7c0 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -362,11 +362,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-data -Version: v1.11.0 +Version: v1.12.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.11.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.12.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 1dba6c193e9..2686db1f9d4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v1.0.2 - github.com/elastic/apm-data v1.11.0 + github.com/elastic/apm-data v1.12.0 github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b github.com/elastic/elastic-agent-client/v7 v7.15.0 github.com/elastic/elastic-agent-libs v0.10.0 diff --git a/go.sum b/go.sum index d2b620791ed..2d58361cb68 100644 --- a/go.sum +++ b/go.sum @@ -131,8 +131,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elastic/apm-aggregation v1.0.2 h1:nvplp5pUGHAvRf8M6G4+UAxx55i+TpEpIda02JE/DEU= github.com/elastic/apm-aggregation v1.0.2/go.mod h1:cprQyd9TAvUTmyqY/J1ZX4cwU88vbVP9uhNKdEC/dsg= -github.com/elastic/apm-data v1.11.0 h1:fbbKbsodp1meGnsI1uPKxNcjEOnV1rDb1Pjcox022Ts= -github.com/elastic/apm-data v1.11.0/go.mod h1:8ZgwhOPHxaLPb9I3YglGNYPWfoPsDkunhYOp3zDyRX0= +github.com/elastic/apm-data v1.12.0 h1:cvd519HNnoYYmOYCsBvEtF5i8uVgXWDk6HFbXHQvegU= +github.com/elastic/apm-data v1.12.0/go.mod h1:dkXFH/LWAf5DekU11/f0wvFLO2et34SZ8duJCbTm+y0= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b h1:i3WdOw//Vg76Vseqxgn47GPTVqUSucAdE42jkih1ViM= github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b/go.mod h1:m8d3EBr2z7YvWfxRgoE7SsUNbMTpyWXi67fG7B7vn7w= github.com/elastic/elastic-agent-autodiscover v0.8.1 h1:u6TWqh7wfevu6S4GUq4SIxYBRo4b/P5RZmx/rSvT10A=