From 848af21f33ed102041faad10b0694d807db55f6b Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Wed, 11 Dec 2024 18:56:41 +0100 Subject: [PATCH 1/4] add public telemetrygen package --- pkg/telemetrygen/config.go | 108 +++++++++++++++++++++++++++++ pkg/telemetrygen/doc.go | 23 ++++++ pkg/telemetrygen/generator.go | 81 ++++++++++++++++++++++ pkg/telemetrygen/generator_test.go | 57 +++++++++++++++ 4 files changed, 269 insertions(+) create mode 100644 pkg/telemetrygen/config.go create mode 100644 pkg/telemetrygen/doc.go create mode 100644 pkg/telemetrygen/generator.go create mode 100644 pkg/telemetrygen/generator_test.go diff --git a/pkg/telemetrygen/config.go b/pkg/telemetrygen/config.go new file mode 100644 index 0000000..a4fa101 --- /dev/null +++ b/pkg/telemetrygen/config.go @@ -0,0 +1,108 @@ +// licensed to elasticsearch b.v. under one or more contributor +// license agreements. see the notice file distributed with +// this work for additional information regarding copyright +// ownership. elasticsearch b.v. licenses this file to you under +// the apache license, version 2.0 (the "license"); you may +// not use this file except in compliance with the license. +// you may obtain a copy of the license at +// +// http://www.apache.org/licenses/license-2.0 +// +// unless required by applicable law or agreed to in writing, +// software distributed under the license is distributed on an +// "as is" basis, without warranties or conditions of any +// kind, either express or implied. see the license for the +// specific language governing permissions and limitations +// under the license. + +package telemetrygen + +import ( + "errors" + "fmt" + "net/url" + "strconv" + "strings" + "time" +) + +func DefaultConfig() Config { + return Config{ + // default to run 1 replica of 4 agents (one per type). + AgentReplicas: 1, + // default to expecting a valid TLS certificate. + Secure: true, + // default to 10 events per second + EventRate: RateFlag{Burst: 10, Interval: 1 * time.Second}, + // default to rewrite ids and timestamp to have new events recorded at the current time. + RewriteIDs: true, + RewriteTimestamps: true, + } +} + +type Config struct { + // number of agents replicas to use, each replica launches 4 agents, one for each type + AgentReplicas int + + ServerURL *url.URL + APIKey string + Headers map[string]string + Secure bool + EventRate RateFlag + IgnoreErrors bool + + RewriteIDs bool + RewriteTimestamps bool + RewriteServiceNames bool + RewriteServiceNodeNames bool + RewriteServiceTargetNames bool + RewriteSpanNames bool + RewriteTransactionNames bool + RewriteTransactionTypes bool +} + +func (c Config) Validate() error { + errs := []error{} + + if c.ServerURL == nil { + errs = append(errs, fmt.Errorf("ServerURL is required")) + } + + return errors.Join(errs...) +} + +type RateFlag struct { + Burst int + Interval time.Duration +} + +func (f *RateFlag) String() string { + return fmt.Sprintf("%d/%s", f.Burst, f.Interval) +} + +func (f *RateFlag) Set(s string) error { + before, after, ok := strings.Cut(s, "/") + if !ok || before == "" || after == "" { + return fmt.Errorf("invalid rate %q, expected format burst/duration", s) + } + + burst, err := strconv.Atoi(before) + if err != nil { + return fmt.Errorf("invalid burst %s in event rate: %w", before, err) + } + + if !(after[0] >= '0' && after[0] <= '9') { + after = "1" + after + } + interval, err := time.ParseDuration(after) + if err != nil { + return fmt.Errorf("invalid interval %q in event rate: %w", after, err) + } + if interval <= 0 { + return fmt.Errorf("invalid interval %q, must be positive", after) + } + + f.Burst = burst + f.Interval = interval + return nil +} diff --git a/pkg/telemetrygen/doc.go b/pkg/telemetrygen/doc.go new file mode 100644 index 0000000..9eeca1a --- /dev/null +++ b/pkg/telemetrygen/doc.go @@ -0,0 +1,23 @@ +// licensed to elasticsearch b.v. under one or more contributor +// license agreements. see the notice file distributed with +// this work for additional information regarding copyright +// ownership. elasticsearch b.v. licenses this file to you under +// the apache license, version 2.0 (the "license"); you may +// not use this file except in compliance with the license. +// you may obtain a copy of the license at +// +// http://www.apache.org/licenses/license-2.0 +// +// unless required by applicable law or agreed to in writing, +// software distributed under the license is distributed on an +// "as is" basis, without warranties or conditions of any +// kind, either express or implied. see the license for the +// specific language governing permissions and limitations +// under the license. + +// telemetrygen package provides an easy way to generate +// dummy APM events from a corpus of canned events. +// Is a public package that is expected to be used outside +// of this module, as such backward compatibility rules +// apply. +package telemetrygen diff --git a/pkg/telemetrygen/generator.go b/pkg/telemetrygen/generator.go new file mode 100644 index 0000000..519a32f --- /dev/null +++ b/pkg/telemetrygen/generator.go @@ -0,0 +1,81 @@ +package telemetrygen + +import ( + "context" + cryptorand "crypto/rand" + "encoding/binary" + "fmt" + "math/rand" + + "github.com/elastic/apm-perf/internal/loadgen" + "go.uber.org/zap" + + "golang.org/x/sync/errgroup" + "golang.org/x/time/rate" +) + +type Generator struct { + Config Config + Logger *zap.Logger +} + +func New(cfg Config) (*Generator, error) { + if err := cfg.Validate(); err != nil { + return &Generator{}, fmt.Errorf("cannot create generator, configuration is invalid: %w", err) + } + + return &Generator{Config: cfg, Logger: zap.NewNop()}, nil +} + +func (g *Generator) RunBlocking(ctx context.Context) error { + limiter := loadgen.GetNewLimiter(g.Config.EventRate.Burst, g.Config.EventRate.Interval) + errg, gCtx := errgroup.WithContext(ctx) + + var rngseed int64 + err := binary.Read(cryptorand.Reader, binary.LittleEndian, &rngseed) + if err != nil { + return fmt.Errorf("failed to generate seed for math/rand: %w", err) + } + + for i := 0; i < g.Config.AgentReplicas; i++ { + for _, expr := range []string{`apm-go*.ndjson`, `apm-nodejs*.ndjson`, `apm-python*.ndjson`, `apm-ruby*.ndjson`} { + expr := expr + errg.Go(func() error { + rng := rand.New(rand.NewSource(rngseed)) + return runAgent(gCtx, g.Logger, expr, limiter, rng, g.Config) + }) + } + } + + return errg.Wait() +} + +func runAgent(ctx context.Context, l *zap.Logger, expr string, limiter *rate.Limiter, rng *rand.Rand, cfg Config) error { + handler, err := loadgen.NewEventHandler(loadgen.EventHandlerParams{ + Logger: l, + URL: cfg.ServerURL.String(), + Path: expr, + APIKey: cfg.APIKey, + Limiter: limiter, + Rand: rng, + RewriteIDs: cfg.RewriteIDs, + RewriteServiceNames: cfg.RewriteServiceNames, + RewriteServiceNodeNames: cfg.RewriteServiceNodeNames, + RewriteServiceTargetNames: cfg.RewriteServiceTargetNames, + RewriteSpanNames: cfg.RewriteSpanNames, + RewriteTransactionNames: cfg.RewriteTransactionNames, + RewriteTransactionTypes: cfg.RewriteTransactionTypes, + RewriteTimestamps: cfg.RewriteTimestamps, + Headers: cfg.Headers, + Protocol: "apm/http", + }) + if err != nil { + return err + } + + if _, err := handler.SendBatches(ctx); err != nil { + return err + } + + return nil +} diff --git a/pkg/telemetrygen/generator_test.go b/pkg/telemetrygen/generator_test.go new file mode 100644 index 0000000..9530f28 --- /dev/null +++ b/pkg/telemetrygen/generator_test.go @@ -0,0 +1,57 @@ +package telemetrygen_test + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + + "github.com/elastic/apm-perf/pkg/telemetrygen" + "github.com/stretchr/testify/require" +) + +func TestGeneration(t *testing.T) { + srv, m := newTestServer(t, testServerConfig{http.StatusServiceUnavailable}) + + cfg := telemetrygen.DefaultConfig() + cfg.Secure = false + + u, err := url.Parse(srv.URL) + cfg.ServerURL = u + + cfg.EventRate.Set("1000/s") + g, err := telemetrygen.New(cfg) + // g.Logger = zap.Must(zap.NewDevelopment()) + require.NoError(t, err) + + err = g.RunBlocking(context.Background()) + require.NoError(t, err) + require.Greater(t, m.Load(), int32(0)) +} + +type testServerConfig struct { + responseStatus int +} +type metrics struct { + Received *atomic.Int32 +} + +func newTestServer(t *testing.T, cfg testServerConfig) (*httptest.Server, *atomic.Int32) { + t.Helper() + + mux := http.NewServeMux() + requestsReceived := &atomic.Int32{} + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + requestsReceived.Add(1) + w.Write([]byte("ok")) + return + }) + + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + return srv, requestsReceived +} From d4842d42b0164fafd571a9fbb4cf060f6373f431 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Fri, 13 Dec 2024 10:36:52 +0100 Subject: [PATCH 2/4] fix godoc --- pkg/telemetrygen/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/telemetrygen/doc.go b/pkg/telemetrygen/doc.go index 9eeca1a..e88b877 100644 --- a/pkg/telemetrygen/doc.go +++ b/pkg/telemetrygen/doc.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the license. -// telemetrygen package provides an easy way to generate +// Package telemetrygen provides an easy way to generate // dummy APM events from a corpus of canned events. // Is a public package that is expected to be used outside // of this module, as such backward compatibility rules From 979ba9db03f97042457bd0875e1d5c2d651d44da Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Fri, 13 Dec 2024 10:37:02 +0100 Subject: [PATCH 3/4] fix lint --- pkg/telemetrygen/generator_test.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/telemetrygen/generator_test.go b/pkg/telemetrygen/generator_test.go index 9530f28..b2d7952 100644 --- a/pkg/telemetrygen/generator_test.go +++ b/pkg/telemetrygen/generator_test.go @@ -13,12 +13,13 @@ import ( ) func TestGeneration(t *testing.T) { - srv, m := newTestServer(t, testServerConfig{http.StatusServiceUnavailable}) + srv, m := newTestServer(t) cfg := telemetrygen.DefaultConfig() cfg.Secure = false u, err := url.Parse(srv.URL) + require.NoError(t, err) cfg.ServerURL = u cfg.EventRate.Set("1000/s") @@ -31,14 +32,7 @@ func TestGeneration(t *testing.T) { require.Greater(t, m.Load(), int32(0)) } -type testServerConfig struct { - responseStatus int -} -type metrics struct { - Received *atomic.Int32 -} - -func newTestServer(t *testing.T, cfg testServerConfig) (*httptest.Server, *atomic.Int32) { +func newTestServer(t *testing.T) (*httptest.Server, *atomic.Int32) { t.Helper() mux := http.NewServeMux() @@ -47,7 +41,6 @@ func newTestServer(t *testing.T, cfg testServerConfig) (*httptest.Server, *atomi mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { requestsReceived.Add(1) w.Write([]byte("ok")) - return }) srv := httptest.NewServer(mux) From d1d7602614f5fcc15ab60d745aad290f569c6134 Mon Sep 17 00:00:00 2001 From: Edoardo Tenani Date: Fri, 13 Dec 2024 10:48:10 +0100 Subject: [PATCH 4/4] update license and goimports --- pkg/telemetrygen/config.go | 19 +++---------------- pkg/telemetrygen/doc.go | 19 +++---------------- pkg/telemetrygen/generator.go | 7 ++++++- pkg/telemetrygen/generator_test.go | 7 ++++++- 4 files changed, 18 insertions(+), 34 deletions(-) diff --git a/pkg/telemetrygen/config.go b/pkg/telemetrygen/config.go index a4fa101..5ee9d0a 100644 --- a/pkg/telemetrygen/config.go +++ b/pkg/telemetrygen/config.go @@ -1,19 +1,6 @@ -// licensed to elasticsearch b.v. under one or more contributor -// license agreements. see the notice file distributed with -// this work for additional information regarding copyright -// ownership. elasticsearch b.v. licenses this file to you under -// the apache license, version 2.0 (the "license"); you may -// not use this file except in compliance with the license. -// you may obtain a copy of the license at -// -// http://www.apache.org/licenses/license-2.0 -// -// unless required by applicable law or agreed to in writing, -// software distributed under the license is distributed on an -// "as is" basis, without warranties or conditions of any -// kind, either express or implied. see the license for the -// specific language governing permissions and limitations -// under the license. +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. package telemetrygen diff --git a/pkg/telemetrygen/doc.go b/pkg/telemetrygen/doc.go index e88b877..bcd9558 100644 --- a/pkg/telemetrygen/doc.go +++ b/pkg/telemetrygen/doc.go @@ -1,19 +1,6 @@ -// licensed to elasticsearch b.v. under one or more contributor -// license agreements. see the notice file distributed with -// this work for additional information regarding copyright -// ownership. elasticsearch b.v. licenses this file to you under -// the apache license, version 2.0 (the "license"); you may -// not use this file except in compliance with the license. -// you may obtain a copy of the license at -// -// http://www.apache.org/licenses/license-2.0 -// -// unless required by applicable law or agreed to in writing, -// software distributed under the license is distributed on an -// "as is" basis, without warranties or conditions of any -// kind, either express or implied. see the license for the -// specific language governing permissions and limitations -// under the license. +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. // Package telemetrygen provides an easy way to generate // dummy APM events from a corpus of canned events. diff --git a/pkg/telemetrygen/generator.go b/pkg/telemetrygen/generator.go index 519a32f..a88a55e 100644 --- a/pkg/telemetrygen/generator.go +++ b/pkg/telemetrygen/generator.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + package telemetrygen import ( @@ -7,9 +11,10 @@ import ( "fmt" "math/rand" - "github.com/elastic/apm-perf/internal/loadgen" "go.uber.org/zap" + "github.com/elastic/apm-perf/internal/loadgen" + "golang.org/x/sync/errgroup" "golang.org/x/time/rate" ) diff --git a/pkg/telemetrygen/generator_test.go b/pkg/telemetrygen/generator_test.go index b2d7952..045297e 100644 --- a/pkg/telemetrygen/generator_test.go +++ b/pkg/telemetrygen/generator_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + package telemetrygen_test import ( @@ -8,8 +12,9 @@ import ( "sync/atomic" "testing" - "github.com/elastic/apm-perf/pkg/telemetrygen" "github.com/stretchr/testify/require" + + "github.com/elastic/apm-perf/pkg/telemetrygen" ) func TestGeneration(t *testing.T) {