Introduce a configurable default service environment (#4861)

* Add transform.Processor interface

* beater: add default service environment

* model/modelprocessor: update PprofProfile too

* apmpackage: update README about namespace

(cherry picked from commit 992699d)

# Conflicts:
#	apmpackage/apm/0.1.0/manifest.yml
#	apmpackage/apm/0.2.0/README.template.md
#	apmpackage/apm/0.2.0/docs/README.md
#	beater/api/config/agent/handler_test.go
#	beater/beater.go
#	beater/config/config.go
#	changelogs/head.asciidoc
#	model/modelprocessor/environment.go
#	model/modelprocessor/environment_test.go
#	transform/transform.go
#	x-pack/apm-server/main.go

apmpackage/apm/0.1.0/manifest.yml

@@ -0,0 +1,128 @@
+format_version: 1.0.0
+name: apm
+title: Elastic APM
+version: 0.1.0
+license: basic
+description: Ingest APM data
+type: integration
+categories:
+  - monitoring
+release: experimental # experimental / beta / ga
+conditions:
+  kibana.version: '^7.12.0'
+icons:
+  - src: /img/logo_apm.svg
+    title: APM Logo
+    size: 32x32
+    type: image/svg+xml
+policy_templates:
+- name: apmserver
+  title: Elastic APM Integration
+  description: Elastic APM Integration
+  multiple: false
+  inputs:
+  - type: apm
+    title: Collect application traces
+    description: Collect application traces
+    vars:
+      - name: host
+        type: string
+        title: Host
+        required: true
+        show_user: true
+        default: localhost:8200
+      - name: secret_token
+        type: string
+        title: Secret token for Agent authentication
+        required: false
+        show_user: true
+      - name: api_key_enabled
+        type: bool
+        title: API Key for Agent authentication
+        description: Enable API Key auth between APM Server and APM Agents.
+        required: false
+        show_user: true
+        default: false
+      - name: kibana_api_key
+        type: string
+        title: API Key for Central Configuration
+        required: false
+        description: API Key for APM central configuration feature. Enter as <Id>:<API Key>
+        show_user: true
+      - name: enable_rum
+        type: bool
+        title: Enable RUM
+        description: Enable Real User Monitoring (RUM).
+        required: true
+        show_user: true
+        default: false
+      - name: default_service_environment
+        type: string
+        title: Default Service Environment
+        description: Default service environment to record in events which have no service environment defined.
+        required: false
+        show_user: false
+      - name: rum_allow_origins
+        type: string
+        title: RUM - Origin Headers
+        description: Allowed Origin headers to be sent by User Agents.
+        multi: true
+        required: false
+        show_user: false
+        default: ['*']
+      - name: rum_allow_headers
+        type: string
+        title: RUM - Access-Control-Allow-Headers
+        description: Supported Access-Control-Allow-Headers in addition to "Content-Type", "Content-Encoding" and "Accept".
+        multi: true
+        required: false
+        show_user: false
+      - name: rum_response_headers
+        type: yaml
+        title: RUM - Custom HTTP response headers
+        description: Added to RUM responses, e.g. for security policy compliance.
+        required: false
+        show_user: false
+      - name: rum_event_rate_limit
+        type: int
+        title: RUM - Rate limit events per IP
+        description: Maximum number of events allowed per IP per second.
+        required: false
+        show_user: false
+        default: 300
+      - name: rum_event_rate_lru_size
+        type: int
+        title: RUM - Rate limit cache size
+        description: Number of unique IPs to be cached for the rate limiter.
+        required: false
+        show_user: false
+        default: 1000
+      - name: sourcemap_api_key
+        type: string
+        title: RUM - API Key for Sourcemaps
+        required: false
+        description: API Key for sourcemap feature. Enter as <Id>:<API Key>
+        show_user: false
+      - name: api_key_limit
+        type: int
+        title: Maximum number of API Keys for Agent authentication
+        description: Restrict number of unique API Keys per minute, used for auth between APM Agents and Server.
+        required: false
+        show_user: false
+        default: 100
+      - name: max_event_bytes
+        type: int
+        title: Maximum size per event (bytes)
+        required: false
+        show_user: false
+        default: 307200
+      - name: capture_personal_data
+        type: bool
+        title: Capture personal data
+        description: Capture personal data such as IP or User Agent.
+        required: false
+        show_user: false
+        default: true
+    template_path: template.yml.hbs
+owner:
+  github: elastic/apm-server

apmpackage/apm/0.2.0/README.template.md

@@ -20,13 +20,21 @@ on the same machines that your instrumented services run.
 #### Data Streams
 
 When using the APM integration, apm events are indexed into data streams. Data stream names contain the event type,
+<<<<<<< HEAD:apmpackage/apm/0.2.0/README.template.md
 service name, and a user-configurable namespace.
 
 There is no specific recommendation for what to use as a namespace; it is intentionally flexible.
 You might use the environment, like `production`, `testing`, or `development`,
 or you could namespace data by business unit. It is your choice.
 
 See [APM data streams](https://ela.st/apm-data-streams) for more information.
+=======
+the service name, and a user configurable namespace.
+
+There is no specific recommendaton for what to use as a namespace; it is intentionally flexible.
+You might use the environment (production, testing, development) as the namespace,
+or alternatively you could namespace data by business unit. It is your choice.
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861):apmpackage/apm/0.1.0/_dev/docs/README.template.md
 
 ## Compatibility and limitations
 

apmpackage/apm/0.2.0/docs/README.md

@@ -20,13 +20,21 @@ on the same machines that your instrumented services run.
 #### Data Streams
 
 When using the APM integration, apm events are indexed into data streams. Data stream names contain the event type,
+<<<<<<< HEAD:apmpackage/apm/0.2.0/docs/README.md
 service name, and a user-configurable namespace.
 
 There is no specific recommendation for what to use as a namespace; it is intentionally flexible.
 You might use the environment, like `production`, `testing`, or `development`,
 or you could namespace data by business unit. It is your choice.
 
 See [APM data streams](https://ela.st/apm-data-streams) for more information.
+=======
+the service name, and a user configurable namespace.
+
+There is no specific recommendaton for what to use as a namespace; it is intentionally flexible.
+You might use the environment (production, testing, development) as the namespace,
+or alternatively you could namespace data by business unit. It is your choice.
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861):apmpackage/apm/0.1.0/docs/README.md
 
 ## Compatibility and limitations
 

beater/api/config/agent/handler_test.go

@@ -180,14 +180,22 @@ func TestAgentConfigHandler(t *testing.T) {
 
 	for name, tc := range testcases {
 
+<<<<<<< HEAD
 		runTest := func(t *testing.T, expectedBody map[string]string, authorized bool) {
+=======
+		runTest := func(t *testing.T, expectedBody map[string]string, auth authorization.Authorization) {
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 			h := Handler(tc.kbClient, &cfg, "")
 			r := httptest.NewRequest(tc.method, target(tc.queryParams), nil)
 			for k, v := range tc.requestHeader {
 				r.Header.Set(k, v)
 			}
 			ctx, w := newRequestContext(r)
+<<<<<<< HEAD
 			ctx.AuthResult.Authorized = authorized
+=======
+			ctx.Authorization = auth
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 			h(ctx)
 
 			require.Equal(t, tc.respStatus, w.Code)

beater/beater.go

@@ -47,7 +47,10 @@ import (
 	"github.com/elastic/apm-server/elasticsearch"
 	"github.com/elastic/apm-server/ingest/pipeline"
 	logs "github.com/elastic/apm-server/log"
+<<<<<<< HEAD
 	"github.com/elastic/apm-server/model"
+=======
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 	"github.com/elastic/apm-server/model/modelprocessor"
 	"github.com/elastic/apm-server/publish"
 	"github.com/elastic/apm-server/sampling"
@@ -355,6 +358,20 @@ func (s *serverRunner) run() error {
 	// Send config to telemetry.
 	recordAPMServerConfig(s.config)
 
+<<<<<<< HEAD
+=======
+	runServer := runServer
+	if s.tracerServer != nil {
+		runServer = runServerWithTracerServer(runServer, s.tracerServer, s.tracer)
+	}
+	if s.wrapRunServer != nil {
+		// Wrap runServer function, enabling injection of
+		// behaviour into the processing/reporting pipeline.
+		runServer = s.wrapRunServer(runServer)
+	}
+	runServer = s.wrapRunServerWithPreprocessors(runServer)
+
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 	transformConfig, err := newTransformConfig(s.beat.Info, s.config)
 	if err != nil {
 		return err
@@ -418,12 +435,16 @@ func (s *serverRunner) run() error {
 }
 
 func (s *serverRunner) wrapRunServerWithPreprocessors(runServer RunServerFunc) RunServerFunc {
+<<<<<<< HEAD
 	processors := []model.BatchProcessor{
 		modelprocessor.SetSystemHostname{},
 		modelprocessor.SetServiceNodeName{},
 		// Set metricset.name for well-known agent metrics.
 		modelprocessor.SetMetricsetName{},
 	}
+=======
+	var processors []transform.Processor
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 	if s.config.DefaultServiceEnvironment != "" {
 		processors = append(processors, &modelprocessor.SetDefaultServiceEnvironment{
 			DefaultServiceEnvironment: s.config.DefaultServiceEnvironment,
@@ -627,13 +648,20 @@ func newSourcemapStore(beatInfo beat.Info, cfg *config.SourceMapping) (*sourcema
 }
 
 // WrapRunServerWithProcessors wraps runServer such that it wraps args.Reporter
+<<<<<<< HEAD
 // with a function that event batches are first passed through the given processors
 // in order.
 func WrapRunServerWithProcessors(runServer RunServerFunc, processors ...model.BatchProcessor) RunServerFunc {
+=======
+// with a function that transformables are first passed through the given
+// processors in order.
+func WrapRunServerWithProcessors(runServer RunServerFunc, processors ...transform.Processor) RunServerFunc {
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 	if len(processors) == 0 {
 		return runServer
 	}
 	return func(ctx context.Context, args ServerParams) error {
+<<<<<<< HEAD
 		processors = append(processors, args.BatchProcessor)
 		args.BatchProcessor = modelprocessor.Chained(processors)
 		return runServer(ctx, args)
@@ -650,3 +678,19 @@ func (p *reporterBatchProcessor) ProcessBatch(ctx context.Context, batch *model.
 	disableTracing, _ := ctx.Value(disablePublisherTracingKey{}).(bool)
 	return p.reporter(ctx, publish.PendingReq{Transformable: batch, Trace: !disableTracing})
 }
+=======
+		origReporter := args.Reporter
+		args.Reporter = func(ctx context.Context, req publish.PendingReq) error {
+			var err error
+			for _, p := range processors {
+				req.Transformables, err = p.ProcessTransformables(ctx, req.Transformables)
+				if err != nil {
+					return err
+				}
+			}
+			return origReporter(ctx, req)
+		}
+		return runServer(ctx, args)
+	}
+}
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)

beater/config/config.go

@@ -75,7 +75,10 @@ type Config struct {
 	MaxConnections            int                     `config:"max_connections"`
 	ResponseHeaders           map[string][]string     `config:"response_headers"`
 	Expvar                    *ExpvarConfig           `config:"expvar"`
+<<<<<<< HEAD
 	Pprof                     *PprofConfig            `config:"pprof"`
+=======
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 	AugmentEnabled            bool                    `config:"capture_personal_data"`
 	SelfInstrumentation       *InstrumentationConfig  `config:"instrumentation"`
 	RumConfig                 *RumConfig              `config:"rum"`

changelogs/head.asciidoc

@@ -0,0 +1,22 @@
+[[release-notes-head]]
+== APM Server version HEAD
+
+https://github.com/elastic/apm-server/compare/7.12\...master[View commits]
+
+[float]
+==== Breaking Changes
+
+[float]
+==== Bug fixes
+
+[float]
+==== Intake API Changes
+
+[float]
+==== Added
+* Add support for Node.js wall time profiles {pull}4728[4728]
+* Add metricset.name field to metric docs {pull}4857[4857]
+* Add `apm-server.default_service_environment` config {pull}4861[4861]
+
+[float]
+==== Deprecated

model/modelprocessor/environment.go

@@ -21,6 +21,10 @@ import (
 	"context"
 
 	"github.com/elastic/apm-server/model"
+<<<<<<< HEAD
+=======
+	"github.com/elastic/apm-server/transform"
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 )
 
 // SetDefaultServiceEnvironment is a transform.Processor that sets a default
@@ -31,6 +35,7 @@ type SetDefaultServiceEnvironment struct {
 	DefaultServiceEnvironment string
 }
 
+<<<<<<< HEAD
 // ProcessBatch sets a default service.value for events without one already set.
 func (s *SetDefaultServiceEnvironment) ProcessBatch(ctx context.Context, b *model.Batch) error {
 	return MetadataProcessorFunc(s.setDefaultServiceEnvironment).ProcessBatch(ctx, b)
@@ -41,4 +46,33 @@ func (s *SetDefaultServiceEnvironment) setDefaultServiceEnvironment(ctx context.
 		meta.Service.Environment = s.DefaultServiceEnvironment
 	}
 	return nil
+=======
+// ProcessTransformables sets a default service.value for events without one already set.
+func (s *SetDefaultServiceEnvironment) ProcessTransformables(ctx context.Context, in []transform.Transformable) ([]transform.Transformable, error) {
+	for _, t := range in {
+		switch t := t.(type) {
+		case *model.Transaction:
+			if t.Metadata.Service.Environment == "" {
+				t.Metadata.Service.Environment = s.DefaultServiceEnvironment
+			}
+		case *model.Span:
+			if t.Metadata.Service.Environment == "" {
+				t.Metadata.Service.Environment = s.DefaultServiceEnvironment
+			}
+		case *model.Metricset:
+			if t.Metadata.Service.Environment == "" {
+				t.Metadata.Service.Environment = s.DefaultServiceEnvironment
+			}
+		case *model.Error:
+			if t.Metadata.Service.Environment == "" {
+				t.Metadata.Service.Environment = s.DefaultServiceEnvironment
+			}
+		case *model.PprofProfile:
+			if t.Metadata.Service.Environment == "" {
+				t.Metadata.Service.Environment = s.DefaultServiceEnvironment
+			}
+		}
+	}
+	return in, nil
+>>>>>>> 992699dc8... Introduce a configurable default service environment (#4861)
 }