From c6d5b33b807747278472af3e49f67bb0bb24d8ab Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:48:18 +0100 Subject: [PATCH] feat: remove API Key CLI functionality (#14790) update systemtest to create apikey using api remove unused functions --- internal/beatcmd/apikey.go | 512 ------------------------- internal/beatcmd/cmd.go | 1 - internal/elasticsearch/client_test.go | 21 +- internal/elasticsearch/security_api.go | 87 ----- systemtest/agentconfig_test.go | 2 +- systemtest/apikeycmd_test.go | 269 ------------- systemtest/auth_test.go | 17 +- systemtest/elasticsearch.go | 59 ++- systemtest/instrumentation_test.go | 7 +- x-pack/apm-server/root_test.go | 1 - 10 files changed, 59 insertions(+), 917 deletions(-) delete mode 100644 internal/beatcmd/apikey.go delete mode 100644 systemtest/apikeycmd_test.go diff --git a/internal/beatcmd/apikey.go b/internal/beatcmd/apikey.go deleted file mode 100644 index f90b25af08b..00000000000 --- a/internal/beatcmd/apikey.go +++ /dev/null @@ -1,512 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package beatcmd - -import ( - "context" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "strings" - "time" - - "github.com/dustin/go-humanize" - "github.com/spf13/cobra" - - agentconfig "github.com/elastic/elastic-agent-libs/config" - - "github.com/elastic/apm-server/internal/beater/config" - "github.com/elastic/apm-server/internal/beater/headers" - - "github.com/elastic/apm-server/internal/beater/auth" - es "github.com/elastic/apm-server/internal/elasticsearch" -) - -const apikeyDeprecationNotice = `NOTE: "apm-server apikey" is deprecated, and will be removed in a future release. -See https://www.elastic.co/guide/en/apm/guide/current/api-key.html for managing API Keys.` - -func genApikeyCmd() *cobra.Command { - - short := "Manage API Keys for communication between APM agents and server (deprecated)" - apikeyCmd := cobra.Command{ - Use: "apikey", - Short: short, - Long: short + `. -Most operations require the "manage_api_key" cluster privilege. Ensure to configure "apm-server.api_key.*" or -"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; -you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests. -Check the Elastic Security API documentation for details. - -` + apikeyDeprecationNotice, - } - - apikeyCmd.AddCommand( - createApikeyCmd(), - invalidateApikeyCmd(), - getApikeysCmd(), - verifyApikeyCmd(), - ) - return &apikeyCmd -} - -func createApikeyCmd() *cobra.Command { - var keyName, expiration string - var ingest, sourcemap, agentConfig, json bool - short := "Create an API Key with the specified privilege(s)" - create := &cobra.Command{ - Use: "create", - Short: short, - Long: short + `. -If no privilege(s) are specified, the API Key will be valid for all.`, - Run: makeAPIKeyRun(&json, func(client *es.Client, config *config.Config, args []string) error { - privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) - if len(privileges) == 0 { - // No privileges specified, grant all. - privileges = auth.AllPrivilegeActions() - } - return createAPIKey(client, keyName, expiration, privileges, json) - }), - } - create.Flags().StringVar(&keyName, "name", "apm-key", "API Key name") - create.Flags().StringVar(&expiration, "expiration", "", - `expiration for the key, eg. "1d" (default never)`) - create.Flags().BoolVar(&ingest, "ingest", false, - fmt.Sprintf("give the %v privilege to this key, required for ingesting events", auth.PrivilegeEventWrite)) - create.Flags().BoolVar(&sourcemap, "sourcemap", false, - fmt.Sprintf("give the %v privilege to this key, required for uploading sourcemaps", - auth.PrivilegeSourcemapWrite)) - create.Flags().BoolVar(&agentConfig, "agent-config", false, - fmt.Sprintf("give the %v privilege to this key, required for agents to read configuration remotely", - auth.PrivilegeAgentConfigRead)) - create.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - // this actually means "preserve sorting given in code" and not reorder them alphabetically - create.Flags().SortFlags = false - return create -} - -func invalidateApikeyCmd() *cobra.Command { - var id, name string - var json bool - short := "Invalidate API Key(s) by Id or Name" - invalidate := &cobra.Command{ - Use: "invalidate", - Short: short, - Long: short + `. -If both "id" and "name" are supplied, only "id" will be used. -If neither of them are, an error will be returned.`, - Run: makeAPIKeyRun(&json, func(client *es.Client, config *config.Config, args []string) error { - if id == "" && name == "" { - // TODO(axw) this should trigger usage - return errors.New(`either "id" or "name" are required`) - } - return invalidateAPIKey(client, id, name, json) - }), - } - invalidate.Flags().StringVar(&id, "id", "", "id of the API Key to delete") - invalidate.Flags().StringVar(&name, "name", "", - "name of the API Key(s) to delete (several might match)") - invalidate.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - invalidate.Flags().SortFlags = false - return invalidate -} - -func getApikeysCmd() *cobra.Command { - var id, name string - var validOnly, json bool - short := "Query API Key(s) by Id or Name" - info := &cobra.Command{ - Use: "info", - Short: short, - Long: short + `. -If both "id" and "name" are supplied, only "id" will be used. -If neither of them are, an error will be returned.`, - Run: makeAPIKeyRun(&json, func(client *es.Client, config *config.Config, args []string) error { - if id == "" && name == "" { - // TODO(axw) this should trigger usage - return errors.New(`either "id" or "name" are required`) - } - return getAPIKey(client, &id, &name, validOnly, json) - }), - } - info.Flags().StringVar(&id, "id", "", "id of the API Key to query") - info.Flags().StringVar(&name, "name", "", - "name of the API Key(s) to query (several might match)") - info.Flags().BoolVar(&validOnly, "valid-only", false, - "only return valid API Keys (not expired or invalidated)") - info.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - info.Flags().SortFlags = false - return info -} - -func verifyApikeyCmd() *cobra.Command { - var credentials string - var ingest, sourcemap, agentConfig, json bool - short := `Check if a "credentials" string has the given privilege(s)` - long := short + `. -If no privilege(s) are specified, the credentials will be queried for all.` - verify := &cobra.Command{ - Use: "verify", - Short: short, - Long: long, - Run: makeAPIKeyRun(&json, func(client *es.Client, config *config.Config, args []string) error { - privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) - if len(privileges) == 0 { - privileges = auth.AllPrivilegeActions() - } - return verifyAPIKey(config, privileges, credentials, json) - }), - } - verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges (required)`) - verify.Flags().BoolVar(&ingest, "ingest", false, - fmt.Sprintf("ask for the %v privilege, required for ingesting events", auth.PrivilegeEventWrite)) - verify.Flags().BoolVar(&sourcemap, "sourcemap", false, - fmt.Sprintf("ask for the %v privilege, required for uploading sourcemaps", - auth.PrivilegeSourcemapWrite)) - verify.Flags().BoolVar(&agentConfig, "agent-config", false, - fmt.Sprintf("ask for the %v privilege, required for agents to read configuration remotely", - auth.PrivilegeAgentConfigRead)) - verify.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - verify.MarkFlagRequired("credentials") - verify.Flags().SortFlags = false - - return verify -} - -type apikeyRunFunc func(client *es.Client, config *config.Config, args []string) error - -type cobraRunFunc func(cmd *cobra.Command, args []string) - -func makeAPIKeyRun(json *bool, f apikeyRunFunc) cobraRunFunc { - return func(cmd *cobra.Command, args []string) { - var failed bool - client, config, err := bootstrap() - if err != nil { - failed = true - printErr(err, *json) - } else if err := f(client, config, args); err != nil { - failed = true - printErr(err, *json) - } - fmt.Fprintf(os.Stderr, "\n%s\n", apikeyDeprecationNotice) - if failed { - os.Exit(1) - } - } -} - -// apm-server.api_key.enabled is implicitly true -func bootstrap() (*es.Client, *config.Config, error) { - cfg, _, _, err := LoadConfig(WithMergeConfig( - agentconfig.MustNewConfigFrom(map[string]interface{}{ - "apm-server.auth.api_key.enabled": true, - }), - )) - if err != nil { - return nil, nil, err - } - - var esOutputCfg *agentconfig.C - if cfg.Output.Name() == "elasticsearch" { - esOutputCfg = cfg.Output.Config() - } - - beaterConfig, err := config.NewConfig(cfg.APMServer, esOutputCfg) - if err != nil { - return nil, nil, err - } - client, err := es.NewClient(beaterConfig.AgentAuth.APIKey.ESConfig) - if err != nil { - return nil, nil, err - } - return client, beaterConfig, nil -} - -func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.PrivilegeAction { - privileges := make([]es.PrivilegeAction, 0) - if ingest { - privileges = append(privileges, auth.PrivilegeEventWrite.Action) - } - if sourcemap { - privileges = append(privileges, auth.PrivilegeSourcemapWrite.Action) - } - if agentConfig { - privileges = append(privileges, auth.PrivilegeAgentConfigRead.Action) - } - return privileges -} - -func createAPIKey(client *es.Client, keyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) error { - - // Elasticsearch will allow a user without the right apm privileges to create API keys, but the keys won't validate - // check first whether the user has the right privileges, and bail out early if not - // is not possible to always do it automatically, because file-based users and roles are not queryable - hasPrivileges, err := es.HasPrivileges(context.Background(), client, es.HasPrivilegesRequest{ - Applications: []es.Application{ - { - Name: auth.Application, - Privileges: privileges, - Resources: []es.Resource{auth.ResourceInternal}, - }, - }, - }, "") - if err != nil { - return err - } - if !hasPrivileges.HasAll { - var missingPrivileges []string - for action, hasPrivilege := range hasPrivileges.Application[auth.Application][auth.ResourceInternal] { - if !hasPrivilege { - missingPrivileges = append(missingPrivileges, string(action)) - } - } - return fmt.Errorf(`%s is missing the following requested privilege(s): %s. - -You might try with the superuser, or add the APM application privileges to the role of the authenticated user, eg.: -PUT /_security/role/my_role { - ... - "applications": [{ - "application": "apm", - "privileges": ["sourcemap:write", "event:write", "config_agent:read"], - "resources": ["*"] - }], - ... -} - `, hasPrivileges.Username, strings.Join(missingPrivileges, ", ")) - } - - printText, printJSON := printers(asJSON) - - apikeyRequest := es.CreateAPIKeyRequest{ - Name: keyName, - RoleDescriptors: es.RoleDescriptor{ - auth.Application: es.Applications{ - Applications: []es.Application{ - { - Name: auth.Application, - Privileges: privileges, - Resources: []es.Resource{"*"}, - }, - }, - }, - }, - Metadata: map[string]interface{}{"application": "apm"}, - } - if expiry != "" { - apikeyRequest.Expiration = &expiry - } - - response, err := es.CreateAPIKey(context.Background(), client, apikeyRequest) - if err != nil { - return err - } - - type APIKey struct { - es.CreateAPIKeyResponse - Credentials string `json:"credentials"` - } - apikey := APIKey{ - CreateAPIKeyResponse: response, - Credentials: base64.StdEncoding.EncodeToString([]byte(response.ID + ":" + response.Key)), - } - - printText("API Key created:") - printText("") - printText("Name ........... %s", apikey.Name) - printText("Expiration ..... %s", humanTime(apikey.ExpirationMs)) - printText("Id ............. %s", apikey.ID) - printText("API Key ........ %s (won't be shown again)", apikey.Key) - printText(`Credentials .... %s (use it as "Authorization: APIKey " header to communicate with APM Server, won't be shown again)`, apikey.Credentials) - - printJSON(apikey) - return nil -} - -func getAPIKey(client *es.Client, id, name *string, validOnly, asJSON bool) error { - if isSet(id) { - name = nil - } else if isSet(name) { - id = nil - } - request := es.GetAPIKeyRequest{ - APIKeyQuery: es.APIKeyQuery{ - ID: id, - Name: name, - }, - } - - apikeys, err := es.GetAPIKeys(context.Background(), client, request) - if err != nil { - return err - } - - transform := es.GetAPIKeyResponse{APIKeys: make([]es.APIKeyResponse, 0)} - printText, printJSON := printers(asJSON) - for _, apikey := range apikeys.APIKeys { - expiry := humanTime(apikey.ExpirationMs) - if validOnly && (apikey.Invalidated || expiry == "expired") { - continue - } - creation := time.Unix(apikey.Creation/1000, 0).Format("2006-02-01 15:04") - printText("Username ....... %s", apikey.Username) - printText("Api Key Name ... %s", apikey.Name) - printText("Id ............. %s", apikey.ID) - printText("Creation ....... %s", creation) - printText("Invalidated .... %t", apikey.Invalidated) - if !apikey.Invalidated { - printText("Expiration ..... %s", expiry) - } - printText("") - transform.APIKeys = append(transform.APIKeys, apikey) - } - printText("%d API Keys found", len(transform.APIKeys)) - printJSON(transform) - return nil -} - -func invalidateAPIKey(client *es.Client, id string, name string, asJSON bool) error { - invalidateKeysRequest := es.InvalidateAPIKeyRequest{} - if id != "" { - invalidateKeysRequest.IDs = []string{id} - } else if name != "" { - invalidateKeysRequest.Name = &name - } - invalidation, err := es.InvalidateAPIKey(context.Background(), client, invalidateKeysRequest) - if err != nil { - return err - } - - printText, printJSON := printers(asJSON) - printText("Invalidated keys ... %s", strings.Join(invalidation.Invalidated, ", ")) - printText("Error count ........ %d", invalidation.ErrorCount) - printJSON(invalidation) - return nil -} - -func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, credentials string, asJSON bool) error { - authenticator, err := auth.NewAuthenticator(config.AgentAuth) - if err != nil { - return err - } - _, authz, err := authenticator.Authenticate(context.Background(), headers.APIKey, credentials) - if err != nil { - return err - } - perms := make(es.Permissions) - printText, printJSON := printers(asJSON) - for _, privilege := range privileges { - var action auth.Action - switch privilege { - case auth.PrivilegeAgentConfigRead.Action: - action = auth.ActionAgentConfig - case auth.PrivilegeEventWrite.Action: - action = auth.ActionEventIngest - case auth.PrivilegeSourcemapWrite.Action: - action = auth.ActionSourcemapUpload - } - - authorized := true - if err := authz.Authorize(context.Background(), action, auth.Resource{}); err != nil { - if errors.Is(err, auth.ErrUnauthorized) { - authorized = false - } else { - return err - } - } - perms[privilege] = authorized - printText("Authorized for %s...: %s", humanPrivilege(privilege), humanBool(authorized)) - } - printJSON(perms) - return nil -} - -func humanBool(b bool) string { - if b { - return "Yes" - } - return "No" -} - -func humanPrivilege(privilege es.PrivilegeAction) string { - return fmt.Sprintf("privilege \"%v\"", privilege) -} - -func humanTime(millis *int64) string { - if millis == nil { - return "never" - } - expiry := time.Unix(*millis/1000, 0) - if !expiry.After(time.Now()) { - return "expired" - } - return humanize.Time(expiry) -} - -// returns 2 printers, one for text and one for JSON -// one of them will be a noop based on the boolean argument -func printers(b bool) (func(string, ...interface{}), func(interface{})) { - var w1 io.Writer = os.Stdout - var w2 = io.Discard - if b { - w1 = io.Discard - w2 = os.Stdout - } - return func(f string, i ...interface{}) { - fmt.Fprintf(w1, f, i...) - fmt.Fprintln(w1) - }, func(i interface{}) { - data, err := json.MarshalIndent(i, "", "\t") - if err != nil { - fmt.Fprintln(w2, err) - } - fmt.Fprintln(w2, string(data)) - } -} - -// prints an Elasticsearch error to stderr -func printErr(err error, asJSON bool) { - if asJSON { - var data []byte - var m map[string]interface{} - e := json.Unmarshal([]byte(err.Error()), &m) - if e == nil { - // err.Error() has JSON shape, likely coming from Elasticsearch - data, _ = json.MarshalIndent(m, "", "\t") - } else { - // err.Error() is a bare string, likely coming from apm-server - data, _ = json.MarshalIndent(struct { - Error string `json:"error"` - }{ - Error: err.Error(), - }, "", "\t") - } - fmt.Fprintln(os.Stdout, string(data)) - } else { - fmt.Fprintln(os.Stderr, err.Error()) - } -} - -func isSet(s *string) bool { - return s != nil && *s != "" -} diff --git a/internal/beatcmd/cmd.go b/internal/beatcmd/cmd.go index db0f66bfdd0..10ee94b966a 100644 --- a/internal/beatcmd/cmd.go +++ b/internal/beatcmd/cmd.go @@ -77,7 +77,6 @@ func NewRootCommand(beatParams BeatParams) *cobra.Command { rootCommand.AddCommand(keystoreCommand) rootCommand.AddCommand(versionCommand) rootCommand.AddCommand(genTestCmd(beatParams)) - rootCommand.AddCommand(genApikeyCmd()) return rootCommand } diff --git a/internal/elasticsearch/client_test.go b/internal/elasticsearch/client_test.go index 301c7075e4c..d3416a4c8ae 100644 --- a/internal/elasticsearch/client_test.go +++ b/internal/elasticsearch/client_test.go @@ -19,7 +19,6 @@ package elasticsearch import ( "bytes" - "context" "fmt" "net/http" "net/http/httptest" @@ -50,9 +49,11 @@ func TestClient(t *testing.T) { } func TestClientCustomHeaders(t *testing.T) { - var requestHeaders http.Header + wait := make(chan struct{}) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestHeaders = r.Header + w.Header().Set("X-Elastic-Product", "Elasticsearch") + assert.Equal(t, "header", r.Header.Get("custom")) + close(wait) })) defer srv.Close() @@ -63,13 +64,20 @@ func TestClientCustomHeaders(t *testing.T) { client, err := NewClient(&cfg) require.NoError(t, err) - CreateAPIKey(context.Background(), client, CreateAPIKeyRequest{}) - assert.Equal(t, "header", requestHeaders.Get("custom")) + _, err = client.Bulk(bytes.NewReader([]byte("{}"))) + require.NoError(t, err) + select { + case <-wait: + case <-time.After(1 * time.Second): + t.Fatal("timed out while waiting for request") + } + } func TestClientCustomUserAgent(t *testing.T) { wait := make(chan struct{}) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Elastic-Product", "Elasticsearch") assert.Equal(t, fmt.Sprintf("Elastic-APM-Server/%s go-elasticsearch/%s", apmVersion.Version, esv8.Version), r.Header.Get("User-Agent")) close(wait) })) @@ -81,7 +89,8 @@ func TestClientCustomUserAgent(t *testing.T) { client, err := NewClient(&cfg) require.NoError(t, err) - CreateAPIKey(context.Background(), client, CreateAPIKeyRequest{}) + _, err = client.Bulk(bytes.NewReader([]byte("{}"))) + require.NoError(t, err) select { case <-wait: case <-time.After(1 * time.Second): diff --git a/internal/elasticsearch/security_api.go b/internal/elasticsearch/security_api.go index 34f66bee9e8..d504f40bb05 100644 --- a/internal/elasticsearch/security_api.go +++ b/internal/elasticsearch/security_api.go @@ -25,35 +25,6 @@ import ( "github.com/elastic/go-elasticsearch/v8/esutil" ) -// CreateAPIKey requires manage_api_key cluster privilege -func CreateAPIKey(ctx context.Context, client *Client, apikeyReq CreateAPIKeyRequest) (CreateAPIKeyResponse, error) { - var apikey CreateAPIKeyResponse - req := esapi.SecurityCreateAPIKeyRequest{Body: esutil.NewJSONReader(apikeyReq)} - err := doRequest(ctx, client, req, &apikey) - return apikey, err -} - -// GetAPIKeys requires manage_api_key cluster privilege -func GetAPIKeys(ctx context.Context, client *Client, apikeyReq GetAPIKeyRequest) (GetAPIKeyResponse, error) { - req := esapi.SecurityGetAPIKeyRequest{} - if apikeyReq.ID != nil { - req.ID = *apikeyReq.ID - } else if apikeyReq.Name != nil { - req.Name = *apikeyReq.Name - } - var apikey GetAPIKeyResponse - err := doRequest(ctx, client, req, &apikey) - return apikey, err -} - -// InvalidateAPIKey requires manage_api_key cluster privilege -func InvalidateAPIKey(ctx context.Context, client *Client, apikeyReq InvalidateAPIKeyRequest) (InvalidateAPIKeyResponse, error) { - var confirmation InvalidateAPIKeyResponse - req := esapi.SecurityInvalidateAPIKeyRequest{Body: esutil.NewJSONReader(apikeyReq)} - err := doRequest(ctx, client, req, &confirmation) - return confirmation, err -} - func HasPrivileges(ctx context.Context, client *Client, privileges HasPrivilegesRequest, credentials string) (HasPrivilegesResponse, error) { var info HasPrivilegesResponse req := esapi.SecurityHasPrivilegesRequest{Body: esutil.NewJSONReader(privileges)} @@ -66,27 +37,6 @@ func HasPrivileges(ctx context.Context, client *Client, privileges HasPrivileges return info, err } -type CreateAPIKeyRequest struct { - Name string `json:"name"` - Expiration *string `json:"expiration,omitempty"` - RoleDescriptors RoleDescriptor `json:"role_descriptors"` - Metadata map[string]interface{} `json:"metadata,omitempty"` -} - -type CreateAPIKeyResponse struct { - APIKey - Key string `json:"api_key"` -} - -type GetAPIKeyRequest struct { - APIKeyQuery - Owner bool `json:"owner"` -} - -type GetAPIKeyResponse struct { - APIKeys []APIKeyResponse `json:"api_keys"` -} - type HasPrivilegesRequest struct { // can't reuse the `Applications` type because here the JSON attribute must be singular Applications []Application `json:"application"` @@ -97,49 +47,12 @@ type HasPrivilegesResponse struct { Application map[AppName]PermissionsPerResource `json:"application"` } -type InvalidateAPIKeyRequest struct { - // normally the Elasticsearch API will require either Ids or Name, but not both - IDs []string `json:"ids,omitempty"` - Name *string `json:"name,omitempty"` -} - -type InvalidateAPIKeyResponse struct { - Invalidated []string `json:"invalidated_api_keys"` - ErrorCount int `json:"error_count"` -} - -type RoleDescriptor map[AppName]Applications - -type Applications struct { - Applications []Application `json:"applications"` -} - type Application struct { Name AppName `json:"application"` Privileges []PrivilegeAction `json:"privileges"` Resources []Resource `json:"resources"` } -type APIKeyResponse struct { - APIKey - Creation int64 `json:"creation"` - Invalidated bool `json:"invalidated"` - Username string `json:"username"` - Metadata map[string]interface{} `json:"metadata,omitempty"` -} - -type APIKeyQuery struct { - // normally the Elasticsearch API will require either Id or Name, but not both - ID *string `json:"id,omitempty"` - Name *string `json:"name,omitempty"` -} - -type APIKey struct { - ID string `json:"id"` - Name string `json:"name"` - ExpirationMs *int64 `json:"expiration,omitempty"` -} - type Permissions map[PrivilegeAction]bool type PermissionsPerResource map[Resource]Permissions diff --git a/systemtest/agentconfig_test.go b/systemtest/agentconfig_test.go index 8cd2a8683a7..81c56d7ec7a 100644 --- a/systemtest/agentconfig_test.go +++ b/systemtest/agentconfig_test.go @@ -249,7 +249,7 @@ func TestAgentConfigForbiddenOnInvalidConfig(t *testing.T) { systemtest.InvalidateAPIKeyByName(t, apiKeyName) }) // Create an API Key without agent config read privileges - apiKeyBase64 := createAPIKey(t, apiKeyName, "--sourcemap") + apiKeyBase64 := systemtest.CreateAPIKey(t, apiKeyName, []string{"sourcemap:write"}) apiKeyBytes, err := base64.StdEncoding.DecodeString(apiKeyBase64) require.NoError(t, err) srv := apmservertest.NewUnstartedServerTB(t) diff --git a/systemtest/apikeycmd_test.go b/systemtest/apikeycmd_test.go deleted file mode 100644 index 3bdffc3c8a7..00000000000 --- a/systemtest/apikeycmd_test.go +++ /dev/null @@ -1,269 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package systemtest_test - -import ( - "bytes" - "context" - "encoding/json" - "io" - "net/http" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/elastic/apm-tools/pkg/espoll" - "github.com/elastic/go-elasticsearch/v8/esapi" - - "github.com/elastic/apm-server/systemtest" - "github.com/elastic/apm-server/systemtest/apmservertest" -) - -func apiKeyCommand(subcommand string, args ...string) *apmservertest.ServerCmd { - cfg := apmservertest.DefaultConfig() - return apiKeyCommandConfig(cfg, subcommand, args...) -} - -func apiKeyCommandConfig(cfg apmservertest.Config, subcommand string, args ...string) *apmservertest.ServerCmd { - cfgargs, err := cfg.Args() - if err != nil { - panic(err) - } - - var esargs []string - for i := 1; i < len(cfgargs); i += 2 { - if !strings.HasPrefix(cfgargs[i], "output.elasticsearch") { - continue - } - esargs = append(esargs, "-E", cfgargs[i]) - } - - userargs := args - args = append([]string{subcommand}, esargs...) - args = append(args, userargs...) - return apmservertest.ServerCommand(context.Background(), "apikey", args...) -} - -func TestAPIKeyCreate(t *testing.T) { - systemtest.InvalidateAPIKeys(t) - defer systemtest.InvalidateAPIKeys(t) - - cmd := apiKeyCommand("create", "--name", t.Name(), "--json") - out, err := cmd.Output() - require.NoError(t, err) - - attrs := decodeJSONMap(t, bytes.NewReader(out)) - assert.Equal(t, t.Name(), attrs["name"]) - assert.Contains(t, attrs, "id") - assert.Contains(t, attrs, "api_key") - assert.Contains(t, attrs, "credentials") - - es := systemtest.NewElasticsearchClientWithAPIKey(attrs["credentials"].(string)) - assertAuthenticateSucceeds(t, es) - - // Check that the API Key has expected metadata. - type apiKey struct { - ID string `json:"id"` - Metadata map[string]interface{} `json:"metadata"` - } - var resp struct { - APIKeys []apiKey `json:"api_keys"` - } - _, err = systemtest.Elasticsearch.Do(context.Background(), &esapi.SecurityGetAPIKeyRequest{ - ID: attrs["id"].(string), - }, &resp) - require.NoError(t, err) - require.Len(t, resp.APIKeys, 1) - assert.Equal(t, map[string]interface{}{"application": "apm"}, resp.APIKeys[0].Metadata) -} - -func TestAPIKeyCreateExpiration(t *testing.T) { - systemtest.InvalidateAPIKeys(t) - defer systemtest.InvalidateAPIKeys(t) - - cmd := apiKeyCommand("create", "--name", t.Name(), "--json", "--expiration=1d") - out, err := cmd.Output() - require.NoError(t, err) - - attrs := decodeJSONMap(t, bytes.NewReader(out)) - assert.Contains(t, attrs, "expiration") -} - -func TestAPIKeyCreateInvalidUser(t *testing.T) { - // heartbeat_user lacks cluster privileges, and cannot create keys - // beats_user has cluster privileges, but not APM application privileges - for _, username := range []string{"heartbeat_user", "beats_user"} { - cfg := apmservertest.DefaultConfig() - cfg.Output.Elasticsearch.Username = username - cfg.Output.Elasticsearch.Password = "changeme" - - cmd := apiKeyCommandConfig(cfg, "create", "--name", t.Name(), "--json") - out, err := cmd.Output() - require.Error(t, err) - attrs := decodeJSONMap(t, bytes.NewReader(out)) - assert.Regexp(t, username+` is missing the following requested privilege\(s\): .*`, attrs["error"]) - } -} - -func TestAPIKeyInvalidateName(t *testing.T) { - systemtest.InvalidateAPIKeys(t) - defer systemtest.InvalidateAPIKeys(t) - - var clients []*espoll.Client - for i := 0; i < 2; i++ { - cmd := apiKeyCommand("create", "--name", t.Name(), "--json") - out, err := cmd.Output() - require.NoError(t, err) - - attrs := decodeJSONMap(t, bytes.NewReader(out)) - es := systemtest.NewElasticsearchClientWithAPIKey(attrs["credentials"].(string)) - assertAuthenticateSucceeds(t, es) - clients = append(clients, es) - } - - cmd := apiKeyCommand("invalidate", "--name", t.Name(), "--json") - out, err := cmd.Output() - require.NoError(t, err) - - result := decodeJSONMap(t, bytes.NewReader(out)) - assert.Len(t, result["invalidated_api_keys"], 2) - assert.Equal(t, float64(0), result["error_count"]) - - for _, es := range clients { - assertAuthenticateFails(t, es) - } -} - -func TestAPIKeyInvalidateID(t *testing.T) { - systemtest.InvalidateAPIKeys(t) - defer systemtest.InvalidateAPIKeys(t) - - cmd := apiKeyCommand("create", "--json") - out, err := cmd.Output() - require.NoError(t, err) - attrs := decodeJSONMap(t, bytes.NewReader(out)) - - es := systemtest.NewElasticsearchClientWithAPIKey(attrs["credentials"].(string)) - assertAuthenticateSucceeds(t, es) - - // NOTE(axw) it is important to use "--id=" rather than "--id" , - // as API keys may begin with a hyphen and be interpreted as flags. - cmd = apiKeyCommand("invalidate", "--json", "--id="+attrs["id"].(string)) - out, err = cmd.Output() - require.NoError(t, err) - result := decodeJSONMap(t, bytes.NewReader(out)) - - assert.Equal(t, []interface{}{attrs["id"]}, result["invalidated_api_keys"]) - assert.Equal(t, float64(0), result["error_count"]) - assertAuthenticateFails(t, es) -} - -func TestAPIKeyVerify(t *testing.T) { - systemtest.InvalidateAPIKeys(t) - defer systemtest.InvalidateAPIKeys(t) - - cmd := apiKeyCommand("create", "--name", t.Name(), "--json", "--ingest", "--agent-config") - out, err := cmd.Output() - require.NoError(t, err) - attrs := decodeJSONMap(t, bytes.NewReader(out)) - credentials := attrs["credentials"].(string) - - cmd = apiKeyCommand("verify", "--json", "--credentials="+credentials) - out, err = cmd.Output() - require.NoError(t, err) - attrs = decodeJSONMap(t, bytes.NewReader(out)) - assert.Equal(t, map[string]interface{}{ - "event:write": true, - "config_agent:read": true, - "sourcemap:write": false, - }, attrs) - - cmd = apiKeyCommand("verify", "--json", "--credentials="+credentials, "--ingest") - out, err = cmd.Output() - require.NoError(t, err) - attrs = decodeJSONMap(t, bytes.NewReader(out)) - assert.Equal(t, map[string]interface{}{"event:write": true}, attrs) -} - -func TestAPIKeyInfo(t *testing.T) { - systemtest.InvalidateAPIKeys(t) - defer systemtest.InvalidateAPIKeys(t) - - var ids []string - for i := 0; i < 2; i++ { - cmd := apiKeyCommand("create", "--name", t.Name(), "--json", "--ingest", "--agent-config") - out, err := cmd.Output() - require.NoError(t, err) - attrs := decodeJSONMap(t, bytes.NewReader(out)) - ids = append(ids, attrs["id"].(string)) - } - - type apiKey struct { - ID string `json:"id"` - Name string `json:"name"` - } - var result struct { - APIKeys []apiKey `json:"api_keys"` - } - - cmd := apiKeyCommand("info", "--json", "--id="+ids[0]) - out, err := cmd.Output() - require.NoError(t, err) - err = json.Unmarshal(out, &result) - require.NoError(t, err) - assert.Equal(t, []apiKey{{ - ID: ids[0], - Name: t.Name(), - }}, result.APIKeys) - - result.APIKeys = nil - cmd = apiKeyCommand("info", "--json", "--name="+t.Name()) - out, err = cmd.Output() - require.NoError(t, err) - err = json.Unmarshal(out, &result) - require.NoError(t, err) - // Should be at least 2, possibly more; Elasticsearch may - // hold invalidated keys from previous test runs. - assert.GreaterOrEqual(t, len(result.APIKeys), 2) -} - -func assertAuthenticateSucceeds(t testing.TB, es *espoll.Client) { - t.Helper() - resp, err := es.Security.Authenticate() - require.NoError(t, err) - assert.False(t, resp.IsError()) - assert.NoError(t, resp.Body.Close()) -} - -func assertAuthenticateFails(t testing.TB, es *espoll.Client) { - t.Helper() - resp, err := es.Security.Authenticate() - require.NoError(t, err) - assert.True(t, resp.IsError()) - assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) - assert.NoError(t, resp.Body.Close()) -} - -func decodeJSONMap(t *testing.T, r io.Reader) map[string]interface{} { - var m map[string]interface{} - err := json.NewDecoder(r).Decode(&m) - require.NoError(t, err) - return m -} diff --git a/systemtest/auth_test.go b/systemtest/auth_test.go index 3e6c68b3811..bd0000f5ddf 100644 --- a/systemtest/auth_test.go +++ b/systemtest/auth_test.go @@ -54,10 +54,10 @@ func TestAuth(t *testing.T) { err := srv.Start() require.NoError(t, err) - apiKey := createAPIKey(t, t.Name()+":all") - apiKeySourcemap := createAPIKey(t, t.Name()+":sourcemap", "--sourcemap") - apiKeyIngest := createAPIKey(t, t.Name()+":ingest", "--ingest") - apiKeyAgentConfig := createAPIKey(t, t.Name()+":agentconfig", "--agent-config") + apiKey := systemtest.CreateAPIKey(t, t.Name()+":all", []string{"config_agent:read", "sourcemap:write", "event:write"}) + apiKeySourcemap := systemtest.CreateAPIKey(t, t.Name()+":sourcemap", []string{"sourcemap:write"}) + apiKeyIngest := systemtest.CreateAPIKey(t, t.Name()+":ingest", []string{"event:write"}) + apiKeyAgentConfig := systemtest.CreateAPIKey(t, t.Name()+":agentconfig", []string{"config_agent:read"}) runWithMethods := func(t *testing.T, name string, f func(t *testing.T, apiKey string, headers http.Header)) { t.Run(name, func(t *testing.T) { @@ -189,12 +189,3 @@ func copyHeaders(to, from http.Header) { } } } - -func createAPIKey(t *testing.T, name string, args ...string) string { - args = append([]string{"--name", name, "--json"}, args...) - cmd := apiKeyCommand("create", args...) - out, err := cmd.CombinedOutput() - require.NoError(t, err) - attrs := decodeJSONMap(t, bytes.NewReader(out)) - return attrs["credentials"].(string) -} diff --git a/systemtest/elasticsearch.go b/systemtest/elasticsearch.go index 5bb6c4f48a5..e259bb4b6f0 100644 --- a/systemtest/elasticsearch.go +++ b/systemtest/elasticsearch.go @@ -19,6 +19,8 @@ package systemtest import ( "context" + "encoding/json" + "io" "net/url" "testing" "time" @@ -55,18 +57,6 @@ func initElasticSearch() { Elasticsearch = &espoll.Client{Client: client} } -// NewElasticsearchClientWithAPIKey returns a new espoll.Client, -// configured to use apiKey for authentication. -func NewElasticsearchClientWithAPIKey(apiKey string) *espoll.Client { - cfg := newElasticsearchConfig() - cfg.APIKey = apiKey - client, err := elasticsearch.NewClient(cfg) - if err != nil { - panic(err) - } - return &espoll.Client{Client: client} -} - func newElasticsearchConfig() elasticsearch.Config { var addresses []string for _, host := range apmservertest.DefaultConfig().Output.Elasticsearch.Hosts { @@ -115,22 +105,49 @@ func ChangeUserPassword(t testing.TB, username, password string) { } } -// InvalidateAPIKeys invalidates all API Keys for the apm-server user. -func InvalidateAPIKeys(t testing.TB) { - req := esapi.SecurityInvalidateAPIKeyRequest{ - Body: esutil.NewJSONReader(map[string]interface{}{ - "username": apmservertest.DefaultConfig().Output.Elasticsearch.Username, +func CreateAPIKey(t testing.TB, name string, privileges []string) string { + req := esapi.SecurityCreateAPIKeyRequest{ + Body: esutil.NewJSONReader(map[string]any{ + "name": name, + "role_descriptors": map[string]any{ + "apm": map[string]any{ + "applications": []map[string]any{ + { + "application": "apm", + "privileges": privileges, + "resources": []string{"*"}, + }, + }, + }, + }, + "metadata": map[string]any{"application": "apm"}, }), } - if _, err := Elasticsearch.Do(context.Background(), req, nil); err != nil { + + res, err := Elasticsearch.Do(context.Background(), req, nil) + if err != nil { + t.Fatal(err) + } + + b, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + + m := make(map[string]any) + if err := json.Unmarshal(b, &m); err != nil { t.Fatal(err) } + + return m["encoded"].(string) } -// InvalidateAPIKey invalidates the API Key with the given ID. -func InvalidateAPIKey(t testing.TB, id string) { +// InvalidateAPIKeys invalidates all API Keys for the apm-server user. +func InvalidateAPIKeys(t testing.TB) { req := esapi.SecurityInvalidateAPIKeyRequest{ - Body: esutil.NewJSONReader(map[string]interface{}{"id": id}), + Body: esutil.NewJSONReader(map[string]interface{}{ + "username": apmservertest.DefaultConfig().Output.Elasticsearch.Username, + }), } if _, err := Elasticsearch.Do(context.Background(), req, nil); err != nil { t.Fatal(err) diff --git a/systemtest/instrumentation_test.go b/systemtest/instrumentation_test.go index c5305fefecf..09d67469e92 100644 --- a/systemtest/instrumentation_test.go +++ b/systemtest/instrumentation_test.go @@ -18,7 +18,6 @@ package systemtest_test import ( - "bytes" "encoding/json" "net/http" "net/http/httptest" @@ -138,11 +137,7 @@ func TestAPMServerInstrumentationAuth(t *testing.T) { systemtest.InvalidateAPIKeys(t) defer systemtest.InvalidateAPIKeys(t) - cmd := apiKeyCommand("create", "--name", t.Name(), "--json") - out, err := cmd.CombinedOutput() - require.NoError(t, err) - attrs := decodeJSONMap(t, bytes.NewReader(out)) - srv.Config.Instrumentation.APIKey = attrs["credentials"].(string) + srv.Config.Instrumentation.APIKey = systemtest.CreateAPIKey(t, t.Name(), []string{"config_agent:read", "sourcemap:write", "event:write"}) } err := srv.Start() diff --git a/x-pack/apm-server/root_test.go b/x-pack/apm-server/root_test.go index 39fbf2aa30d..9be226c59b0 100644 --- a/x-pack/apm-server/root_test.go +++ b/x-pack/apm-server/root_test.go @@ -22,7 +22,6 @@ func TestSubCommands(t *testing.T) { } assert.ElementsMatch(t, []string{ - "apikey", "export", "keystore", "run",