From 5ecd6f0ab8b0ac063eebd46d7249a9e64caf326c Mon Sep 17 00:00:00 2001 From: Steve Ramage Date: Fri, 19 Jan 2024 05:46:52 -0800 Subject: [PATCH] Resolves #444 - Support regex replacement of URL --- README.md | 56 ++++++++++++++++++------------- external/httpclient/httpclient.go | 36 ++++++++++++++++++++ external/resources/resources.go | 36 ++++++++++---------- 3 files changed, 87 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 3c8d1f5..e984d71 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ # Elastic Path Composable Commerce Command Line Interface - ## Overview -This project is designed as a tool for power users to interact with the [Elastic Path Composable Commerce API](https://documentation.elasticpath.com/commerce-cloud/docs/api/) via the command line and the project is designed to fill three distinct niches: + +This project is designed as a tool for power users to interact with the [Elastic Path Composable Commerce API](https://documentation.elasticpath.com/commerce-cloud/docs/api/) via the command line and the project is designed to fill three distinct +niches: + 1. Provide a fast way for users familiar with the API to interact with it. 2. Provide a simpler way to do scripting with the API (i.e., instead of using curl and creating JSON in the shell) 3. Provide a reusable set of scripts for creating data sets with [Runbooks](docs/runbook-development.md). -This tool is not meant for new users unfamiliar with the API, new users are highly encouraged to use the [Elastic Path Composable Commerce Postman Collection](https://elasticpath.dev/docs/commerce-cloud/api-overview/test-with-postman-collection) instead of this tool. +This tool is not meant for new users unfamiliar with the API, new users are highly encouraged to use the [Elastic Path Composable Commerce Postman Collection](https://elasticpath.dev/docs/commerce-cloud/api-overview/test-with-postman-collection) +instead of this tool. Additionally, this tool is not necessarily meant to be a new command line equivalent of Commerce Manager, it should just feel at all times like you are interacting with a JSON based REST API. @@ -26,6 +29,7 @@ It is highly recommended that new users check out the [Tutorial](docs/tutorial.m The following is a summary of the main commands, in general you can type `epcc help` to get an updated list and see all commands as well as flags. #### CRUD Commands + | Command | Description | |-------------------------------------------------------------|---------------------------------------------------------------------------| | `epcc get [ID] ... [QUERY_PARAM_KEY] [VAL] ...` | Retrieves either a list of objects, or an particular object from the API. | @@ -43,7 +47,8 @@ The following is a summary of the main commands, in general you can type `epcc h | `epcc login implicit` | Login to the API using an Implicit Token | | `epcc login status` | Determine the current state of the login | -#### Debugging Commands +#### Debugging Commands + | Command | Description | |----------------------------------------------------|------------------------------------------------------------------------------| | `epcc docs ` | Open the API docs for a resource in your browser | @@ -66,21 +71,21 @@ The following is a summary of the main commands, in general you can type `epcc h 1. `--execution-timeout` will control how long the `epcc` process can run before timing out. 2. `--rate-limit` will control the number of requests per second to EPCC. 3. `--max-concurrency` will control the maximum number of concurrent commands that can run simultaneously. - * This differs from the rate limit in that if a request takes 2 seconds, a rate limit of 3 will allow 6 requests in flight at a time, whereas `--max-concurrency` would limit you to 3. A higher value will slow down initial start time. + * This differs from the rate limit in that if a request takes 2 seconds, a rate limit of 3 will allow 6 requests in flight at a time, whereas `--max-concurrency` would limit you to 3. A higher value will slow down initial start time. #### Headers Headers can be set in one of three ways, depending on what is most convenient 1. Via the `-H` argument. - * This header will be one time only. + * This header will be one time only. 2. Via the `EPCC_CLI_HTTP_HEADER_0` environment variable. - * This header will be always be set. + * This header will be always be set. 3. Via the `epcc header set` - * These headers will be set in the current profile and will stay until unset. You can see what headers are set with `epcc headers status` - * Headers set this way support aliases. - * You can also additionally group headers into groups with `--group` and then clear all headers with `epcc headers clear ` - + * These headers will be set in the current profile and will stay until unset. You can see what headers are set with `epcc headers status` + * Headers set this way support aliases. + * You can also additionally group headers into groups with `--group` and then clear all headers with `epcc headers clear ` + ### Configuration #### Via Prompts @@ -91,16 +96,18 @@ Run the `epcc configure` and it will prompt you for the required settings, when The following environment variables can be set up to control which environment and store to use with the EPCC CLI. -| Environment Variable | Description | -|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| EPCC_API_BASE_URL | This is the API base URL which can be retrieved via CM. | -| EPCC_BETA_API_FEATURES | This variable allows you to set [Beta Headers](https://documentation.elasticpath.com/commerce-cloud/docs/api/basics/api-contract.html#beta-apis) for all API calls. | -| EPCC_CLI_HTTP_HEADER_**N** | Setting any environment variable like this (where N is a number) will cause it's value to be parsed and added to all HTTP headers (e.g., `EPCC_CLI_HTTP_HEADER_0=Cache-Control: no-cache` will add `Cache-Control: no-cache` as a header). FYI, the surprising syntax is due to different encoding rules. You can also specify headers using `-H` or `epcc headers` | -| EPCC_CLI_SUPPRESS_NO_AUTH_MESSAGES | This will supress warning messages about not being authenticated or logged out | -| EPCC_CLIENT_ID | This is the Client ID which can be retrieved via CM. | -| EPCC_CLIENT_SECRET | This is the Client Secret which can be retrieved via CM. | -| EPCC_PROFILE | A profile name that allows for an independent session and isolation (e.g., distinct histories) | -| EPCC_RUNBOOK_DIRECTORY | A directory that will be scanned for runbook, a runbook ends with `.epcc.yml` | +| Environment Variable | Description | +|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| EPCC_API_BASE_URL | This is the API base URL which can be retrieved via CM. | +| EPCC_BETA_API_FEATURES | This variable allows you to set [Beta Headers](https://documentation.elasticpath.com/commerce-cloud/docs/api/basics/api-contract.html#beta-apis) for all API calls. | +| EPCC_CLI_HTTP_HEADER_**N** | Setting any environment variable like this (where N is a number) will cause it's value to be parsed and added to all HTTP headers (e.g., `EPCC_CLI_HTTP_HEADER_0=Cache-Control: no-cache` will add `Cache-Control: no-cache` as a header). FYI, the surprising syntax is due to different encoding rules. You can also specify headers using `-H` or `epcc headers`. | +| EPCC_CLI_SUPPRESS_NO_AUTH_MESSAGES | This will supress warning messages about not being authenticated or logged out | +| EPCC_CLI_URL_MATCH_REGEXP_**N** | Setting this value causes the _path_ section of a URL to be matched and replaced with a corresponding value from the `EPCC_CLI_URL_MATCH_SUBSTITION_**N**` header, if not set the empty string is used. | +| EPCC_CLI_URL_MATCH_SUBSTITION_**N** | The replacement string to use when a match is found. Capture groups and back references are supported (see [ReplaceAllString](https://pkg.go.dev/regexp#Regexp.ReplaceAllString)). | +| EPCC_CLIENT_ID | This is the Client ID which can be retrieved via CM. | +| EPCC_CLIENT_SECRET | This is the Client Secret which can be retrieved via CM. | +| EPCC_PROFILE | A profile name that allows for an independent session and isolation (e.g., distinct histories). | +| EPCC_RUNBOOK_DIRECTORY | A directory that will be scanned for runbook, a runbook ends with `.epcc.yml`. | It is recommended to set EPCC_API_BASE_URL, EPCC_CLIENT_ID, and EPCC_CLIENT_SECRET to be able to interact with most things in the CLI. @@ -123,7 +130,9 @@ You will need to start a new shell for this setup to take effect #### Bash -You will need to have the [bash-completion](https://github.com/scop/bash-completion) (e.g., [Ubuntu](https://packages.ubuntu.com/search?keywords=bash-completion), [Arch](https://archlinux.org/packages/extra/any/bash-completion/), [Gentoo](https://packages.gentoo.org/packages/app-shells/bash-completion)) package installed, and restart your bash session. +You will need to have the [bash-completion](https://github.com/scop/bash-completion) ( +e.g., [Ubuntu](https://packages.ubuntu.com/search?keywords=bash-completion), [Arch](https://archlinux.org/packages/extra/any/bash-completion/), [Gentoo](https://packages.gentoo.org/packages/app-shells/bash-completion)) package installed, and restart +your bash session. To load completions for each session, execute once: @@ -157,7 +166,6 @@ To load completions for each session, execute once: `epcc completion fish > ~/.config/fish/completions/epcc.fish` - ## Tips ### JQ Output @@ -207,6 +215,7 @@ this is based on [GoJQ which has a number of differences](https://github.com/itc The `--retry-while-jq` argument can be used to wait for certain conditions to happen (e.g., a catalog publication, or an eventual consistency condition). For example: + ```bash epcc get pcm-catalog-release --retry-while-jq '.data.meta.release_status != "PUBLISHED"' name=Ranges_Catalog last_release ``` @@ -219,6 +228,7 @@ this is based on [GoJQ which has a number of differences](https://github.com/itc ```bash epcc runbooks run misc get-store-info ``` + ## Development Tips ### Fast rebuilds diff --git a/external/httpclient/httpclient.go b/external/httpclient/httpclient.go index 8401db2..5b8f6e6 100644 --- a/external/httpclient/httpclient.go +++ b/external/httpclient/httpclient.go @@ -18,6 +18,7 @@ import ( "net/http/httputil" "net/url" "os" + "regexp" "runtime" "sort" "strings" @@ -29,6 +30,12 @@ var RawHeaders []string const EnvNameHttpPrefix = "EPCC_CLI_HTTP_HEADER_" +const EnvUrlMatch = "EPCC_CLI_URL_MATCH_REGEXP_(\\d+)" + +const EnvUrlMatchPrefix = "EPCC_CLI_URL_MATCH_SUBSTITUTION_" + +var urlSubstitions = map[*regexp.Regexp]string{} + var httpHeaders = map[string]string{} var DontLog2xxs = false @@ -43,6 +50,9 @@ var stats = struct { func init() { stats.respCodes = make(map[int]int) + + urlMatchRegexp := regexp.MustCompile(EnvUrlMatch) + for _, env := range os.Environ() { splitEnv := strings.SplitN(env, "=", 2) @@ -58,6 +68,18 @@ func init() { httpHeaders[headersSplit[0]] = headersSplit[1] } } + + if groups := urlMatchRegexp.FindStringSubmatch(envName); groups != nil { + if groups != nil { + r, err := regexp.Compile(envValue) + + if err != nil { + log.Warnf("Environment variable %s has a malformed regex and substition cannot be performed, %v", env, err) + } else { + urlSubstitions[r] = os.Getenv(EnvUrlMatchPrefix + groups[1]) + } + } + } } } } @@ -181,7 +203,21 @@ func doRequestInternal(ctx context.Context, method string, contentType string, p log.Fatalf("Error when parsing default host, this is a bug, %s", config.DefaultUrl) } } + + origPath := path + + for r, substitution := range urlSubstitions { + if r.MatchString(path) { + path = r.ReplaceAllString(path, substitution) + } + } + + if origPath != path { + log.Trace("URL Replacement transformed %s to %s", origPath, path) + } + reqURL.Path = path + reqURL.RawQuery = query var bodyBuf []byte diff --git a/external/resources/resources.go b/external/resources/resources.go index 6e381e0..4d50d7d 100644 --- a/external/resources/resources.go +++ b/external/resources/resources.go @@ -20,7 +20,7 @@ var topLevelResourceRegexp = regexp.MustCompile("^/v2/[^/]+/\\{[^}]+}/?$") type Resource struct { // The type as far as the EPCC CLI is concerned. - Type string + Type string `yaml:"-"` // A link to the generic documentation page about a type in the EPCC API Docs string `yaml:"docs"` @@ -29,40 +29,40 @@ type Resource struct { JsonApiType string `yaml:"json-api-type"` // Alterative types used for aliases - AlternateJsonApiTypesForAliases []string `yaml:"alternate-json-type-for-aliases"` + AlternateJsonApiTypesForAliases []string `yaml:"alternate-json-type-for-aliases,omitempty"` // Some resources (e.g., PCM, accelerator svc, bury most attributes under the attributes key). This is considered "compliant", other services just bury attributes under data, this is "legacy. JsonApiFormat string `yaml:"json-api-format"` // Information about how to get a collection - GetCollectionInfo *CrudEntityInfo `yaml:"get-collection"` + GetCollectionInfo *CrudEntityInfo `yaml:"get-collection,omitempty"` // Information about how to get a single object. - GetEntityInfo *CrudEntityInfo `yaml:"get-entity"` + GetEntityInfo *CrudEntityInfo `yaml:"get-entity,omitempty"` // Information about how to create an entity. - CreateEntityInfo *CrudEntityInfo `yaml:"create-entity"` + CreateEntityInfo *CrudEntityInfo `yaml:"create-entity,omitempty"` // Information about how to update an entity. - UpdateEntityInfo *CrudEntityInfo `yaml:"update-entity"` + UpdateEntityInfo *CrudEntityInfo `yaml:"update-entity,omitempty"` // Information about how to delete an entity. - DeleteEntityInfo *CrudEntityInfo `yaml:"delete-entity"` + DeleteEntityInfo *CrudEntityInfo `yaml:"delete-entity,omitempty"` - Attributes map[string]*CrudEntityAttribute `yaml:"attributes"` + Attributes map[string]*CrudEntityAttribute `yaml:"attributes,omitempty"` // If true, don't wrap json in a data tag - NoWrapping bool `yaml:"no-wrapping"` + NoWrapping bool `yaml:"no-wrapping,omitempty"` // The singular name version of the resource. SingularName string `yaml:"singular-name"` - PluralName string + PluralName string `yaml:"-"` // Use this value to silence warnings about a resource not supporting resets. // This should only be used for cases where we manually fix things, or where // a store reset would clear a resource another way (e.g., the resource represents a projection). - SuppressResetWarning bool `yaml:"suppress-reset-warning"` + SuppressResetWarning bool `yaml:"suppress-reset-warning,omitempty"` } type CrudEntityInfo struct { @@ -74,26 +74,26 @@ type CrudEntityInfo struct { Url string `yaml:"url"` // Content type to send - ContentType string `yaml:"content-type"` + ContentType string `yaml:"content-type,omitempty"` // A list of valid query parameters - QueryParameters string `yaml:"query"` + QueryParameters string `yaml:"query,omitempty"` // Minimum resources so we don't keep trying to delete in - MinResources int `yaml:"min"` + MinResources int `yaml:"min,omitempty"` // Override the attribute we use in the URL for a specific key - ParentResourceValueOverrides map[string]string `yaml:"parent_resource_value_overrides"` + ParentResourceValueOverrides map[string]string `yaml:"parent_resource_value_overrides,omitempty"` } type CrudEntityAttribute struct { // The name of the attribute - Key string + Key string `yaml:"-"` // The type of the attribute Type string `yaml:"type"` - AutoFill string `yaml:"autofill"` - AliasAttribute string `yaml:"alias_attribute"` + AutoFill string `yaml:"autofill,omitempty"` + AliasAttribute string `yaml:"alias_attribute,omitempty"` } func GetPluralResourceNames() []string {