From c237fd6a490e27211cc51b6708765e7a04cf9912 Mon Sep 17 00:00:00 2001 From: Muhammed Efe Cetin Date: Sun, 6 Aug 2023 14:40:20 +0300 Subject: [PATCH 1/3] :fire: add config to enable splitting by comma in parsers :fire: add config to enable splitting by comma in parsers --- app.go | 9 ++++++ ctx.go | 25 ++++++++------- ctx_test.go | 70 +++++++++++++++++++++++++++++++++++++--- docs/api/fiber.md | 81 ++++++++++++++++++++++++----------------------- path_test.go | 11 +++++++ 5 files changed, 140 insertions(+), 56 deletions(-) diff --git a/app.go b/app.go index e15383a63e..8a2d9f1e72 100644 --- a/app.go +++ b/app.go @@ -390,6 +390,15 @@ type Config struct { // // Optional. Default: DefaultMethods RequestMethods []string + + // EnableSplittingOnParsers splits the query/body/header parameters by comma when it's true. + // For example, you can use it to parse multiple values from a query parameter like this: + // /api?foo=bar,baz == foo[]=bar&foo[]=baz + // If there is an escape character before the comma, parsers won't split it like the example below: + // /api?foo=bar\,baz == foo=bar,baz + // + // Optional. Default: false + EnableSplittingOnParsers bool `json:"enable_splitting_on_parsers"` } // Static defines configuration options when defining static assets. diff --git a/ctx.go b/ctx.go index 901b51744c..dc5c511399 100644 --- a/ctx.go +++ b/ctx.go @@ -335,13 +335,13 @@ func (c *Ctx) BodyParser(out interface{}) error { k, err = parseParamSquareBrackets(k) } - if strings.Contains(v, ",") && equalFieldType(out, reflect.Slice, k) { - values := strings.Split(v, ",") + if findNextNonEscapedCharsetPosition(v, []byte(",")) != -1 && equalFieldType(out, reflect.Slice, k) && c.app.config.EnableSplittingOnParsers { + values := splitNonEscaped(v, ",") for i := 0; i < len(values); i++ { - data[k] = append(data[k], values[i]) + data[k] = append(data[k], RemoveEscapeChar(values[i])) } } else { - data[k] = append(data[k], v) + data[k] = append(data[k], RemoveEscapeChar(v)) } }) @@ -1159,13 +1159,13 @@ func (c *Ctx) QueryParser(out interface{}) error { k, err = parseParamSquareBrackets(k) } - if strings.Contains(v, ",") && equalFieldType(out, reflect.Slice, k) { - values := strings.Split(v, ",") + if findNextNonEscapedCharsetPosition(v, []byte(",")) != -1 && equalFieldType(out, reflect.Slice, k) && c.app.config.EnableSplittingOnParsers { + values := splitNonEscaped(v, ",") for i := 0; i < len(values); i++ { - data[k] = append(data[k], values[i]) + data[k] = append(data[k], RemoveEscapeChar(values[i])) } } else { - data[k] = append(data[k], v) + data[k] = append(data[k], RemoveEscapeChar(v)) } }) @@ -1208,13 +1208,14 @@ func (c *Ctx) ReqHeaderParser(out interface{}) error { k := c.app.getString(key) v := c.app.getString(val) - if strings.Contains(v, ",") && equalFieldType(out, reflect.Slice, k) { - values := strings.Split(v, ",") + if findNextNonEscapedCharsetPosition(v, []byte(",")) != -1 && equalFieldType(out, reflect.Slice, k) && c.app.config.EnableSplittingOnParsers { + values := splitNonEscaped(v, ",") for i := 0; i < len(values); i++ { - data[k] = append(data[k], values[i]) + data[k] = append(data[k], RemoveEscapeChar(values[i])) } } else { - data[k] = append(data[k], v) + //fmt.Println(v) + data[k] = append(data[k], RemoveEscapeChar(v)) } }) diff --git a/ctx_test.go b/ctx_test.go index 38e83ca9e2..fa54d0af2f 100644 --- a/ctx_test.go +++ b/ctx_test.go @@ -3917,7 +3917,7 @@ func Benchmark_Ctx_Queries(b *testing.B) { // go test -run Test_Ctx_QueryParser -v func Test_Ctx_QueryParser(t *testing.T) { t.Parallel() - app := New() + app := New(Config{EnableSplittingOnParsers: true}) c := app.AcquireCtx(&fasthttp.RequestCtx{}) defer app.ReleaseCtx(c) type Query struct { @@ -3937,6 +3937,11 @@ func Test_Ctx_QueryParser(t *testing.T) { utils.AssertEqual(t, nil, c.QueryParser(q)) utils.AssertEqual(t, 2, len(q.Hobby)) + c.Request().URI().SetQueryString("id=1&name=tom&hobby=basketball\\,football") + q = new(Query) + utils.AssertEqual(t, nil, c.QueryParser(q)) + utils.AssertEqual(t, 1, len(q.Hobby)) + c.Request().URI().SetQueryString("id=1&name=tom&hobby=scoccer&hobby=basketball,football") q = new(Query) utils.AssertEqual(t, nil, c.QueryParser(q)) @@ -3988,6 +3993,29 @@ func Test_Ctx_QueryParser(t *testing.T) { utils.AssertEqual(t, 2, len(aq.Data)) } +// go test -run Test_Ctx_QueryParser -v +func Test_Ctx_QueryParser_WithoutSplitting(t *testing.T) { + t.Parallel() + app := New() + c := app.AcquireCtx(&fasthttp.RequestCtx{}) + defer app.ReleaseCtx(c) + type Query struct { + ID int + Name string + Hobby []string + } + + c.Request().URI().SetQueryString("id=1&name=tom&hobby=basketball,football") + q := new(Query) + utils.AssertEqual(t, nil, c.QueryParser(q)) + utils.AssertEqual(t, 1, len(q.Hobby)) + + c.Request().URI().SetQueryString("id=1&name=tom&hobby=scoccer&hobby=basketball,football") + q = new(Query) + utils.AssertEqual(t, nil, c.QueryParser(q)) + utils.AssertEqual(t, 2, len(q.Hobby)) +} + // go test -run Test_Ctx_QueryParser_WithSetParserDecoder -v func Test_Ctx_QueryParser_WithSetParserDecoder(t *testing.T) { t.Parallel() @@ -4146,7 +4174,7 @@ func Test_Ctx_QueryParser_Schema(t *testing.T) { // go test -run Test_Ctx_ReqHeaderParser -v func Test_Ctx_ReqHeaderParser(t *testing.T) { t.Parallel() - app := New() + app := New(Config{EnableSplittingOnParsers: true}) c := app.AcquireCtx(&fasthttp.RequestCtx{}) defer app.ReleaseCtx(c) type Header struct { @@ -4170,6 +4198,12 @@ func Test_Ctx_ReqHeaderParser(t *testing.T) { utils.AssertEqual(t, nil, c.ReqHeaderParser(q)) utils.AssertEqual(t, 3, len(q.Hobby)) + c.Request().Header.Del("hobby") + c.Request().Header.Add("Hobby", "golang\\,fiber\\,go,fiber") + q = new(Header) + utils.AssertEqual(t, nil, c.ReqHeaderParser(q)) + utils.AssertEqual(t, 2, len(q.Hobby)) + empty := new(Header) c.Request().Header.Del("hobby") utils.AssertEqual(t, nil, c.QueryParser(empty)) @@ -4215,6 +4249,34 @@ func Test_Ctx_ReqHeaderParser(t *testing.T) { utils.AssertEqual(t, "failed to decode: name is empty", c.ReqHeaderParser(rh).Error()) } +// go test -run Test_Ctx_ReqHeaderParser -v +func Test_Ctx_ReqHeaderParser_WithoutSplitting(t *testing.T) { + t.Parallel() + app := New() + c := app.AcquireCtx(&fasthttp.RequestCtx{}) + defer app.ReleaseCtx(c) + type Header struct { + ID int + Name string + Hobby []string + } + c.Request().SetBody([]byte(``)) + c.Request().Header.SetContentType("") + + c.Request().Header.Add("id", "1") + c.Request().Header.Add("Name", "John Doe") + c.Request().Header.Add("Hobby", "golang,fiber") + q := new(Header) + utils.AssertEqual(t, nil, c.ReqHeaderParser(q)) + utils.AssertEqual(t, 1, len(q.Hobby)) + + c.Request().Header.Del("hobby") + c.Request().Header.Add("Hobby", "golang,fiber,go") + q = new(Header) + utils.AssertEqual(t, nil, c.ReqHeaderParser(q)) + utils.AssertEqual(t, 1, len(q.Hobby)) +} + // go test -run Test_Ctx_ReqHeaderParser_WithSetParserDecoder -v func Test_Ctx_ReqHeaderParser_WithSetParserDecoder(t *testing.T) { t.Parallel() @@ -4439,7 +4501,7 @@ func Benchmark_Ctx_parseQuery(b *testing.B) { // go test -v -run=^$ -bench=Benchmark_Ctx_QueryParser_Comma -benchmem -count=4 func Benchmark_Ctx_QueryParser_Comma(b *testing.B) { - app := New() + app := New(Config{EnableSplittingOnParsers: true}) c := app.AcquireCtx(&fasthttp.RequestCtx{}) defer app.ReleaseCtx(c) type Query struct { @@ -4465,7 +4527,7 @@ func Benchmark_Ctx_QueryParser_Comma(b *testing.B) { // go test -v -run=^$ -bench=Benchmark_Ctx_ReqHeaderParser -benchmem -count=4 func Benchmark_Ctx_ReqHeaderParser(b *testing.B) { - app := New() + app := New(Config{EnableSplittingOnParsers: true}) c := app.AcquireCtx(&fasthttp.RequestCtx{}) defer app.ReleaseCtx(c) type ReqHeader struct { diff --git a/docs/api/fiber.md b/docs/api/fiber.md index fa0ad2412d..9f092fed47 100644 --- a/docs/api/fiber.md +++ b/docs/api/fiber.md @@ -39,47 +39,48 @@ app := fiber.New(fiber.Config{ **Config fields** -| Property | Type | Description | Default | -| ---------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------- | -| AppName | `string` | This allows to setup app name for the app | `""` | -| BodyLimit | `int` | Sets the maximum allowed size for a request body, if the size exceeds the configured limit, it sends `413 - Request Entity Too Large` response. | `4 * 1024 * 1024` | -| CaseSensitive | `bool` | When enabled, `/Foo` and `/foo` are different routes. When disabled, `/Foo`and `/foo` are treated the same. | `false` | -| ColorScheme | [`Colors`](https://github.com/gofiber/fiber/blob/master/color.go) | You can define custom color scheme. They'll be used for startup message, route list and some middlewares. | [`DefaultColors`](https://github.com/gofiber/fiber/blob/master/color.go) | -| CompressedFileSuffix | `string` | Adds a suffix to the original file name and tries saving the resulting compressed file under the new file name. | `".fiber.gz"` | -| Concurrency | `int` | Maximum number of concurrent connections. | `256 * 1024` | -| DisableDefaultContentType | `bool` | When set to true, causes the default Content-Type header to be excluded from the Response. | `false` | -| DisableDefaultDate | `bool` | When set to true causes the default date header to be excluded from the response. | `false` | -| DisableHeaderNormalizing | `bool` | By default all header names are normalized: conteNT-tYPE -> Content-Type | `false` | -| DisableKeepalive | `bool` | Disable keep-alive connections, the server will close incoming connections after sending the first response to the client | `false` | -| DisablePreParseMultipartForm | `bool` | Will not pre parse Multipart Form data if set to true. This option is useful for servers that desire to treat multipart form data as a binary blob, or choose when to parse the data. | `false` | -| DisableStartupMessage | `bool` | When set to true, it will not print out debug information | `false` | -| ETag | `bool` | Enable or disable ETag header generation, since both weak and strong etags are generated using the same hashing method \(CRC-32\). Weak ETags are the default when enabled. | `false` | -| EnableIPValidation | `bool` | If set to true, `c.IP()` and `c.IPs()` will validate IP addresses before returning them. Also, `c.IP()` will return only the first valid IP rather than just the raw header value that may be a comma seperated string.

**WARNING:** There is a small performance cost to doing this validation. Keep disabled if speed is your only concern and your application is behind a trusted proxy that already validates this header. | `false` | -| EnablePrintRoutes | `bool` | EnablePrintRoutes enables print all routes with their method, path, name and handler.. | `false` | +| Property | Type | Description | Default | +| ---------------------------- | --------------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------------------- | +| AppName | `string` | This allows to setup app name for the app | `""` | +| BodyLimit | `int` | Sets the maximum allowed size for a request body, if the size exceeds the configured limit, it sends `413 - Request Entity Too Large` response. | `4 * 1024 * 1024` | +| CaseSensitive | `bool` | When enabled, `/Foo` and `/foo` are different routes. When disabled, `/Foo`and `/foo` are treated the same. | `false` | +| ColorScheme | [`Colors`](https://github.com/gofiber/fiber/blob/master/color.go) | You can define custom color scheme. They'll be used for startup message, route list and some middlewares. | [`DefaultColors`](https://github.com/gofiber/fiber/blob/master/color.go) | +| CompressedFileSuffix | `string` | Adds a suffix to the original file name and tries saving the resulting compressed file under the new file name. | `".fiber.gz"` | +| Concurrency | `int` | Maximum number of concurrent connections. | `256 * 1024` | +| DisableDefaultContentType | `bool` | When set to true, causes the default Content-Type header to be excluded from the Response. | `false` | +| DisableDefaultDate | `bool` | When set to true causes the default date header to be excluded from the response. | `false` | +| DisableHeaderNormalizing | `bool` | By default all header names are normalized: conteNT-tYPE -> Content-Type | `false` | +| DisableKeepalive | `bool` | Disable keep-alive connections, the server will close incoming connections after sending the first response to the client | `false` | +| DisablePreParseMultipartForm | `bool` | Will not pre parse Multipart Form data if set to true. This option is useful for servers that desire to treat multipart form data as a binary blob, or choose when to parse the data. | `false` | +| DisableStartupMessage | `bool` | When set to true, it will not print out debug information | `false` | +| ETag | `bool` | Enable or disable ETag header generation, since both weak and strong etags are generated using the same hashing method \(CRC-32\). Weak ETags are the default when enabled. | `false` | +| EnableIPValidation | `bool` | If set to true, `c.IP()` and `c.IPs()` will validate IP addresses before returning them. Also, `c.IP()` will return only the first valid IP rather than just the raw header value that may be a comma seperated string.

**WARNING:** There is a small performance cost to doing this validation. Keep disabled if speed is your only concern and your application is behind a trusted proxy that already validates this header. | `false` | +| EnablePrintRoutes | `bool` | EnablePrintRoutes enables print all routes with their method, path, name and handler.. | `false` | +| EnableSplittingOnParsers | `bool` | EnableSplittingOnParsers splits the query/body/header parameters by comma when it's true.

For example, you can use it to parse multiple values from a query parameter like this: `/api?foo=bar,baz == foo[]=bar&foo[]=baz`

If there is an escape character before the comma, parsers won't split it like the example below: `/api?foo=bar\,baz == foo=bar,baz` | `false` | | EnableTrustedProxyCheck | `bool` | When set to true, fiber will check whether proxy is trusted, using TrustedProxies list.

By default `c.Protocol()` will get value from X-Forwarded-Proto, X-Forwarded-Protocol, X-Forwarded-Ssl or X-Url-Scheme header, `c.IP()` will get value from `ProxyHeader` header, `c.Hostname()` will get value from X-Forwarded-Host header.
If `EnableTrustedProxyCheck` is true, and `RemoteIP` is in the list of `TrustedProxies` `c.Protocol()`, `c.IP()`, and `c.Hostname()` will have the same behaviour when `EnableTrustedProxyCheck` disabled, if `RemoteIP` isn't in the list, `c.Protocol()` will return https in case when tls connection is handled by the app, or http otherwise, `c.IP()` will return RemoteIP() from fasthttp context, `c.Hostname()` will return `fasthttp.Request.URI().Host()` | `false` | -| ErrorHandler | `ErrorHandler` | ErrorHandler is executed when an error is returned from fiber.Handler. Mounted fiber error handlers are retained by the top-level app and applied on prefix associated requests. | `DefaultErrorHandler` | -| GETOnly | `bool` | Rejects all non-GET requests if set to true. This option is useful as anti-DoS protection for servers accepting only GET requests. The request size is limited by ReadBufferSize if GETOnly is set. | `false` | -| IdleTimeout | `time.Duration` | The maximum amount of time to wait for the next request when keep-alive is enabled. If IdleTimeout is zero, the value of ReadTimeout is used. | `nil` | -| Immutable | `bool` | When enabled, all values returned by context methods are immutable. By default, they are valid until you return from the handler; see issue [\#185](https://github.com/gofiber/fiber/issues/185). | `false` | -| JSONDecoder | `utils.JSONUnmarshal` | Allowing for flexibility in using another json library for decoding. | `json.Unmarshal` | -| JSONEncoder | `utils.JSONMarshal` | Allowing for flexibility in using another json library for encoding. | `json.Marshal` | -| Network | `string` | Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only)

**WARNING:** When prefork is set to true, only "tcp4" and "tcp6" can be chosen. | `NetworkTCP4` | -| PassLocalsToViews | `bool` | PassLocalsToViews Enables passing of the locals set on a fiber.Ctx to the template engine. See our **Template Middleware** for supported engines. | `false` | -| Prefork | `bool` | Enables use of the[`SO_REUSEPORT`](https://lwn.net/Articles/542629/)socket option. This will spawn multiple Go processes listening on the same port. learn more about [socket sharding](https://www.nginx.com/blog/socket-sharding-nginx-release-1-9-1/). **NOTE: if enabled, the application will need to be ran through a shell because prefork mode sets environment variables. If you're using Docker, make sure the app is ran with `CMD ./app` or `CMD ["sh", "-c", "/app"]`. For more info, see** [**this**](https://github.com/gofiber/fiber/issues/1021#issuecomment-730537971) **issue comment.** | `false` | -| ProxyHeader | `string` | This will enable `c.IP()` to return the value of the given header key. By default `c.IP()`will return the Remote IP from the TCP connection, this property can be useful if you are behind a load balancer e.g. _X-Forwarded-\*_. | `""` | -| ReadBufferSize | `int` | per-connection buffer size for requests' reading. This also limits the maximum header size. Increase this buffer if your clients send multi-KB RequestURIs and/or multi-KB headers \(for example, BIG cookies\). | `4096` | -| ReadTimeout | `time.Duration` | The amount of time allowed to read the full request, including the body. The default timeout is unlimited. | `nil` | -| RequestMethods | `[]string` | RequestMethods provides customizibility for HTTP methods. You can add/remove methods as you wish. | `DefaultMethods` | -| ServerHeader | `string` | Enables the `Server` HTTP header with the given value. | `""` | -| StreamRequestBody | `bool` | StreamRequestBody enables request body streaming, and calls the handler sooner when given body is larger then the current limit. | `false` | -| StrictRouting | `bool` | When enabled, the router treats `/foo` and `/foo/` as different. Otherwise, the router treats `/foo` and `/foo/` as the same. | `false` | -| TrustedProxies | `[]string` | Contains the list of trusted proxy IP's. Look at `EnableTrustedProxyCheck` doc.

It can take IP or IP range addresses. If it gets IP range, it iterates all possible addresses. | `[]string*__*` | -| UnescapePath | `bool` | Converts all encoded characters in the route back before setting the path for the context, so that the routing can also work with URL encoded special characters | `false` | -| Views | `Views` | Views is the interface that wraps the Render function. See our **Template Middleware** for supported engines. | `nil` | -| ViewsLayout | `string` | Views Layout is the global layout for all template render until override on Render function. See our **Template Middleware** for supported engines. | `""` | -| WriteBufferSize | `int` | Per-connection buffer size for responses' writing. | `4096` | -| WriteTimeout | `time.Duration` | The maximum duration before timing out writes of the response. The default timeout is unlimited. | `nil` | -| XMLEncoder | `utils.XMLMarshal` | Allowing for flexibility in using another XML library for encoding. | `xml.Marshal` | +| ErrorHandler | `ErrorHandler` | ErrorHandler is executed when an error is returned from fiber.Handler. Mounted fiber error handlers are retained by the top-level app and applied on prefix associated requests. | `DefaultErrorHandler` | +| GETOnly | `bool` | Rejects all non-GET requests if set to true. This option is useful as anti-DoS protection for servers accepting only GET requests. The request size is limited by ReadBufferSize if GETOnly is set. | `false` | +| IdleTimeout | `time.Duration` | The maximum amount of time to wait for the next request when keep-alive is enabled. If IdleTimeout is zero, the value of ReadTimeout is used. | `nil` | +| Immutable | `bool` | When enabled, all values returned by context methods are immutable. By default, they are valid until you return from the handler; see issue [\#185](https://github.com/gofiber/fiber/issues/185). | `false` | +| JSONDecoder | `utils.JSONUnmarshal` | Allowing for flexibility in using another json library for decoding. | `json.Unmarshal` | +| JSONEncoder | `utils.JSONMarshal` | Allowing for flexibility in using another json library for encoding. | `json.Marshal` | +| Network | `string` | Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only)

**WARNING:** When prefork is set to true, only "tcp4" and "tcp6" can be chosen. | `NetworkTCP4` | +| PassLocalsToViews | `bool` | PassLocalsToViews Enables passing of the locals set on a fiber.Ctx to the template engine. See our **Template Middleware** for supported engines. | `false` | +| Prefork | `bool` | Enables use of the[`SO_REUSEPORT`](https://lwn.net/Articles/542629/)socket option. This will spawn multiple Go processes listening on the same port. learn more about [socket sharding](https://www.nginx.com/blog/socket-sharding-nginx-release-1-9-1/). **NOTE: if enabled, the application will need to be ran through a shell because prefork mode sets environment variables. If you're using Docker, make sure the app is ran with `CMD ./app` or `CMD ["sh", "-c", "/app"]`. For more info, see** [**this**](https://github.com/gofiber/fiber/issues/1021#issuecomment-730537971) **issue comment.** | `false` | +| ProxyHeader | `string` | This will enable `c.IP()` to return the value of the given header key. By default `c.IP()`will return the Remote IP from the TCP connection, this property can be useful if you are behind a load balancer e.g. _X-Forwarded-\*_. | `""` | +| ReadBufferSize | `int` | per-connection buffer size for requests' reading. This also limits the maximum header size. Increase this buffer if your clients send multi-KB RequestURIs and/or multi-KB headers \(for example, BIG cookies\). | `4096` | +| ReadTimeout | `time.Duration` | The amount of time allowed to read the full request, including the body. The default timeout is unlimited. | `nil` | +| RequestMethods | `[]string` | RequestMethods provides customizibility for HTTP methods. You can add/remove methods as you wish. | `DefaultMethods` | +| ServerHeader | `string` | Enables the `Server` HTTP header with the given value. | `""` | +| StreamRequestBody | `bool` | StreamRequestBody enables request body streaming, and calls the handler sooner when given body is larger then the current limit. | `false` | +| StrictRouting | `bool` | When enabled, the router treats `/foo` and `/foo/` as different. Otherwise, the router treats `/foo` and `/foo/` as the same. | `false` | +| TrustedProxies | `[]string` | Contains the list of trusted proxy IP's. Look at `EnableTrustedProxyCheck` doc.

It can take IP or IP range addresses. If it gets IP range, it iterates all possible addresses. | `[]string*__*` | +| UnescapePath | `bool` | Converts all encoded characters in the route back before setting the path for the context, so that the routing can also work with URL encoded special characters | `false` | +| Views | `Views` | Views is the interface that wraps the Render function. See our **Template Middleware** for supported engines. | `nil` | +| ViewsLayout | `string` | Views Layout is the global layout for all template render until override on Render function. See our **Template Middleware** for supported engines. | `""` | +| WriteBufferSize | `int` | Per-connection buffer size for responses' writing. | `4096` | +| WriteTimeout | `time.Duration` | The maximum duration before timing out writes of the response. The default timeout is unlimited. | `nil` | +| XMLEncoder | `utils.XMLMarshal` | Allowing for flexibility in using another XML library for encoding. | `xml.Marshal` | ## NewError diff --git a/path_test.go b/path_test.go index 65dfdd6f72..d0d11d07a2 100644 --- a/path_test.go +++ b/path_test.go @@ -195,6 +195,17 @@ func Test_Utils_RemoveEscapeChar(t *testing.T) { utils.AssertEqual(t, "noEscapeChar", res) } +func Benchmark_Utils_RemoveEscapeChar(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + var res string + for n := 0; n < b.N; n++ { + res = RemoveEscapeChar(":test\\:bla") + } + + utils.AssertEqual(b, ":test:bla", res) +} + // go test -race -run Test_Path_matchParams func Benchmark_Path_matchParams(t *testing.B) { var ctxParams [maxParams]string From a30dd6fce49758fc2a5fc7330df3e6b0960d5786 Mon Sep 17 00:00:00 2001 From: Muhammed Efe Cetin Date: Sat, 12 Aug 2023 14:07:11 +0300 Subject: [PATCH 2/3] optimize if statements, remove escape char support optimize if statements, remove escape char support --- app.go | 2 -- ctx.go | 25 ++++++++++++------------- ctx_test.go | 11 ----------- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/app.go b/app.go index 8a2d9f1e72..ca598fce9e 100644 --- a/app.go +++ b/app.go @@ -394,8 +394,6 @@ type Config struct { // EnableSplittingOnParsers splits the query/body/header parameters by comma when it's true. // For example, you can use it to parse multiple values from a query parameter like this: // /api?foo=bar,baz == foo[]=bar&foo[]=baz - // If there is an escape character before the comma, parsers won't split it like the example below: - // /api?foo=bar\,baz == foo=bar,baz // // Optional. Default: false EnableSplittingOnParsers bool `json:"enable_splitting_on_parsers"` diff --git a/ctx.go b/ctx.go index dc5c511399..97c83d1598 100644 --- a/ctx.go +++ b/ctx.go @@ -335,13 +335,13 @@ func (c *Ctx) BodyParser(out interface{}) error { k, err = parseParamSquareBrackets(k) } - if findNextNonEscapedCharsetPosition(v, []byte(",")) != -1 && equalFieldType(out, reflect.Slice, k) && c.app.config.EnableSplittingOnParsers { - values := splitNonEscaped(v, ",") + if c.app.config.EnableSplittingOnParsers && strings.Contains(v, ",") && equalFieldType(out, reflect.Slice, k) { + values := strings.Split(v, ",") for i := 0; i < len(values); i++ { - data[k] = append(data[k], RemoveEscapeChar(values[i])) + data[k] = append(data[k], values[i]) } } else { - data[k] = append(data[k], RemoveEscapeChar(v)) + data[k] = append(data[k], v) } }) @@ -1159,13 +1159,13 @@ func (c *Ctx) QueryParser(out interface{}) error { k, err = parseParamSquareBrackets(k) } - if findNextNonEscapedCharsetPosition(v, []byte(",")) != -1 && equalFieldType(out, reflect.Slice, k) && c.app.config.EnableSplittingOnParsers { - values := splitNonEscaped(v, ",") + if c.app.config.EnableSplittingOnParsers && strings.Contains(v, ",") && equalFieldType(out, reflect.Slice, k) { + values := strings.Split(v, ",") for i := 0; i < len(values); i++ { - data[k] = append(data[k], RemoveEscapeChar(values[i])) + data[k] = append(data[k], values[i]) } } else { - data[k] = append(data[k], RemoveEscapeChar(v)) + data[k] = append(data[k], v) } }) @@ -1208,14 +1208,13 @@ func (c *Ctx) ReqHeaderParser(out interface{}) error { k := c.app.getString(key) v := c.app.getString(val) - if findNextNonEscapedCharsetPosition(v, []byte(",")) != -1 && equalFieldType(out, reflect.Slice, k) && c.app.config.EnableSplittingOnParsers { - values := splitNonEscaped(v, ",") + if c.app.config.EnableSplittingOnParsers && strings.Contains(v, ",") && equalFieldType(out, reflect.Slice, k) { + values := strings.Split(v, ",") for i := 0; i < len(values); i++ { - data[k] = append(data[k], RemoveEscapeChar(values[i])) + data[k] = append(data[k], values[i]) } } else { - //fmt.Println(v) - data[k] = append(data[k], RemoveEscapeChar(v)) + data[k] = append(data[k], v) } }) diff --git a/ctx_test.go b/ctx_test.go index fa54d0af2f..a34778dec6 100644 --- a/ctx_test.go +++ b/ctx_test.go @@ -3937,11 +3937,6 @@ func Test_Ctx_QueryParser(t *testing.T) { utils.AssertEqual(t, nil, c.QueryParser(q)) utils.AssertEqual(t, 2, len(q.Hobby)) - c.Request().URI().SetQueryString("id=1&name=tom&hobby=basketball\\,football") - q = new(Query) - utils.AssertEqual(t, nil, c.QueryParser(q)) - utils.AssertEqual(t, 1, len(q.Hobby)) - c.Request().URI().SetQueryString("id=1&name=tom&hobby=scoccer&hobby=basketball,football") q = new(Query) utils.AssertEqual(t, nil, c.QueryParser(q)) @@ -4198,12 +4193,6 @@ func Test_Ctx_ReqHeaderParser(t *testing.T) { utils.AssertEqual(t, nil, c.ReqHeaderParser(q)) utils.AssertEqual(t, 3, len(q.Hobby)) - c.Request().Header.Del("hobby") - c.Request().Header.Add("Hobby", "golang\\,fiber\\,go,fiber") - q = new(Header) - utils.AssertEqual(t, nil, c.ReqHeaderParser(q)) - utils.AssertEqual(t, 2, len(q.Hobby)) - empty := new(Header) c.Request().Header.Del("hobby") utils.AssertEqual(t, nil, c.QueryParser(empty)) From 8fba3d702db53b9212f5d36bbda14e9cc6883fab Mon Sep 17 00:00:00 2001 From: Muhammed Efe Cetin Date: Sun, 20 Aug 2023 16:11:31 +0300 Subject: [PATCH 3/3] update --- docs/api/fiber.md | 82 +++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/docs/api/fiber.md b/docs/api/fiber.md index 9f092fed47..1f9a91b8bf 100644 --- a/docs/api/fiber.md +++ b/docs/api/fiber.md @@ -39,48 +39,48 @@ app := fiber.New(fiber.Config{ **Config fields** -| Property | Type | Description | Default | -| ---------------------------- | --------------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------------------- | -| AppName | `string` | This allows to setup app name for the app | `""` | -| BodyLimit | `int` | Sets the maximum allowed size for a request body, if the size exceeds the configured limit, it sends `413 - Request Entity Too Large` response. | `4 * 1024 * 1024` | -| CaseSensitive | `bool` | When enabled, `/Foo` and `/foo` are different routes. When disabled, `/Foo`and `/foo` are treated the same. | `false` | -| ColorScheme | [`Colors`](https://github.com/gofiber/fiber/blob/master/color.go) | You can define custom color scheme. They'll be used for startup message, route list and some middlewares. | [`DefaultColors`](https://github.com/gofiber/fiber/blob/master/color.go) | -| CompressedFileSuffix | `string` | Adds a suffix to the original file name and tries saving the resulting compressed file under the new file name. | `".fiber.gz"` | -| Concurrency | `int` | Maximum number of concurrent connections. | `256 * 1024` | -| DisableDefaultContentType | `bool` | When set to true, causes the default Content-Type header to be excluded from the Response. | `false` | -| DisableDefaultDate | `bool` | When set to true causes the default date header to be excluded from the response. | `false` | -| DisableHeaderNormalizing | `bool` | By default all header names are normalized: conteNT-tYPE -> Content-Type | `false` | -| DisableKeepalive | `bool` | Disable keep-alive connections, the server will close incoming connections after sending the first response to the client | `false` | -| DisablePreParseMultipartForm | `bool` | Will not pre parse Multipart Form data if set to true. This option is useful for servers that desire to treat multipart form data as a binary blob, or choose when to parse the data. | `false` | -| DisableStartupMessage | `bool` | When set to true, it will not print out debug information | `false` | -| ETag | `bool` | Enable or disable ETag header generation, since both weak and strong etags are generated using the same hashing method \(CRC-32\). Weak ETags are the default when enabled. | `false` | -| EnableIPValidation | `bool` | If set to true, `c.IP()` and `c.IPs()` will validate IP addresses before returning them. Also, `c.IP()` will return only the first valid IP rather than just the raw header value that may be a comma seperated string.

**WARNING:** There is a small performance cost to doing this validation. Keep disabled if speed is your only concern and your application is behind a trusted proxy that already validates this header. | `false` | -| EnablePrintRoutes | `bool` | EnablePrintRoutes enables print all routes with their method, path, name and handler.. | `false` | -| EnableSplittingOnParsers | `bool` | EnableSplittingOnParsers splits the query/body/header parameters by comma when it's true.

For example, you can use it to parse multiple values from a query parameter like this: `/api?foo=bar,baz == foo[]=bar&foo[]=baz`

If there is an escape character before the comma, parsers won't split it like the example below: `/api?foo=bar\,baz == foo=bar,baz` | `false` | +| Property | Type | Description | Default | +| ---------------------------- | --------------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------------------- | +| AppName | `string` | This allows to setup app name for the app | `""` | +| BodyLimit | `int` | Sets the maximum allowed size for a request body, if the size exceeds the configured limit, it sends `413 - Request Entity Too Large` response. | `4 * 1024 * 1024` | +| CaseSensitive | `bool` | When enabled, `/Foo` and `/foo` are different routes. When disabled, `/Foo`and `/foo` are treated the same. | `false` | +| ColorScheme | [`Colors`](https://github.com/gofiber/fiber/blob/master/color.go) | You can define custom color scheme. They'll be used for startup message, route list and some middlewares. | [`DefaultColors`](https://github.com/gofiber/fiber/blob/master/color.go) | +| CompressedFileSuffix | `string` | Adds a suffix to the original file name and tries saving the resulting compressed file under the new file name. | `".fiber.gz"` | +| Concurrency | `int` | Maximum number of concurrent connections. | `256 * 1024` | +| DisableDefaultContentType | `bool` | When set to true, causes the default Content-Type header to be excluded from the Response. | `false` | +| DisableDefaultDate | `bool` | When set to true causes the default date header to be excluded from the response. | `false` | +| DisableHeaderNormalizing | `bool` | By default all header names are normalized: conteNT-tYPE -> Content-Type | `false` | +| DisableKeepalive | `bool` | Disable keep-alive connections, the server will close incoming connections after sending the first response to the client | `false` | +| DisablePreParseMultipartForm | `bool` | Will not pre parse Multipart Form data if set to true. This option is useful for servers that desire to treat multipart form data as a binary blob, or choose when to parse the data. | `false` | +| DisableStartupMessage | `bool` | When set to true, it will not print out debug information | `false` | +| ETag | `bool` | Enable or disable ETag header generation, since both weak and strong etags are generated using the same hashing method \(CRC-32\). Weak ETags are the default when enabled. | `false` | +| EnableIPValidation | `bool` | If set to true, `c.IP()` and `c.IPs()` will validate IP addresses before returning them. Also, `c.IP()` will return only the first valid IP rather than just the raw header value that may be a comma seperated string.

**WARNING:** There is a small performance cost to doing this validation. Keep disabled if speed is your only concern and your application is behind a trusted proxy that already validates this header. | `false` | +| EnablePrintRoutes | `bool` | EnablePrintRoutes enables print all routes with their method, path, name and handler.. | `false` | +| EnableSplittingOnParsers | `bool` | EnableSplittingOnParsers splits the query/body/header parameters by comma when it's true.

For example, you can use it to parse multiple values from a query parameter like this: `/api?foo=bar,baz == foo[]=bar&foo[]=baz` | `false` | | EnableTrustedProxyCheck | `bool` | When set to true, fiber will check whether proxy is trusted, using TrustedProxies list.

By default `c.Protocol()` will get value from X-Forwarded-Proto, X-Forwarded-Protocol, X-Forwarded-Ssl or X-Url-Scheme header, `c.IP()` will get value from `ProxyHeader` header, `c.Hostname()` will get value from X-Forwarded-Host header.
If `EnableTrustedProxyCheck` is true, and `RemoteIP` is in the list of `TrustedProxies` `c.Protocol()`, `c.IP()`, and `c.Hostname()` will have the same behaviour when `EnableTrustedProxyCheck` disabled, if `RemoteIP` isn't in the list, `c.Protocol()` will return https in case when tls connection is handled by the app, or http otherwise, `c.IP()` will return RemoteIP() from fasthttp context, `c.Hostname()` will return `fasthttp.Request.URI().Host()` | `false` | -| ErrorHandler | `ErrorHandler` | ErrorHandler is executed when an error is returned from fiber.Handler. Mounted fiber error handlers are retained by the top-level app and applied on prefix associated requests. | `DefaultErrorHandler` | -| GETOnly | `bool` | Rejects all non-GET requests if set to true. This option is useful as anti-DoS protection for servers accepting only GET requests. The request size is limited by ReadBufferSize if GETOnly is set. | `false` | -| IdleTimeout | `time.Duration` | The maximum amount of time to wait for the next request when keep-alive is enabled. If IdleTimeout is zero, the value of ReadTimeout is used. | `nil` | -| Immutable | `bool` | When enabled, all values returned by context methods are immutable. By default, they are valid until you return from the handler; see issue [\#185](https://github.com/gofiber/fiber/issues/185). | `false` | -| JSONDecoder | `utils.JSONUnmarshal` | Allowing for flexibility in using another json library for decoding. | `json.Unmarshal` | -| JSONEncoder | `utils.JSONMarshal` | Allowing for flexibility in using another json library for encoding. | `json.Marshal` | -| Network | `string` | Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only)

**WARNING:** When prefork is set to true, only "tcp4" and "tcp6" can be chosen. | `NetworkTCP4` | -| PassLocalsToViews | `bool` | PassLocalsToViews Enables passing of the locals set on a fiber.Ctx to the template engine. See our **Template Middleware** for supported engines. | `false` | -| Prefork | `bool` | Enables use of the[`SO_REUSEPORT`](https://lwn.net/Articles/542629/)socket option. This will spawn multiple Go processes listening on the same port. learn more about [socket sharding](https://www.nginx.com/blog/socket-sharding-nginx-release-1-9-1/). **NOTE: if enabled, the application will need to be ran through a shell because prefork mode sets environment variables. If you're using Docker, make sure the app is ran with `CMD ./app` or `CMD ["sh", "-c", "/app"]`. For more info, see** [**this**](https://github.com/gofiber/fiber/issues/1021#issuecomment-730537971) **issue comment.** | `false` | -| ProxyHeader | `string` | This will enable `c.IP()` to return the value of the given header key. By default `c.IP()`will return the Remote IP from the TCP connection, this property can be useful if you are behind a load balancer e.g. _X-Forwarded-\*_. | `""` | -| ReadBufferSize | `int` | per-connection buffer size for requests' reading. This also limits the maximum header size. Increase this buffer if your clients send multi-KB RequestURIs and/or multi-KB headers \(for example, BIG cookies\). | `4096` | -| ReadTimeout | `time.Duration` | The amount of time allowed to read the full request, including the body. The default timeout is unlimited. | `nil` | -| RequestMethods | `[]string` | RequestMethods provides customizibility for HTTP methods. You can add/remove methods as you wish. | `DefaultMethods` | -| ServerHeader | `string` | Enables the `Server` HTTP header with the given value. | `""` | -| StreamRequestBody | `bool` | StreamRequestBody enables request body streaming, and calls the handler sooner when given body is larger then the current limit. | `false` | -| StrictRouting | `bool` | When enabled, the router treats `/foo` and `/foo/` as different. Otherwise, the router treats `/foo` and `/foo/` as the same. | `false` | -| TrustedProxies | `[]string` | Contains the list of trusted proxy IP's. Look at `EnableTrustedProxyCheck` doc.

It can take IP or IP range addresses. If it gets IP range, it iterates all possible addresses. | `[]string*__*` | -| UnescapePath | `bool` | Converts all encoded characters in the route back before setting the path for the context, so that the routing can also work with URL encoded special characters | `false` | -| Views | `Views` | Views is the interface that wraps the Render function. See our **Template Middleware** for supported engines. | `nil` | -| ViewsLayout | `string` | Views Layout is the global layout for all template render until override on Render function. See our **Template Middleware** for supported engines. | `""` | -| WriteBufferSize | `int` | Per-connection buffer size for responses' writing. | `4096` | -| WriteTimeout | `time.Duration` | The maximum duration before timing out writes of the response. The default timeout is unlimited. | `nil` | -| XMLEncoder | `utils.XMLMarshal` | Allowing for flexibility in using another XML library for encoding. | `xml.Marshal` | +| ErrorHandler | `ErrorHandler` | ErrorHandler is executed when an error is returned from fiber.Handler. Mounted fiber error handlers are retained by the top-level app and applied on prefix associated requests. | `DefaultErrorHandler` | +| GETOnly | `bool` | Rejects all non-GET requests if set to true. This option is useful as anti-DoS protection for servers accepting only GET requests. The request size is limited by ReadBufferSize if GETOnly is set. | `false` | +| IdleTimeout | `time.Duration` | The maximum amount of time to wait for the next request when keep-alive is enabled. If IdleTimeout is zero, the value of ReadTimeout is used. | `nil` | +| Immutable | `bool` | When enabled, all values returned by context methods are immutable. By default, they are valid until you return from the handler; see issue [\#185](https://github.com/gofiber/fiber/issues/185). | `false` | +| JSONDecoder | `utils.JSONUnmarshal` | Allowing for flexibility in using another json library for decoding. | `json.Unmarshal` | +| JSONEncoder | `utils.JSONMarshal` | Allowing for flexibility in using another json library for encoding. | `json.Marshal` | +| Network | `string` | Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only)

**WARNING:** When prefork is set to true, only "tcp4" and "tcp6" can be chosen. | `NetworkTCP4` | +| PassLocalsToViews | `bool` | PassLocalsToViews Enables passing of the locals set on a fiber.Ctx to the template engine. See our **Template Middleware** for supported engines. | `false` | +| Prefork | `bool` | Enables use of the[`SO_REUSEPORT`](https://lwn.net/Articles/542629/)socket option. This will spawn multiple Go processes listening on the same port. learn more about [socket sharding](https://www.nginx.com/blog/socket-sharding-nginx-release-1-9-1/). **NOTE: if enabled, the application will need to be ran through a shell because prefork mode sets environment variables. If you're using Docker, make sure the app is ran with `CMD ./app` or `CMD ["sh", "-c", "/app"]`. For more info, see** [**this**](https://github.com/gofiber/fiber/issues/1021#issuecomment-730537971) **issue comment.** | `false` | +| ProxyHeader | `string` | This will enable `c.IP()` to return the value of the given header key. By default `c.IP()`will return the Remote IP from the TCP connection, this property can be useful if you are behind a load balancer e.g. _X-Forwarded-\*_. | `""` | +| ReadBufferSize | `int` | per-connection buffer size for requests' reading. This also limits the maximum header size. Increase this buffer if your clients send multi-KB RequestURIs and/or multi-KB headers \(for example, BIG cookies\). | `4096` | +| ReadTimeout | `time.Duration` | The amount of time allowed to read the full request, including the body. The default timeout is unlimited. | `nil` | +| RequestMethods | `[]string` | RequestMethods provides customizibility for HTTP methods. You can add/remove methods as you wish. | `DefaultMethods` | +| ServerHeader | `string` | Enables the `Server` HTTP header with the given value. | `""` | +| StreamRequestBody | `bool` | StreamRequestBody enables request body streaming, and calls the handler sooner when given body is larger then the current limit. | `false` | +| StrictRouting | `bool` | When enabled, the router treats `/foo` and `/foo/` as different. Otherwise, the router treats `/foo` and `/foo/` as the same. | `false` | +| TrustedProxies | `[]string` | Contains the list of trusted proxy IP's. Look at `EnableTrustedProxyCheck` doc.

It can take IP or IP range addresses. If it gets IP range, it iterates all possible addresses. | `[]string*__*` | +| UnescapePath | `bool` | Converts all encoded characters in the route back before setting the path for the context, so that the routing can also work with URL encoded special characters | `false` | +| Views | `Views` | Views is the interface that wraps the Render function. See our **Template Middleware** for supported engines. | `nil` | +| ViewsLayout | `string` | Views Layout is the global layout for all template render until override on Render function. See our **Template Middleware** for supported engines. | `""` | +| WriteBufferSize | `int` | Per-connection buffer size for responses' writing. | `4096` | +| WriteTimeout | `time.Duration` | The maximum duration before timing out writes of the response. The default timeout is unlimited. | `nil` | +| XMLEncoder | `utils.XMLMarshal` | Allowing for flexibility in using another XML library for encoding. | `xml.Marshal` | ## NewError