From 433227323d66149f9319d8dac3ebc573f7d9224b Mon Sep 17 00:00:00 2001 From: rinfx <893383980@qq.com> Date: Thu, 5 Dec 2024 18:39:00 +0800 Subject: [PATCH] extension mechanism for custom logs and span attributes (#1451) --- .../wasm-go/examples/custom-log/config.yaml | 68 ++++++++++++++++ plugins/wasm-go/examples/custom-log/go.mod | 20 +++++ plugins/wasm-go/examples/custom-log/go.sum | 20 +++++ plugins/wasm-go/examples/custom-log/main.go | 67 +++++++++++++++ plugins/wasm-go/pkg/wrapper/plugin_wrapper.go | 81 ++++++++++++++++++- plugins/wasm-go/pkg/wrapper/utils.go | 36 +++++++++ 6 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 plugins/wasm-go/examples/custom-log/config.yaml create mode 100644 plugins/wasm-go/examples/custom-log/go.mod create mode 100644 plugins/wasm-go/examples/custom-log/go.sum create mode 100644 plugins/wasm-go/examples/custom-log/main.go create mode 100644 plugins/wasm-go/pkg/wrapper/utils.go diff --git a/plugins/wasm-go/examples/custom-log/config.yaml b/plugins/wasm-go/examples/custom-log/config.yaml new file mode 100644 index 0000000000..0dd7e18815 --- /dev/null +++ b/plugins/wasm-go/examples/custom-log/config.yaml @@ -0,0 +1,68 @@ +static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 8080 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + log_format: + text_format_source: + inline_string: "{\"custom_log\":\"%FILTER_STATE(wasm.custom_log:PLAIN)%\",\"ai_log\":\"%FILTER_STATE(wasm.ai_log:PLAIN)%\"} + + " + path: /dev/stdout + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - name: get + match: + prefix: "/get" + route: + cluster: httpbin + http_filters: + - name: test + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + value: + config: + name: test + vm_config: + runtime: envoy.wasm.runtime.v8 + code: + local: + filename: main.wasm + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: {} + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: httpbin + connect_timeout: 600s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: httpbin + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: httpbin.org + port_value: 80 diff --git a/plugins/wasm-go/examples/custom-log/go.mod b/plugins/wasm-go/examples/custom-log/go.mod new file mode 100644 index 0000000000..5eacb33589 --- /dev/null +++ b/plugins/wasm-go/examples/custom-log/go.mod @@ -0,0 +1,20 @@ +module github.com/alibaba/higress/plugins/wasm-go/extensions/custom-logs + +go 1.18 + +replace github.com/alibaba/higress/plugins/wasm-go => ../.. + +require ( + github.com/alibaba/higress/plugins/wasm-go v0.0.0 + github.com/higress-group/proxy-wasm-go-sdk v1.0.0 +) + +require ( + github.com/google/uuid v1.3.0 // indirect + github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 // indirect + github.com/magefile/mage v1.14.0 // indirect + github.com/tidwall/gjson v1.17.3 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/resp v0.1.1 // indirect +) diff --git a/plugins/wasm-go/examples/custom-log/go.sum b/plugins/wasm-go/examples/custom-log/go.sum new file mode 100644 index 0000000000..b4ab172fe2 --- /dev/null +++ b/plugins/wasm-go/examples/custom-log/go.sum @@ -0,0 +1,20 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 h1:IHDghbGQ2DTIXHBHxWfqCYQW1fKjyJ/I7W1pMyUDeEA= +github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520/go.mod h1:Nz8ORLaFiLWotg6GeKlJMhv8cci8mM43uEnLA5t8iew= +github.com/higress-group/proxy-wasm-go-sdk v1.0.0 h1:BZRNf4R7jr9hwRivg/E29nkVaKEak5MWjBDhWjuHijU= +github.com/higress-group/proxy-wasm-go-sdk v1.0.0/go.mod h1:iiSyFbo+rAtbtGt/bsefv8GU57h9CCLYGJA74/tF5/0= +github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= +github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE= +github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/plugins/wasm-go/examples/custom-log/main.go b/plugins/wasm-go/examples/custom-log/main.go new file mode 100644 index 0000000000..f6801c380c --- /dev/null +++ b/plugins/wasm-go/examples/custom-log/main.go @@ -0,0 +1,67 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "math/rand" + + "github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types" + + "github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper" +) + +func main() { + wrapper.SetCtx( + "custom-log", + wrapper.ProcessRequestHeadersBy(onHttpRequestHeaders), + ) +} + +type CustomLogConfig struct { +} + +// Method 1: write custom log +func writeLog(ctx wrapper.HttpContext) { + ctx.SetUserAttribute("question", "当然可以。在Python中，你可以创建一个函数来计算一系列数字的和。下面是一个简单的例子，该函数接受一个数字列表作为输入，并返回它们的总和。\n\n```python\ndef sum_of_numbers(numbers):\n \"\"\"\n 计算列表中所有数字的和。\n \n 参数:\n numbers (list of int or float): 一个包含数字的列表。\n \n 返回:\n int or float: 列表中所有数字的总和。\n \"\"\"\n total_sum = sum(numbers) # 使用Python内置的sum函数计算总和\n return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers(numbers_list)) # 输出：The sum is: 15\n```\n\n在这段代码中，我们定义了一个名为 `sum_of_numbers` 的函数，它接收一个参数 `numbers`，这是一个包含整数或浮点数的列表。函数内部使用了Python的内置函数 `sum()` 来计算这些数字的总和，并将结果返回。\n\n你也可以手动实现求和逻辑，而不是使用内置的 `sum()` 函数，如下所示：\n\n```python\ndef sum_of_numbers_manual(numbers):\n \"\"\"\n 手动计算列表中所有数字的和。\n \n 参数:\n numbers (list of int or float): 一个包含数字的列表。\n \n 返回:\n int or float: 列表中所有数字的总和。\n \"\"\"\n total_sum = 0\n for number in numbers:\n total_sum += number\n return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers_manual(numbers_list)) # 输出：The sum is: 15\n```\n\n在这个版本中，我们初始化 `total_sum` 为0，然后遍历列表中的每个元素，并将其加到 `total_sum` 上。最后返回这个累加的结果。这两种方法都可以达到相同的目的，但是使用内置函数通常更简洁且效率更高。") + ctx.SetUserAttribute("k2", 2213.22) + ctx.WriteUserAttributeToLog() +} + +// Methods 2: write custom log with specific key +func writeLogWithKey(ctx wrapper.HttpContext, key string) { + ctx.SetUserAttribute("k2", 2213.22) + _ = ctx.WriteUserAttributeToLogWithKey(key) + ctx.SetUserAttribute("k2", 212939.22) + ctx.SetUserAttribute("k3", 123) + _ = ctx.WriteUserAttributeToLogWithKey(key) +} + +// Methods 2: write custom log with specific key +func writeTraceAttribute(ctx wrapper.HttpContext) { + ctx.SetUserAttribute("question", "当然可以。在Python中，你可以创建一个函数来计算一系列数字的和。下面是一个简单的例子，该函数接受一个数字列表作为输入，并返回它们的总和。\n\n```python\ndef sum_of_numbers(numbers):\n \"\"\"\n 计算列表中所有数字的和。\n \n 参数:\n numbers (list of int or float): 一个包含数字的列表。\n \n 返回:\n int or float: 列表中所有数字的总和。\n \"\"\"\n total_sum = sum(numbers) # 使用Python内置的sum函数计算总和\n return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers(numbers_list)) # 输出：The sum is: 15\n```\n\n在这段代码中，我们定义了一个名为 `sum_of_numbers` 的函数，它接收一个参数 `numbers`，这是一个包含整数或浮点数的列表。函数内部使用了Python的内置函数 `sum()` 来计算这些数字的总和，并将结果返回。\n\n你也可以手动实现求和逻辑，而不是使用内置的 `sum()` 函数，如下所示：\n\n```python\ndef sum_of_numbers_manual(numbers):\n \"\"\"\n 手动计算列表中所有数字的和。\n \n 参数:\n numbers (list of int or float): 一个包含数字的列表。\n \n 返回:\n int or float: 列表中所有数字的总和。\n \"\"\"\n total_sum = 0\n for number in numbers:\n total_sum += number\n return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers_manual(numbers_list)) # 输出：The sum is: 15\n```\n\n在这个版本中，我们初始化 `total_sum` 为0，然后遍历列表中的每个元素，并将其加到 `total_sum` 上。最后返回这个累加的结果。这两种方法都可以达到相同的目的，但是使用内置函数通常更简洁且效率更高。") + ctx.SetUserAttribute("k2", 2213.22) + ctx.WriteUserAttributeToTrace() +} + +func onHttpRequestHeaders(ctx wrapper.HttpContext, config CustomLogConfig, log wrapper.Log) types.Action { + if rand.Intn(10)%3 == 1 { + writeLog(ctx) + } else if rand.Intn(10)%3 == 2 { + writeLogWithKey(ctx, "ai_log") + } else { + writeTraceAttribute(ctx) + } + return types.ActionContinue +} diff --git a/plugins/wasm-go/pkg/wrapper/plugin_wrapper.go b/plugins/wasm-go/pkg/wrapper/plugin_wrapper.go index 3690f4fa44..be9144adfc 100644 --- a/plugins/wasm-go/pkg/wrapper/plugin_wrapper.go +++ b/plugins/wasm-go/pkg/wrapper/plugin_wrapper.go @@ -15,6 +15,8 @@ package wrapper import ( + "encoding/json" + "fmt" "strconv" "time" "unsafe" @@ -26,6 +28,12 @@ import ( "github.com/alibaba/higress/plugins/wasm-go/pkg/matcher" ) +const ( + CustomLogKey = "custom_log" + AILogKey = "ai_log" + TraceSpanTagPrefix = "trace_span_tag." +) + type HttpContext interface { Scheme() string Host() string @@ -35,6 +43,14 @@ type HttpContext interface { GetContext(key string) interface{} GetBoolContext(key string, defaultValue bool) bool GetStringContext(key, defaultValue string) string + GetUserAttribute(key string) interface{} + SetUserAttribute(key string, value interface{}) + // You can call this function to set custom log + WriteUserAttributeToLog() error + // You can call this function to set custom log with your specific key + WriteUserAttributeToLogWithKey(key string) error + // You can call this function to set custom trace span attribute + WriteUserAttributeToTrace() error // If the onHttpRequestBody handle is not set, the request body will not be read by default DontReadRequestBody() // If the onHttpResponseBody handle is not set, the request body will not be read by default @@ -335,9 +351,10 @@ func (ctx *CommonPluginCtx[PluginConfig]) OnTick() { func (ctx *CommonPluginCtx[PluginConfig]) NewHttpContext(contextID uint32) types.HttpContext { httpCtx := &CommonHttpCtx[PluginConfig]{ - plugin: ctx, - contextID: contextID, - userContext: map[string]interface{}{}, + plugin: ctx, + contextID: contextID, + userContext: map[string]interface{}{}, + userAttribute: map[string]interface{}{}, } if ctx.vm.onHttpRequestBody != nil || ctx.vm.onHttpStreamingRequestBody != nil { httpCtx.needRequestBody = true @@ -367,6 +384,7 @@ type CommonHttpCtx[PluginConfig any] struct { responseBodySize int contextID uint32 userContext map[string]interface{} + userAttribute map[string]interface{} } func (ctx *CommonHttpCtx[PluginConfig]) SetContext(key string, value interface{}) { @@ -377,6 +395,63 @@ func (ctx *CommonHttpCtx[PluginConfig]) GetContext(key string) interface{} { return ctx.userContext[key] } +func (ctx *CommonHttpCtx[PluginConfig]) SetUserAttribute(key string, value interface{}) { + ctx.userAttribute[key] = value +} + +func (ctx *CommonHttpCtx[PluginConfig]) GetUserAttribute(key string) interface{} { + return ctx.userAttribute[key] +} + +func (ctx *CommonHttpCtx[PluginConfig]) WriteUserAttributeToLog() error { + return ctx.WriteUserAttributeToLogWithKey(CustomLogKey) +} + +func (ctx *CommonHttpCtx[PluginConfig]) WriteUserAttributeToLogWithKey(key string) error { + // e.g. {\"field1\":\"value1\",\"field2\":\"value2\"} + preMarshalledJsonLogStr, _ := proxywasm.GetProperty([]string{key}) + newAttributeMap := map[string]interface{}{} + if string(preMarshalledJsonLogStr) != "" { + // e.g. {"field1":"value1","field2":"value2"} + preJsonLogStr := unmarshalStr(fmt.Sprintf(`"%s"`, string(preMarshalledJsonLogStr))) + err := json.Unmarshal([]byte(preJsonLogStr), &newAttributeMap) + if err != nil { + ctx.plugin.vm.log.Warnf("Unmarshal failed, will overwrite %s, pre value is: %s", key, string(preMarshalledJsonLogStr)) + return err + } + } + // update customLog + for k, v := range ctx.userAttribute { + newAttributeMap[k] = v + } + // e.g. {"field1":"value1","field2":2,"field3":"value3"} + jsonStr, _ := json.Marshal(newAttributeMap) + // e.g. {\"field1\":\"value1\",\"field2\":2,\"field3\":\"value3\"} + marshalledJsonStr := marshalStr(string(jsonStr)) + if err := proxywasm.SetProperty([]string{key}, []byte(marshalledJsonStr)); err != nil { + ctx.plugin.vm.log.Warnf("failed to set %s in filter state, raw is %s, err is %v", key, marshalledJsonStr, err) + return err + } + return nil +} + +func (ctx *CommonHttpCtx[PluginConfig]) WriteUserAttributeToTrace() error { + for k, v := range ctx.userAttribute { + traceSpanTag := TraceSpanTagPrefix + k + traceSpanValue := fmt.Sprint(v) + var err error + if traceSpanValue != "" { + err = proxywasm.SetProperty([]string{traceSpanTag}, []byte(traceSpanValue)) + } else { + err = fmt.Errorf("value of %s is empty", traceSpanTag) + } + if err != nil { + ctx.plugin.vm.log.Warnf("Failed to set trace attribute - %s: %s, error message: %v", traceSpanTag, traceSpanValue, err) + } + } + return nil +} + func (ctx *CommonHttpCtx[PluginConfig]) GetBoolContext(key string, defaultValue bool) bool { if b, ok := ctx.userContext[key].(bool); ok { return b diff --git a/plugins/wasm-go/pkg/wrapper/utils.go b/plugins/wasm-go/pkg/wrapper/utils.go new file mode 100644 index 0000000000..b541578588 --- /dev/null +++ b/plugins/wasm-go/pkg/wrapper/utils.go @@ -0,0 +1,36 @@ +package wrapper + +import ( + "encoding/json" + + "github.com/higress-group/proxy-wasm-go-sdk/proxywasm" + "github.com/tidwall/gjson" +) + +func unmarshalStr(marshalledJsonStr string) string { + // e.g. "{\"field1\":\"value1\",\"field2\":\"value2\"}" + var jsonStr string + err := json.Unmarshal([]byte(marshalledJsonStr), &jsonStr) + if err != nil { + proxywasm.LogErrorf("failed to unmarshal json string, raw string is: %s, err is: %v", marshalledJsonStr, err) + return "" + } + // e.g. {"field1":"value1","field2":"value2"} + return jsonStr +} + +func marshalStr(raw string) string { + // e.g. {"field1":"value1","field2":"value2"} + helper := map[string]string{ + "placeholder": raw, + } + marshalledHelper, _ := json.Marshal(helper) + marshalledRaw := gjson.GetBytes(marshalledHelper, "placeholder").Raw + if len(marshalledRaw) >= 2 { + // e.g. {\"field1\":\"value1\",\"field2\":\"value2\"} + return marshalledRaw[1 : len(marshalledRaw)-1] + } else { + proxywasm.LogErrorf("failed to marshal json string, raw string is: %s", raw) + return "" + } +}