diff --git a/go.mod b/go.mod index 91008eeb7a42..da3125422c93 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( go.opentelemetry.io/otel/metric v1.31.0 go.opentelemetry.io/otel/sdk v1.31.0 go.opentelemetry.io/otel/sdk/metric v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 golang.org/x/net v0.30.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 @@ -32,7 +33,6 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect golang.org/x/text v0.19.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect ) diff --git a/stats/opentelemetry/grpc_trace_bin_propagator.go b/stats/opentelemetry/grpc_trace_bin_propagator.go new file mode 100644 index 000000000000..e8a3986d4f4a --- /dev/null +++ b/stats/opentelemetry/grpc_trace_bin_propagator.go @@ -0,0 +1,119 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package opentelemetry + +import ( + "context" + + otelpropagation "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" +) + +// gRPCTraceBinHeaderKey is the gRPC metadata header key `grpc-trace-bin` used +// to propagate trace context in binary format. +const grpcTraceBinHeaderKey = "grpc-trace-bin" + +// GRPCTraceBinPropagator is an OpenTelemetry TextMapPropagator which is used +// to extract and inject trace context data from and into headers exchanged by +// gRPC applications. It propagates trace data in binary format using the +// `grpc-trace-bin` header. +type GRPCTraceBinPropagator struct{} + +// Inject sets OpenTelemetry span context from the Context into the carrier as +// a `grpc-trace-bin` header if span context is valid. +// +// If span context is not valid, it returns without setting `grpc-trace-bin` +// header. +func (GRPCTraceBinPropagator) Inject(ctx context.Context, carrier otelpropagation.TextMapCarrier) { + sc := oteltrace.SpanFromContext(ctx) + if !sc.SpanContext().IsValid() { + return + } + + bd := toBinary(sc.SpanContext()) + carrier.Set(grpcTraceBinHeaderKey, string(bd)) +} + +// Extract reads OpenTelemetry span context from the `grpc-trace-bin` header of +// carrier into the provided context, if present. +// +// If a valid span context is retrieved from `grpc-trace-bin`, it returns a new +// context containing the extracted OpenTelemetry span context marked as +// remote. +// +// If `grpc-trace-bin` header is not present, it returns the context as is. +func (GRPCTraceBinPropagator) Extract(ctx context.Context, carrier otelpropagation.TextMapCarrier) context.Context { + h := carrier.Get(grpcTraceBinHeaderKey) + if h == "" { + return ctx + } + + sc, ok := fromBinary([]byte(h)) + if !ok { + return ctx + } + return oteltrace.ContextWithRemoteSpanContext(ctx, sc) +} + +// Fields returns the keys whose values are set with Inject. +// +// GRPCTraceBinPropagator always returns a slice containing only +// `grpc-trace-bin` key because it only sets the `grpc-trace-bin` header for +// propagating trace context. +func (GRPCTraceBinPropagator) Fields() []string { + return []string{grpcTraceBinHeaderKey} +} + +// toBinary returns the binary format representation of a SpanContext. +// +// If sc is the zero value, returns nil. +func toBinary(sc oteltrace.SpanContext) []byte { + if sc.Equal(oteltrace.SpanContext{}) { + return nil + } + var b [29]byte + traceID := oteltrace.TraceID(sc.TraceID()) + copy(b[2:18], traceID[:]) + b[18] = 1 + spanID := oteltrace.SpanID(sc.SpanID()) + copy(b[19:27], spanID[:]) + b[27] = 2 + b[28] = byte(oteltrace.TraceFlags(sc.TraceFlags())) + return b[:] +} + +// fromBinary returns the SpanContext represented by b with Remote set to true. +// +// It returns with zero value SpanContext and false, if any of the +// below condition is not satisfied: +// - Valid header: len(b) = 29 +// - Valid version: b[0] = 0 +// - Valid traceID prefixed with 0: b[1] = 0 +// - Valid spanID prefixed with 1: b[18] = 1 +// - Valid traceFlags prefixed with 2: b[27] = 2 +func fromBinary(b []byte) (oteltrace.SpanContext, bool) { + if len(b) != 29 || b[0] != 0 || b[1] != 0 || b[18] != 1 || b[27] != 2 { + return oteltrace.SpanContext{}, false + } + + return oteltrace.SpanContext{}.WithTraceID( + oteltrace.TraceID(b[2:18])).WithSpanID( + oteltrace.SpanID(b[19:27])).WithTraceFlags( + oteltrace.TraceFlags(b[28])).WithRemote(true), true +} diff --git a/stats/opentelemetry/grpc_trace_bin_propagator_test.go b/stats/opentelemetry/grpc_trace_bin_propagator_test.go new file mode 100644 index 000000000000..2d575af4a581 --- /dev/null +++ b/stats/opentelemetry/grpc_trace_bin_propagator_test.go @@ -0,0 +1,219 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package opentelemetry + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + oteltrace "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc/metadata" + itracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing" +) + +var validSpanContext = oteltrace.SpanContext{}.WithTraceID( + oteltrace.TraceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}).WithSpanID( + oteltrace.SpanID{17, 18, 19, 20, 21, 22, 23, 24}).WithTraceFlags( + oteltrace.TraceFlags(1)) + +// TestInject_ValidSpanContext verifies that the GRPCTraceBinPropagator +// correctly injects a valid OpenTelemetry span context as `grpc-trace-bin` +// header in the provided carrier's context metadata. +// +// It verifies that if a valid span context is injected, same span context can +// can be retreived from the carrier's context metadata. +func (s) TestInject_ValidSpanContext(t *testing.T) { + p := GRPCTraceBinPropagator{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := itracing.NewOutgoingCarrier(ctx) + ctx = oteltrace.ContextWithSpanContext(ctx, validSpanContext) + + p.Inject(ctx, c) + + md, _ := metadata.FromOutgoingContext(c.Context()) + gotH := md.Get(grpcTraceBinHeaderKey) + if gotH[len(gotH)-1] == "" { + t.Fatalf("got empty value from Carrier's context metadata grpc-trace-bin header, want valid span context: %v", validSpanContext) + } + gotSC, ok := fromBinary([]byte(gotH[len(gotH)-1])) + if !ok { + t.Fatalf("got invalid span context %v from Carrier's context metadata grpc-trace-bin header, want valid span context: %v", gotSC, validSpanContext) + } + if cmp.Equal(validSpanContext, gotSC) { + t.Fatalf("got span context = %v, want span contexts %v", gotSC, validSpanContext) + } +} + +// TestInject_InvalidSpanContext verifies that the GRPCTraceBinPropagator does +// not inject an invalid OpenTelemetry span context as `grpc-trace-bin` header +// in the provided carrier's context metadata. +// +// If an invalid span context is injected, it verifies that `grpc-trace-bin` +// header is not set in the carrier's context metadata. +func (s) TestInject_InvalidSpanContext(t *testing.T) { + p := GRPCTraceBinPropagator{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := itracing.NewOutgoingCarrier(ctx) + ctx = oteltrace.ContextWithSpanContext(ctx, oteltrace.SpanContext{}) + + p.Inject(ctx, c) + + md, _ := metadata.FromOutgoingContext(c.Context()) + if gotH := md.Get(grpcTraceBinHeaderKey); len(gotH) > 0 { + t.Fatalf("got %v value from Carrier's context metadata grpc-trace-bin header, want empty", gotH) + } +} + +// TestExtract verifies that the GRPCTraceBinPropagator correctly extracts +// OpenTelemetry span context data from the provided context using carrier. +// +// If a valid span context was injected, it verifies same trace span context +// is extracted from carrier's metadata for `grpc-trace-bin` header key. +// +// If invalid span context was injected, it verifies that valid trace span +// context is not extracted. +func (s) TestExtract(t *testing.T) { + tests := []struct { + name string + wantSC oteltrace.SpanContext // expected span context from carrier + }{ + { + name: "valid OpenTelemetry span context", + wantSC: validSpanContext.WithRemote(true), + }, + { + name: "invalid OpenTelemetry span context", + wantSC: oteltrace.SpanContext{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + p := GRPCTraceBinPropagator{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ctx = metadata.NewIncomingContext(ctx, metadata.MD{grpcTraceBinHeaderKey: []string{string(toBinary(test.wantSC))}}) + + c := itracing.NewIncomingCarrier(ctx) + + tCtx := p.Extract(ctx, c) + got := oteltrace.SpanContextFromContext(tCtx) + if !got.Equal(test.wantSC) { + t.Fatalf("got span context: %v, want span context: %v", got, test.wantSC) + } + }) + } +} + +// TestBinary verifies that the toBinary() function correctly serializes a valid +// OpenTelemetry span context into its binary format representation. If span +// context is invalid, it verifies that serialization is nil. +func (s) TestToBinary(t *testing.T) { + tests := []struct { + name string + sc oteltrace.SpanContext + want []byte + }{ + { + name: "valid context", + sc: validSpanContext, + want: toBinary(validSpanContext), + }, + { + name: "zero value context", + sc: oteltrace.SpanContext{}, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := toBinary(test.sc); !cmp.Equal(got, test.want) { + t.Fatalf("binary() = %v, want %v", got, test.want) + } + }) + } +} + +// TestFromBinary verifies that the fromBinary() function correctly +// deserializes a binary format representation of a valid OpenTelemetry span +// context into its corresponding span context format. If span context's binary +// representation is invalid, it verifies that deserialization is zero value +// span context. +func (s) TestFromBinary(t *testing.T) { + tests := []struct { + name string + b []byte + want oteltrace.SpanContext + ok bool + }{ + { + name: "valid", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: validSpanContext.WithRemote(true), + ok: true, + }, + { + name: "invalid length", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid version", + b: []byte{1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid traceID field ID", + b: []byte{0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid spanID field ID", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + { + name: "invalid traceFlags field ID", + b: []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 1, 1}, + want: oteltrace.SpanContext{}, + ok: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, ok := fromBinary(test.b) + if ok != test.ok { + t.Fatalf("fromBinary() ok = %v, want %v", ok, test.ok) + return + } + if !got.Equal(test.want) { + t.Fatalf("fromBinary() got = %v, want %v", got, test.want) + } + }) + } +} diff --git a/stats/opentelemetry/internal/tracing/carrier.go b/stats/opentelemetry/internal/tracing/carrier.go new file mode 100644 index 000000000000..214102aaf97a --- /dev/null +++ b/stats/opentelemetry/internal/tracing/carrier.go @@ -0,0 +1,131 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tracing implements the OpenTelemetry carrier for context propagation +// in gRPC tracing. +package tracing + +import ( + "context" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" +) + +var logger = grpclog.Component("otel-plugin") + +// IncomingCarrier is a TextMapCarrier that uses incoming `context.Context` to +// retrieve any propagated key-value pairs in text format. +type IncomingCarrier struct { + ctx context.Context +} + +// NewIncomingCarrier creates a new `IncomingCarrier` with the given context. +// The incoming carrier should be used with propagator's `Extract()` method in +// the incoming rpc path. +func NewIncomingCarrier(ctx context.Context) *IncomingCarrier { + return &IncomingCarrier{ctx: ctx} +} + +// Get returns the string value associated with the passed key from the +// carrier's incoming context metadata. +// +// It returns an empty string if the key is not present in the carrier's +// context or if the value associated with the key is empty. +// +// If multiple values are present for a key, it returns the last one. +func (c *IncomingCarrier) Get(key string) string { + values := metadata.ValueFromIncomingContext(c.ctx, key) + if len(values) == 0 { + return "" + } + return values[len(values)-1] +} + +// Set just logs an error. It implements the `TextMapCarrier` interface but +// should not be used with `IncomingCarrier`. +func (c *IncomingCarrier) Set(string, string) { + logger.Error("Set() should not be used with IncomingCarrier.") +} + +// Keys returns the keys stored in the carrier's context metadata. It returns +// keys from incoming context metadata. +func (c *IncomingCarrier) Keys() []string { + md, ok := metadata.FromIncomingContext(c.ctx) + if !ok { + return nil + } + keys := make([]string, 0, len(md)) + for key := range md { + keys = append(keys, key) + } + return keys +} + +// Context returns the underlying context associated with the +// `IncomingCarrier“. +func (c *IncomingCarrier) Context() context.Context { + return c.ctx +} + +// OutgoingCarrier is a TextMapCarrier that uses outgoing `context.Context` to +// store any propagated key-value pairs in text format. +type OutgoingCarrier struct { + ctx context.Context +} + +// NewOutgoingCarrier creates a new Carrier with the given context. The +// outgoing carrier should be used with propagator's `Inject()` method in the +// outgoing rpc path. +func NewOutgoingCarrier(ctx context.Context) *OutgoingCarrier { + return &OutgoingCarrier{ctx: ctx} +} + +// Get just logs an error and returns an empty string. It implements the +// `TextMapCarrier` interface but should not be used with `OutgoingCarrier`. +func (c *OutgoingCarrier) Get(string) string { + logger.Error("Get() should not be used with `OutgoingCarrier`") + return "" +} + +// Set stores the key-value pair in the carrier's outgoing context metadata. +// +// If the key already exists, given value is appended to the last. +func (c *OutgoingCarrier) Set(key, value string) { + c.ctx = metadata.AppendToOutgoingContext(c.ctx, key, value) +} + +// Keys returns the keys stored in the carrier's context metadata. It returns +// keys from outgoing context metadata. +func (c *OutgoingCarrier) Keys() []string { + md, ok := metadata.FromOutgoingContext(c.ctx) + if !ok { + return nil + } + keys := make([]string, 0, len(md)) + for key := range md { + keys = append(keys, key) + } + return keys +} + +// Context returns the underlying context associated with the +// `OutgoingCarrier“. +func (c *OutgoingCarrier) Context() context.Context { + return c.ctx +} diff --git a/stats/opentelemetry/internal/tracing/carrier_test.go b/stats/opentelemetry/internal/tracing/carrier_test.go new file mode 100644 index 000000000000..a2e22beb08ac --- /dev/null +++ b/stats/opentelemetry/internal/tracing/carrier_test.go @@ -0,0 +1,190 @@ +/* + * + * Copyright 2024 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * htestp://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package tracing + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/metadata" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestIncomingCarrier verifies that `IncomingCarrier.Get()` returns correct +// value for the corresponding key in the carrier's context metadata, if key is +// present. If key is not present, it verifies that empty string is returned. +// +// If multiple values are present for a key, it verifies that last value is +// returned. +// +// If key ends with `-bin`, it verifies that a correct binary value is returned +// in the string format for the binary header. +func (s) TestIncomingCarrier(t *testing.T) { + tests := []struct { + name string + md metadata.MD + key string + want string + wantKeys []string + }{ + { + name: "existing key", + md: metadata.Pairs("key1", "value1"), + key: "key1", + want: "value1", + wantKeys: []string{"key1"}, + }, + { + name: "non-existing key", + md: metadata.Pairs("key1", "value1"), + key: "key2", + want: "", + wantKeys: []string{"key1"}, + }, + { + name: "empty key", + md: metadata.MD{}, + key: "key1", + want: "", + wantKeys: []string{}, + }, + { + name: "more than one key/value pair", + md: metadata.MD{"key1": []string{"value1"}, "key2": []string{"value2"}}, + key: "key2", + want: "value2", + wantKeys: []string{"key1", "key2"}, + }, + { + name: "more than one value for a key", + md: metadata.MD{"key1": []string{"value1", "value2"}}, + key: "key1", + want: "value2", + wantKeys: []string{"key1"}, + }, + { + name: "grpc-trace-bin key", + md: metadata.Pairs("grpc-trace-bin", string([]byte{0x01, 0x02, 0x03})), + key: "grpc-trace-bin", + want: string([]byte{0x01, 0x02, 0x03}), + wantKeys: []string{"grpc-trace-bin"}, + }, + { + name: "grpc-trace-bin key with another string key", + md: metadata.MD{"key1": []string{"value1"}, "grpc-trace-bin": []string{string([]byte{0x01, 0x02, 0x03})}}, + key: "grpc-trace-bin", + want: string([]byte{0x01, 0x02, 0x03}), + wantKeys: []string{"key1", "grpc-trace-bin"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := NewIncomingCarrier(metadata.NewIncomingContext(ctx, test.md)) + got := c.Get(test.key) + if got != test.want { + t.Fatalf("c.Get() = %s, want %s", got, test.want) + } + if gotKeys := c.Keys(); !cmp.Equal(test.wantKeys, gotKeys, cmpopts.SortSlices(func(a, b string) bool { return a < b })) { + t.Fatalf("c.Keys() = keys %v, want %v", gotKeys, test.wantKeys) + } + }) + } +} + +// TestOutgoingCarrier verifies that a key-value pair is set in carrier's +// context metadata using `OutgoingCarrier.Set()`. If key is not present, it +// verifies that key-value pair is insterted. If key is already present, it +// verifies that new value is appended at the end of list for the existing key. +// +// If key ends with `-bin`, it verifies that a binary value is set for +// `-bin` header in string format. +// +// It also verifies that both existing and newly inserted keys are present in +// the carrier's context using `Carrier.Keys()`. +func (s) TestOutgoingCarrier(t *testing.T) { + tests := []struct { + name string + initialMD metadata.MD + setKey string + setValue string + wantValue string // expected value of the set key + wantKeys []string + }{ + { + name: "new key", + initialMD: metadata.MD{}, + setKey: "key1", + setValue: "value1", + wantValue: "value1", + wantKeys: []string{"key1"}, + }, + { + name: "add to existing key", + initialMD: metadata.MD{"key1": []string{"oldvalue"}}, + setKey: "key1", + setValue: "newvalue", + wantValue: "newvalue", + wantKeys: []string{"key1"}, + }, + { + name: "new key with different existing key", + initialMD: metadata.MD{"key2": []string{"value2"}}, + setKey: "key1", + setValue: "value1", + wantValue: "value1", + wantKeys: []string{"key2", "key1"}, + }, + { + name: "grpc-trace-bin binary key", + initialMD: metadata.MD{"key1": []string{"value1"}}, + setKey: "grpc-trace-bin", + setValue: string([]byte{0x01, 0x02, 0x03}), + wantValue: string([]byte{0x01, 0x02, 0x03}), + wantKeys: []string{"key1", "grpc-trace-bin"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := NewOutgoingCarrier(metadata.NewOutgoingContext(ctx, test.initialMD)) + c.Set(test.setKey, test.setValue) + if gotKeys := c.Keys(); !cmp.Equal(test.wantKeys, gotKeys, cmpopts.SortSlices(func(a, b string) bool { return a < b })) { + t.Fatalf("c.Keys() = keys %v, want %v", gotKeys, test.wantKeys) + } + if md, ok := metadata.FromOutgoingContext(c.Context()); ok && md.Get(test.setKey)[len(md.Get(test.setKey))-1] != test.wantValue { + t.Fatalf("got value %s, want %s, for key %s", md.Get(test.setKey)[len(md.Get(test.setKey))-1], test.wantValue, test.setKey) + } + }) + } +}