diff --git a/pkg/ingress/kube/annotations/annotations.go b/pkg/ingress/kube/annotations/annotations.go index be9afe2675..36e4a6dea0 100644 --- a/pkg/ingress/kube/annotations/annotations.go +++ b/pkg/ingress/kube/annotations/annotations.go @@ -69,6 +69,8 @@ type Ingress struct { Auth *AuthConfig + Mirror *MirrorConfig + Destination *DestinationConfig IgnoreCase *IgnoreCaseConfig @@ -161,6 +163,7 @@ func NewAnnotationHandlerManager() AnnotationHandler { localRateLimit{}, fallback{}, auth{}, + mirror{}, destination{}, ignoreCaseMatching{}, match{}, @@ -182,6 +185,7 @@ func NewAnnotationHandlerManager() AnnotationHandler { retry{}, localRateLimit{}, fallback{}, + mirror{}, ignoreCaseMatching{}, match{}, headerControl{}, diff --git a/pkg/ingress/kube/annotations/mirror.go b/pkg/ingress/kube/annotations/mirror.go new file mode 100644 index 0000000000..fa1098c58e --- /dev/null +++ b/pkg/ingress/kube/annotations/mirror.go @@ -0,0 +1,118 @@ +// Copyright (c) 2023 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "github.com/alibaba/higress/pkg/ingress/kube/util" + . "github.com/alibaba/higress/pkg/ingress/log" + wrappers "google.golang.org/protobuf/types/known/wrapperspb" + networking "istio.io/api/networking/v1alpha3" +) + +const ( + mirrorTargetService = "mirror-target-service" + mirrorPercentage = "mirror-percentage" +) + +var ( + _ Parser = &mirror{} + _ RouteHandler = &mirror{} +) + +type MirrorConfig struct { + util.ServiceInfo + Percentage *wrappers.DoubleValue +} + +type mirror struct{} + +func (m mirror) Parse(annotations Annotations, config *Ingress, globalContext *GlobalContext) error { + if !needMirror(annotations) { + return nil + } + + target, err := annotations.ParseStringASAP(mirrorTargetService) + if err != nil { + IngressLog.Errorf("Get mirror target service fail, err: %v", err) + return nil + } + + serviceInfo, err := util.ParseServiceInfo(target, config.Namespace) + if err != nil { + IngressLog.Errorf("Get mirror target service fail, err: %v", err) + return nil + } + + serviceLister, exist := globalContext.ClusterServiceList[config.ClusterId] + if !exist { + IngressLog.Errorf("service lister of cluster %s doesn't exist", config.ClusterId) + return nil + } + + service, err := serviceLister.Services(serviceInfo.Namespace).Get(serviceInfo.Name) + if err != nil { + IngressLog.Errorf("Mirror service %s/%s within ingress %s/%s is not found, with err: %v", + serviceInfo.Namespace, serviceInfo.Name, config.Namespace, config.Name, err) + return nil + } + if service == nil { + IngressLog.Errorf("service %s/%s within ingress %s/%s is empty value", + serviceInfo.Namespace, serviceInfo.Name, config.Namespace, config.Name) + return nil + } + + if serviceInfo.Port == 0 { + // Use the first port + serviceInfo.Port = uint32(service.Spec.Ports[0].Port) + } + + var percentage *wrappers.DoubleValue + if value, err := annotations.ParseIntASAP(mirrorPercentage); err == nil { + if value < 100 { + percentage = &wrappers.DoubleValue{ + Value: float64(value), + } + } + } + + config.Mirror = &MirrorConfig{ + ServiceInfo: serviceInfo, + Percentage: percentage, + } + return nil +} + +func (m mirror) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + if config.Mirror == nil { + return + } + + route.Mirror = &networking.Destination{ + Host: util.CreateServiceFQDN(config.Mirror.Namespace, config.Mirror.Name), + Port: &networking.PortSelector{ + Number: config.Mirror.Port, + }, + } + + if config.Mirror.Percentage != nil { + route.MirrorPercentage = &networking.Percent{ + Value: config.Mirror.Percentage.GetValue(), + } + } +} + +func needMirror(annotations Annotations) bool { + return annotations.HasASAP(mirrorTargetService) +} diff --git a/pkg/ingress/kube/annotations/mirror_test.go b/pkg/ingress/kube/annotations/mirror_test.go new file mode 100644 index 0000000000..1a35d9f700 --- /dev/null +++ b/pkg/ingress/kube/annotations/mirror_test.go @@ -0,0 +1,163 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "github.com/alibaba/higress/pkg/ingress/kube/util" + "github.com/golang/protobuf/proto" + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + "reflect" + "testing" +) + +func TestParseMirror(t *testing.T) { + testCases := []struct { + input []map[string]string + expect *MirrorConfig + }{ + {}, + { + input: []map[string]string{ + {buildHigressAnnotationKey(mirrorTargetService): "test/app"}, + {buildNginxAnnotationKey(mirrorTargetService): "test/app"}, + }, + expect: &MirrorConfig{ + ServiceInfo: util.ServiceInfo{ + NamespacedName: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + }, + }, + }, + { + input: []map[string]string{ + {buildHigressAnnotationKey(mirrorTargetService): "test/app:8080"}, + {buildNginxAnnotationKey(mirrorTargetService): "test/app:8080"}, + }, + expect: &MirrorConfig{ + ServiceInfo: util.ServiceInfo{ + NamespacedName: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 8080, + }, + }, + }, + { + input: []map[string]string{ + {buildHigressAnnotationKey(mirrorTargetService): "test/app:hi"}, + {buildNginxAnnotationKey(mirrorTargetService): "test/app:hi"}, + }, + expect: &MirrorConfig{ + ServiceInfo: util.ServiceInfo{ + NamespacedName: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + }, + }, + }, + { + input: []map[string]string{ + {buildHigressAnnotationKey(mirrorTargetService): "test/app"}, + {buildNginxAnnotationKey(mirrorTargetService): "test/app"}, + }, + expect: &MirrorConfig{ + ServiceInfo: util.ServiceInfo{ + NamespacedName: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + }, + }, + }, + } + + mirror := mirror{} + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + config := &Ingress{ + Meta: Meta{ + Namespace: "test", + ClusterId: "cluster", + }, + } + globalContext, cancel := initGlobalContextForService() + defer cancel() + + for _, in := range testCase.input { + _ = mirror.Parse(in, config, globalContext) + if !reflect.DeepEqual(testCase.expect, config.Mirror) { + t.Log("expect:", *testCase.expect) + t.Log("actual:", *config.Mirror) + t.Fatal("Should be equal") + } + } + }) + } +} + +func TestMirror_ApplyRoute(t *testing.T) { + testCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPRoute + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Mirror: &MirrorConfig{ + ServiceInfo: util.ServiceInfo{ + NamespacedName: model.NamespacedName{ + Namespace: "default", + Name: "test", + }, + Port: 8080, + }, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Mirror: &networking.Destination{ + Host: "test.default.svc.cluster.local", + Port: &networking.PortSelector{ + Number: 8080, + }, + }, + }, + }, + } + + mirror := mirror{} + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + mirror.ApplyRoute(testCase.input, testCase.config) + if !proto.Equal(testCase.input, testCase.expect) { + t.Fatal("Must be equal.") + } + }) + } +} diff --git a/pkg/ingress/kube/util/util.go b/pkg/ingress/kube/util/util.go index ec9c68870d..597f56f2c0 100644 --- a/pkg/ingress/kube/util/util.go +++ b/pkg/ingress/kube/util/util.go @@ -20,8 +20,10 @@ import ( "encoding/hex" "errors" "fmt" + "istio.io/istio/pilot/pkg/model" "os" "path" + "strconv" "strings" "github.com/golang/protobuf/jsonpb" @@ -113,3 +115,44 @@ func BuildPatchStruct(config string) *_struct.Struct { } return val } + +type ServiceInfo struct { + model.NamespacedName + Port uint32 +} + +// convertToPort converts a port string to a uint32. +func convertToPort(v string) (uint32, error) { + p, err := strconv.ParseUint(v, 10, 32) + if err != nil || p > 65535 { + return 0, fmt.Errorf("invalid port %s: %v", v, err) + } + return uint32(p), nil +} + +func ParseServiceInfo(service string, ingressNamespace string) (ServiceInfo, error) { + parts := strings.Split(service, ":") + namespacedName := SplitNamespacedName(parts[0]) + + if namespacedName.Name == "" { + return ServiceInfo{}, errors.New("service name can not be empty") + } + + if namespacedName.Namespace == "" { + namespacedName.Namespace = ingressNamespace + } + + var port uint32 + if len(parts) == 2 { + // If port parse fail, we ignore port and pick the first one. + port, _ = convertToPort(parts[1]) + } + + return ServiceInfo{ + NamespacedName: model.NamespacedName{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + }, + Port: port, + }, nil +} diff --git a/plugins/wasm-go/extensions/oidc/go.mod b/plugins/wasm-go/extensions/oidc/go.mod index 753be9a2db..89ef5f4c7b 100644 --- a/plugins/wasm-go/extensions/oidc/go.mod +++ b/plugins/wasm-go/extensions/oidc/go.mod @@ -1,6 +1,8 @@ module github.com/alibaba/higress/plugins/wasm-go/extensions/oidc -go 1.19 +go 1.21 + +toolchain go1.22.5 replace github.com/alibaba/higress/plugins/wasm-go => ../.. diff --git a/test/e2e/conformance/base/manifests.yaml b/test/e2e/conformance/base/manifests.yaml index eec4b0c316..235dce14dc 100644 --- a/test/e2e/conformance/base/manifests.yaml +++ b/test/e2e/conformance/base/manifests.yaml @@ -178,6 +178,55 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: infra-backend-mirror + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-mirror + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: infra-backend-mirror + namespace: higress-conformance-infra + labels: + app: infra-backend-mirror +spec: + replicas: 1 + selector: + matchLabels: + app: infra-backend-mirror + template: + metadata: + labels: + app: infra-backend-mirror + spec: + containers: + - name: infra-backend-mirror + # image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e + + # From https://github.com/Uncle-Justice/echo-server + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echo-server:1.3.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Service metadata: name: infra-backend-echo-body-v1 namespace: higress-conformance-infra diff --git a/test/e2e/conformance/tests/httproute-mirror-target-service.go b/test/e2e/conformance/tests/httproute-mirror-target-service.go new file mode 100644 index 0000000000..1e138ac188 --- /dev/null +++ b/test/e2e/conformance/tests/httproute-mirror-target-service.go @@ -0,0 +1,107 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/alibaba/higress/test/e2e/conformance/utils/http" + "github.com/alibaba/higress/test/e2e/conformance/utils/suite" + v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +func init() { + Register(HTTPRouteMirrorTargetService) +} + +var HTTPRouteMirrorTargetService = suite.ConformanceTest{ + ShortName: "HTTPRouteMirrorTargetService", + Description: "The Ingress in the higress-conformance-infra namespace mirror request to target service", + Features: []suite.SupportedFeature{suite.HTTPConformanceFeature}, + Manifests: []string{"tests/httproute-mirror-target-service.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + testcases := []http.Assertion{ + { + Meta: http.AssertionMeta{ + TargetBackend: "infra-backend-v1", + TargetNamespace: "higress-conformance-infra", + }, + Request: http.AssertionRequest{ + ActualRequest: http.Request{ + Path: "/mirror", + }, + }, + Response: http.AssertionResponse{ + ExpectedResponse: http.Response{ + StatusCode: 200, + }, + }, + }, + } + + t.Run("HTTPRoute mirror request to target service", func(t *testing.T) { + for _, testcase := range testcases { + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase) + //check mirror's logs for request + cfg, err := config.GetConfig() + if err != nil { + t.Fatalf("[httproute-mirror] get config failed.") + return + } + clientSet, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatalf("[httproute-mirror] init clientset failed.") + return + } + pods, err := clientSet.CoreV1().Pods("higress-conformance-infra").List(context.Background(), meta_v1.ListOptions{ + LabelSelector: meta_v1.FormatLabelSelector(&meta_v1.LabelSelector{MatchLabels: map[string]string{"app": "infra-backend-mirror"}}), + }) + if err != nil || len(pods.Items) == 0 { + t.Fatalf("[httproute-mirror] get pods by label of [\"app\": \"infra-backend-mirror\"] failed.") + return + } + req := clientSet.CoreV1().Pods("higress-conformance-infra").GetLogs(pods.Items[0].Name, &v1.PodLogOptions{ + Container: "infra-backend-mirror", + SinceTime: &meta_v1.Time{Time: time.Now().Add(-time.Second * 10)}, + }) + podLogs, err := req.Stream(context.Background()) + defer podLogs.Close() + if err != nil { + t.Fatalf("[httproute-mirror] init pod logs stream failed.") + return + } + + podBuf := new(bytes.Buffer) + _, err = io.Copy(podBuf, podLogs) + if err != nil { + t.Fatalf("[httproute-mirror] read pod logs stream failed.") + return + } + if !strings.Contains(podBuf.String(), "Echoing back request made to /mirror") { + t.Fatalf("[httproute-mirror] mirror pod hasn't received any mirror requests in logs.") + return + } + } + }) + }, +} diff --git a/test/e2e/conformance/tests/httproute-mirror-target-service.yaml b/test/e2e/conformance/tests/httproute-mirror-target-service.yaml new file mode 100644 index 0000000000..0bcc20fa70 --- /dev/null +++ b/test/e2e/conformance/tests/httproute-mirror-target-service.yaml @@ -0,0 +1,32 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: higress-conformance-infra-mirror-target-service + namespace: higress-conformance-infra + annotations: + nginx.ingress.kubernetes.io/mirror-target-service: "infra-backend-mirror" +spec: + ingressClassName: higress + rules: + - http: + paths: + - pathType: Prefix + path: "/mirror" + backend: + service: + name: infra-backend-v1 + port: + number: 8080