diff --git a/cmd/main.go b/cmd/main.go index 5993d59..c308df3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -53,7 +53,9 @@ func main() { // Create a cache var l log.CLog - cache := cache.NewSnapshotCache(true, cache.IDHash{}, l) + // ADS flag forces a delay in responding to streaming requests until all resources are explicitly named in the request. + // Making it to false to make it convinient for one control plane to send partial responses to clients. + cache := cache.NewSnapshotCache(false, cache.IDHash{}, l) // Start the snapshot refresher.. sn := snapshot.NewSnapshot(cfg, cache) diff --git a/example/Readme.md b/example/Readme.md index ccf0675..526110b 100644 --- a/example/Readme.md +++ b/example/Readme.md @@ -25,20 +25,47 @@ To run example on a local cluster., docker build -t xds.example/xds-controller . docker build -t xds.example/client -f example/Dockerfile --build-arg TYPE=client example docker build -t xds.example/server -f example/Dockerfile --build-arg TYPE=server example +docker pull envoyproxy/envoy:v1.24.0 +``` +### Deployments +#### Pattern 1: CLient server communication using xds:look aside load balancing ``` -### Deploy +kubectl create namespace xds-test +kubectl apply -f example/k8s/xds-controller.deployment.yaml -n xds-test +kubectl apply -f example/k8s/server.deployment.yaml -n xds-test +kubectl apply -f example/k8s/client.deployment.yaml -n xds-test +kubectl apply -f example/k8s/envoy.deployment.yaml -n xds-test +``` +#### Pattern 2: Client with 2 servers communication using xds:look aside load balancing ``` kubectl create namespace xds-test kubectl apply -f example/k8s/xds-controller.deployment.yaml -n xds-test kubectl apply -f example/k8s/server.deployment.yaml -n xds-test +kubectl apply -f example/k8s/server.b.deployment.yaml -n xds-test kubectl apply -f example/k8s/client.deployment.yaml -n xds-test ``` + +#### Pattern 3: External Client proxied via Envoy L7 Load balancer (No xds) using static dns +``` +kubectl create namespace xds-test +kubectl apply -f example/k8s/server.deployment.yaml -n xds-test +kubectl apply -f example/k8s/envoy.deployment.yaml -n xds-test +``` +#### Pattern 4: External Client proxied via Envoy L7 Load balancer. Evoy using xDS stream to discover server +``` +kubectl create namespace xds-test +kubectl apply -f example/k8s/xds-controller.deployment.yaml -n xds-test +kubectl apply -f example/k8s/server.deployment.yaml -n xds-test +kubectl apply -f example/k8s/envoy.deployment.yaml -n xds-test +``` + + ### Metrics xds-controller exposes metrics via /metrics endpoint by default on 8082 port. Portforwarding can be used to hit localhost and see the metric details ``` -// get pods +// get pods kubectl get pods -n xds-test kubectl port-forward {pod-name} 8082:8082 -n xds-test curl localhost:8082/metrics diff --git a/example/k8s/envoy.deployment.yaml b/example/k8s/envoy.deployment.yaml new file mode 100644 index 0000000..a23f4be --- /dev/null +++ b/example/k8s/envoy.deployment.yaml @@ -0,0 +1,149 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-deployment + labels: + app: envoy +spec: + replicas: 1 + selector: + matchLabels: + app: envoy + template: + metadata: + labels: + app: envoy + spec: + containers: + - name: envoy + command: + - envoy + args: + - "-c /etc/envoy/envoy.yaml" + - "--log-level info" + - "--log-format %L. envoy]%v" + - "--log-format-escaped" + - "--enable-fine-grain-logging" + - "--component-log-level upstream:debug,connection:debug" + image: envoyproxy/envoy:v1.24.0 + imagePullPolicy: Never + # env: + # - name: "GRPC_GO_LOG_SEVERITY_LEVEL" + # value: info + ports: + - name: http + containerPort: 8082 + resources: + requests: + cpu: 1 + memory: 1G + limits: + cpu: 1 + memory: 1G + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + volumes: + - name: envoy-config + configMap: + name: envoy-config-map-cewwcs4rfgfy +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + note: envoy + name: envoy +spec: + ports: + - name: http + port: 8082 + protocol: TCP + targetPort: 8082 + - name: admin-http + protocol: TCP + port: 8081 + targetPort: 8090 + selector: + app: envoy + clusterIP: None +--- +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + note: envoy-config-map + name: envoy-config-map-cewwcs4rfgfy +data: + envoy.yaml: | + static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 8082 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + common_http_protocol_options: + idle_timeout: 900s + stat_prefix: ingress_http + http2_protocol_options: + max_concurrent_streams: 100 + access_log: + name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/stdout + typed_json_format: + duration: "%DURATION%" + request_duration: "%REQUEST_DURATION%" + response_duration: "%RESPONSE_DURATION%" + route_name: "%ROUTE_NAME%" + upstream_host: "%UPSTREAM_HOST%" + upstream_failure_reason: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + response_code: "%RESPONSE_CODE%" + response_flag: "%RESPONSE_FLAGS%" + grpc_status: "%GRPC_STATUS%" + response: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%" + envoy_path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + fwd: "%REQ(X-FORWARDED-FOR)%" + route_config: + name: local_route + virtual_hosts: + - name: default + domains: + - "*" + routes: + - match: + prefix: "/hello.Person/" + route: + cluster: person_server + max_grpc_timeout: 100s + clusters: + - name: person_server + type: STRICT_DNS + # Comment out the following line to test on v6 networks + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + close_connections_on_host_health_failure: true + http2_protocol_options: + max_concurrent_streams: 1000000000 + load_assignment: + cluster_name: person_server + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: server.xds-test.svc.cluster.local + port_value: 8080 + admin: + access_log_path: "/dev/stdout" + address: + socket_address: + address: 0.0.0.0 + port_value: 8090 diff --git a/example/k8s/envoy.xds.deployment.yaml b/example/k8s/envoy.xds.deployment.yaml new file mode 100644 index 0000000..06ecd34 --- /dev/null +++ b/example/k8s/envoy.xds.deployment.yaml @@ -0,0 +1,126 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-deployment + labels: + app: envoy +spec: + replicas: 1 + selector: + matchLabels: + app: envoy + template: + metadata: + labels: + app: envoy + spec: + containers: + - name: envoy + command: + - envoy + args: + - "-c /etc/envoy/envoy.yaml" + - "--log-level info" + - "--log-format %L. envoy]%v" + - "--log-format-escaped" + - "--enable-fine-grain-logging" + - "--component-log-level upstream:debug,connection:debug" + image: envoyproxy/envoy:v1.19.0 + imagePullPolicy: Never + # env: + # - name: "GRPC_GO_LOG_SEVERITY_LEVEL" + # value: info + ports: + - name: http + containerPort: 10000 + resources: + requests: + cpu: 1 + memory: 1G + limits: + cpu: 1 + memory: 1G + volumeMounts: + - name: envoy-config + mountPath: /etc/envoy + volumes: + - name: envoy-config + configMap: + name: envoy-config-map-cewwcs4rfgfy +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + note: envoy + name: envoy +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 10000 + - name: admin-http + protocol: TCP + port: 8081 + targetPort: 8090 + selector: + app: envoy + clusterIP: None +--- +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + note: envoy-config-map + name: envoy-config-map-cewwcs4rfgfy +data: + envoy.yaml: | + node: + id: THIS_IS_THE_SNAPSHOT_ID_THAT_MATCHES_TO_XDS_CONTROLLER + cluster: all_services + dynamic_resources: + lds_config: + resource_api_version: V3 + ads: {} + cds_config: + resource_api_version: V3 + ads: {} + ads_config: + # type of request + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + static_resources: + clusters: + - name: xds_cluster + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + # Configure an HTTP/2 keep-alive to detect connection issues and reconnect + # to the admin server if the connection is no longer responsive. + connection_keepalive: + interval: 30s + timeout: 5s + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: xds-controller.xds-test.svc.cluster.local + port_value: 8080 + admin: + access_log_path: "/dev/stdout" + address: + socket_address: + address: 0.0.0.0 + port_value: 8090 diff --git a/example/k8s/server.b.deployment.yaml b/example/k8s/server.b.deployment.yaml new file mode 100644 index 0000000..9d256eb --- /dev/null +++ b/example/k8s/server.b.deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: server-b-deployment + labels: + app: server-b +spec: + replicas: 1 + selector: + matchLabels: + app: server-b + template: + metadata: + labels: + app: server-b + spec: + containers: + - name: server-b + env: + - name: SERVER_PORT + value: "8080" + - name: RANDOM_NUM_LENGTH + value: "2" + image: xds.example/server:latest + imagePullPolicy: Never + ports: + - containerPort: 8080 + name: grpc-port + protocol: TCP + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 100m + memory: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + note: server-b + name: server-b +spec: + ports: + - name: grpc-port + port: 8080 + protocol: TCP + targetPort: grpc-port + selector: + app: server-b + clusterIP: None diff --git a/example/k8s/server.deployment.yaml b/example/k8s/server.deployment.yaml index 7ed5922..913f876 100644 --- a/example/k8s/server.deployment.yaml +++ b/example/k8s/server.deployment.yaml @@ -19,6 +19,8 @@ spec: env: - name: SERVER_PORT value: "8080" + - name: RANDOM_NUM_LENGTH + value: "10" image: xds.example/server:latest imagePullPolicy: Never ports: diff --git a/example/k8s/xds-controller.deployment.yaml b/example/k8s/xds-controller.deployment.yaml index d5515ab..3eb6a85 100644 --- a/example/k8s/xds-controller.deployment.yaml +++ b/example/k8s/xds-controller.deployment.yaml @@ -105,6 +105,19 @@ data: num_retries: 5 backoff_interval_in_sec: 1 backoff_max_interval_in_sec: 5 + - name: grpc-server-b-cdsf + resolver_type: k8 + address: server-b + name_space: xds-test + port: 8080 + lb_policy: ROUND_ROBIN + max_requests: 1024 + retry: + enabled: true + retry_on: "unavailable,resource-exhausted,internal,deadline-exceeded,cancelled" + num_retries: 5 + backoff_interval_in_sec: 1 + backoff_max_interval_in_sec: 5 --- apiVersion: v1 kind: ServiceAccount diff --git a/example/server/main.go b/example/server/main.go index ee19842..6df606b 100644 --- a/example/server/main.go +++ b/example/server/main.go @@ -29,11 +29,15 @@ var ( func main() { // get Env variables serverPort := getEnv("SERVER_PORT", "5432") + randLength := getEnv("RANDOM_NUM_LENGTH", "10") lis, err := net.Listen("tcp", ":"+serverPort) + if err != nil { log.Fatalf("failed to listen: %v", err) } - serverID = randNum(3) + n, _ := strconv.Atoi(randLength) + + serverID = randNum(n) s := grpc.NewServer() personpb.RegisterPersonServer(s, &server{}) log.Println(" Server Started !!") @@ -59,7 +63,7 @@ func getEnv(key, defaultVal string) string { return v } -// 9 for ssn +// generates a rand number of specified length func randNum(length int) int32 { var b string for i := 0; i < length; i++ { diff --git a/example/server/main_test.go b/example/server/main_test.go new file mode 100644 index 0000000..058a13b --- /dev/null +++ b/example/server/main_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "testing" +) + +func Test_randNum(t *testing.T) { + type args struct { + length int + } + tests := []struct { + name string + args args + want int32 + }{ + {"Length test", args{length: 5}, 12345}, + {"Length test", args{length: 6}, 132345}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := randNum(tt.args.length) + t.Log("Matc: randNum() = " + fmt.Sprint(got) + ", want " + fmt.Sprint(tt.want)) + if getLength(got) != getLength(tt.want) { + t.Errorf("randNum() = %v, want %v", got, tt.want) + } + }) + } +} + +func getLength(num int32) int32 { + var c int32 = 0 + for num != 0 { + num /= 10 + c += 1 + } + return c +} diff --git a/go.mod b/go.mod index 07ba907..4a16ba0 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ require ( github.com/envoyproxy/go-control-plane v0.10.3 github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 - github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_golang v1.14.0 google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.25.3 - k8s.io/apimachinery v0.25.3 - k8s.io/client-go v0.25.3 + k8s.io/api v0.25.4 + k8s.io/apimachinery v0.25.4 + k8s.io/client-go v0.25.4 ) diff --git a/go.sum b/go.sum index ac34074..8318a1c 100644 --- a/go.sum +++ b/go.sum @@ -327,13 +327,14 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -836,12 +837,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= -k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= -k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= -k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0= -k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= +k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= +k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= +k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc= +k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= +k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=