From 8e28bbf662d9f2dfbbdfe9b94e7be93d2e6d452f Mon Sep 17 00:00:00 2001 From: Jeffrey Nelson Date: Tue, 6 Feb 2024 16:35:24 -0600 Subject: [PATCH] Merge master into release-1.16 for upcoming v1.16.3 release; remove #2785 (#2788) --- .github/workflows/deps.yml | 4 +- .github/workflows/issue-closed-message.yaml | 6 +- Makefile | 2 +- README.md | 6 +- charts/aws-vpc-cni/README.md | 5 + charts/cni-metrics-helper/README.md | 1 + .../templates/deployment.yaml | 6 + charts/cni-metrics-helper/values.yaml | 2 + cmd/aws-vpc-cni/main.go | 6 +- go.mod | 54 ++--- go.sum | 122 +++++----- pkg/ipamd/datastore/data_store.go | 11 +- pkg/ipamd/datastore/data_store_test.go | 77 +++++-- pkg/ipamd/ipamd.go | 208 +++++++++--------- pkg/ipamd/ipamd_test.go | 81 ++++--- pkg/rpcwrapper/mocks/rpcwrapper_mocks.go | 12 +- rpc/mocks/rpc_mocks.go | 16 +- rpc/rpc.pb.go | 9 +- scripts/generate-cni-yaml.sh | 2 +- scripts/run-static-canary.sh | 1 + test/agent/Dockerfile | 2 +- test/agent/go.mod | 2 +- test/agent/go.sum | 4 +- test/framework/options.go | 2 + .../resources/aws/services/cloudwatch.go | 5 + test/framework/resources/aws/services/ec2.go | 7 + .../resources/k8s/resources/daemonset.go | 2 +- .../cni/pod_traffic_across_az_test.go | 139 ++++++++++-- testdata/deploy-130-pods.yaml | 2 +- testdata/deploy-5000-pods.yaml | 2 +- testdata/deploy-730-pods.yaml | 2 +- 31 files changed, 480 insertions(+), 320 deletions(-) diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index cbd82b64c9..8612ad4068 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -26,14 +26,14 @@ jobs: - id: govulncheck uses: ./.github/actions/govulncheck with: - go-version-input: 1.21.5 + go-version-input: 1.21.6 go-version-file: go.mod cache: false repo-checkout: false - id: govulncheck-tests-agent uses: ./.github/actions/govulncheck with: - go-version-input: 1.21.5 + go-version-input: 1.21.6 go-version-file: test/agent/go.mod cache: false repo-checkout: false diff --git a/.github/workflows/issue-closed-message.yaml b/.github/workflows/issue-closed-message.yaml index eaf88eb149..0d066bc709 100644 --- a/.github/workflows/issue-closed-message.yaml +++ b/.github/workflows/issue-closed-message.yaml @@ -15,7 +15,5 @@ jobs: # These inputs are both required repo-token: "${{ secrets.GITHUB_TOKEN }}" message: | - ### ⚠️COMMENT VISIBILITY WARNING⚠️ - Comments on closed issues are hard for our team to see. - If you need more assistance, please open a new issue that references this one. - If you wish to keep having a conversation with other community members under this issue feel free to do so. + This issue is now closed. Comments on closed issues are hard for our team to see. + If you need more assistance, please either tag a team member or open a new issue that references this one. diff --git a/Makefile b/Makefile index e18ab3b26c..f94cf95aed 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ # VERSION is the source revision that executables and images are built from. VERSION ?= $(shell git describe --tags --always --dirty || echo "unknown") # GOLANG_IMAGE is the building golang container image used. -GOLANG_IMAGE ?= public.ecr.aws/eks-distro-build-tooling/golang:1.21.5-6-gcc-al2 +GOLANG_IMAGE ?= public.ecr.aws/eks-distro-build-tooling/golang:1.21.6-7-gcc-al2 # BASE_IMAGE_CNI is the base layer image for the primary AWS VPC CNI plugin container BASE_IMAGE_CNI ?= public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-iptables:latest.2 # BASE_IMAGE_CNI_INIT is the base layer image for the AWS VPC CNI init container diff --git a/README.md b/README.md index 5dedb76b7b..1eafcf0e7d 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,9 @@ See [here](./docs/iam-policy.md) for required IAM policies. * `make` defaults to `make build-linux` that builds the Linux binaries. * `unit-test`, `format`,`lint` and `vet` provide ways to run the respective tests/tools and should be run before submitting a PR. -* `make docker` will create a docker container using the docker-build with the finished binaries, with a tag of `amazon/amazon-k8s-cni:latest` -* `make docker-build` uses a docker container (golang:1.16) to build the binaries. -* `make docker-unit-tests` uses a docker container (golang:1.16) to run all unit tests. +* `make docker` will create a docker container using `docker buildx` that contains the finished binaries, with a tag of `amazon/amazon-k8s-cni:latest` +* `make docker-unit-tests` uses a docker container to run all unit tests. +* builds for all build and test actions run in docker containers based on `golang:1.21.5-6-gcc-al2` unless a different `GOLANG_IMAGE` tag is passed in. ## Components diff --git a/charts/aws-vpc-cni/README.md b/charts/aws-vpc-cni/README.md index ebb47757c3..b2202141e8 100644 --- a/charts/aws-vpc-cni/README.md +++ b/charts/aws-vpc-cni/README.md @@ -122,6 +122,11 @@ for kind in daemonSet clusterRole clusterRoleBinding serviceAccount; do kubectl -n kube-system annotate --overwrite $kind aws-node meta.helm.sh/release-namespace=kube-system kubectl -n kube-system label --overwrite $kind aws-node app.kubernetes.io/managed-by=Helm done + +kubectl -n kube-system annotate --overwrite configmap amazon-vpc-cni meta.helm.sh/release-name=YOUR_HELM_RELEASE_NAME_HERE +kubectl -n kube-system annotate --overwrite configmap amazon-vpc-cni meta.helm.sh/release-namespace=kube-system +kubectl -n kube-system label --overwrite configmap amazon-vpc-cni app.kubernetes.io/managed-by=Helm + ``` ## Migrate from Helm v2 to Helm v3 diff --git a/charts/cni-metrics-helper/README.md b/charts/cni-metrics-helper/README.md index 8b062ece26..f7e2bf7dce 100644 --- a/charts/cni-metrics-helper/README.md +++ b/charts/cni-metrics-helper/README.md @@ -59,6 +59,7 @@ The following table lists the configurable parameters for this chart and their d | serviceAccount.name | The name of the ServiceAccount to use | nil | | serviceAccount.create | Specifies whether a ServiceAccount should be created | true | | serviceAccount.annotations | Specifies the annotations for ServiceAccount | {} | +| podAnnotations | Specifies the annotations for pods | {} | | revisionHistoryLimit | The number of revisions to keep | 10 | | podSecurityContext | SecurityContext to set on the pod | {} | | containerSecurityContext | SecurityContext to set on the container | {} | diff --git a/charts/cni-metrics-helper/templates/deployment.yaml b/charts/cni-metrics-helper/templates/deployment.yaml index 70f75d4e20..adadf2bf7d 100644 --- a/charts/cni-metrics-helper/templates/deployment.yaml +++ b/charts/cni-metrics-helper/templates/deployment.yaml @@ -12,6 +12,12 @@ spec: k8s-app: cni-metrics-helper template: metadata: + {{- if .Values.podAnnotations }} + annotations: + {{- range $key, $value := .Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} labels: k8s-app: cni-metrics-helper spec: diff --git a/charts/cni-metrics-helper/values.yaml b/charts/cni-metrics-helper/values.yaml index 1de08b33ba..d720e7da5c 100644 --- a/charts/cni-metrics-helper/values.yaml +++ b/charts/cni-metrics-helper/values.yaml @@ -34,3 +34,5 @@ revisionHistoryLimit: 10 podSecurityContext: {} containerSecurityContext: {} + +podAnnotations: {} diff --git a/cmd/aws-vpc-cni/main.go b/cmd/aws-vpc-cni/main.go index 686b2dd3e9..7736be90f0 100644 --- a/cmd/aws-vpc-cni/main.go +++ b/cmd/aws-vpc-cni/main.go @@ -270,8 +270,10 @@ func generateJSON(jsonFile string, outFile string, getPrimaryIP func(ipv4 bool) if egressEnabled { nodeIP, err = getPrimaryIP(false) if err != nil { - log.Errorf("To support IPv6 egress, node primary ENI must have a global IPv6 address, error: %v", err) - return err + // When ENABLE_V6_EGRESS is set, but the node is lacking an IPv6 address, log a warning and disable the egress-v6-cni plugin. + // This allows IPv4-only nodes to function while still alerting the customer to the possibility of a misconfiguration. + log.Warnf("To support IPv6 egress, node primary ENI must have a global IPv6 address, error: %v", err) + egressEnabled = false } } } diff --git a/go.mod b/go.mod index b9a229e897..d06c5a7b11 100644 --- a/go.mod +++ b/go.mod @@ -14,30 +14,29 @@ require ( github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.6.0 - github.com/onsi/ginkgo/v2 v2.13.2 + github.com/onsi/ginkgo/v2 v2.14.0 github.com/onsi/gomega v1.30.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_golang v1.18.0 github.com/prometheus/client_model v0.5.0 - github.com/prometheus/common v0.44.0 - github.com/samber/lo v1.38.1 + github.com/prometheus/common v0.45.0 + github.com/samber/lo v1.39.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 go.uber.org/zap v1.26.0 - golang.org/x/net v0.17.0 - golang.org/x/sys v0.15.0 - google.golang.org/grpc v1.60.1 - google.golang.org/protobuf v1.31.0 + golang.org/x/net v0.19.0 + golang.org/x/sys v0.16.0 + google.golang.org/grpc v1.61.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 - helm.sh/helm/v3 v3.13.2 + helm.sh/helm/v3 v3.14.0 k8s.io/api v0.29.0 k8s.io/apimachinery v0.29.0 k8s.io/cli-runtime v0.29.0 k8s.io/client-go v0.29.0 - sigs.k8s.io/controller-runtime v0.16.3 + sigs.k8s.io/controller-runtime v0.17.0 ) require ( @@ -66,16 +65,16 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -88,7 +87,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/gosuri/uitable v0.0.4 // indirect @@ -111,7 +110,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect @@ -128,13 +127,13 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc5 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/safchain/ethtool v0.3.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -148,30 +147,31 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sync v0.4.0 // indirect + golang.org/x/oauth2 v0.14.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/tools v0.16.1 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.3 // indirect - k8s.io/apiserver v0.28.3 // indirect - k8s.io/component-base v0.28.3 // indirect + k8s.io/apiextensions-apiserver v0.29.0 // indirect + k8s.io/apiserver v0.29.0 // indirect + k8s.io/component-base v0.29.0 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/kubectl v0.28.2 // indirect + k8s.io/kubectl v0.29.0 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) replace gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 => gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 2cf4212f75..cab9ad8eb4 100644 --- a/go.sum +++ b/go.sum @@ -39,7 +39,6 @@ github.com/aws/amazon-vpc-resource-controller-k8s v1.4.1 h1:43uJXFNTjk5Gzi2Qpqk3 github.com/aws/amazon-vpc-resource-controller-k8s v1.4.1/go.mod h1:tXPJP0SFdkVa7ALghDjThtavyYnP0MKO8V0ZHlDNCU8= github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y= github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -75,7 +74,7 @@ github.com/containernetworking/plugins v1.4.0 h1:+w22VPYgk7nQHw7KT92lsRmuToHvb7w github.com/containernetworking/plugins v1.4.0/go.mod h1:UYhcOyjefnrQvKvmmyEKsUA+M9Nfn7tqULPpH0Pkcj0= github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -108,10 +107,10 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= @@ -124,8 +123,8 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= @@ -134,14 +133,13 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -210,8 +208,8 @@ github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHa github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= @@ -240,7 +238,6 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -262,7 +259,6 @@ github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -301,8 +297,8 @@ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/miekg/dns v1.1.25 h1:dFwPR6SfLtrSwgDcIq2bcU/gVutB4sNApq2HBdqcakg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -343,8 +339,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= -github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -359,7 +355,6 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -369,8 +364,8 @@ github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjz github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -378,13 +373,13 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= @@ -393,8 +388,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= -github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= -github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -406,8 +401,8 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -461,14 +456,10 @@ go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1 go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= @@ -479,19 +470,18 @@ golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= +golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -501,8 +491,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -526,12 +516,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= @@ -554,11 +544,10 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -572,13 +561,13 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -594,7 +583,6 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -616,36 +604,36 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= -helm.sh/helm/v3 v3.13.2 h1:IcO9NgmmpetJODLZhR3f3q+6zzyXVKlRizKFwbi7K8w= -helm.sh/helm/v3 v3.13.2/go.mod h1:GIHDwZggaTGbedevTlrQ6DB++LBN6yuQdeGj0HNaDx0= +helm.sh/helm/v3 v3.14.0 h1:TaZIH6uOchn7L27ptwnnuHJiFrT/BsD4dFdp/HLT2nM= +helm.sh/helm/v3 v3.14.0/go.mod h1:2itvvDv2WSZXTllknfQo6j7u3VVgMAvm8POCDgYH424= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= -k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= -k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= -k8s.io/apiserver v0.28.3 h1:8Ov47O1cMyeDzTXz0rwcfIIGAP/dP7L8rWbEljRcg5w= -k8s.io/apiserver v0.28.3/go.mod h1:YIpM+9wngNAv8Ctt0rHG4vQuX/I5rvkEMtZtsxW2rNM= +k8s.io/apiserver v0.29.0 h1:Y1xEMjJkP+BIi0GSEv1BBrf1jLU9UPfAnnGGbbDdp7o= +k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM= k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4= k8s.io/cli-runtime v0.29.0/go.mod h1:VKudXp3X7wR45L+nER85YUzOQIru28HQpXr0mTdeCrk= k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= -k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= -k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= +k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/kubectl v0.28.2 h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM= -k8s.io/kubectl v0.28.2/go.mod h1:6EQWTPySF1fn7yKoQZHYf9TPwIl2AygHEcJoxFekr64= +k8s.io/kubectl v0.29.0 h1:Oqi48gXjikDhrBF67AYuZRTcJV4lg2l42GmvsP7FmYI= +k8s.io/kubectl v0.29.0/go.mod h1:0jMjGWIcMIQzmUaMgAzhSELv5WtHo2a8pq67DtviAJs= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= +sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= @@ -654,5 +642,5 @@ sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index b85b58510b..ee4c5d1e9e 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -841,8 +841,7 @@ func (ds *DataStore) GetEFAENIs() map[string]bool { return ret } -// IsRequiredForWarmIPTarget determines if this ENI has warm IPs that are required to fulfill whatever WARM_IP_TARGET is -// set to. +// IsRequiredForWarmIPTarget determines if this ENI has warm IPs that are required to fulfill whatever WARM_IP_TARGET is set to. func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool { otherWarmIPs := 0 for _, other := range ds.eniPool { @@ -863,8 +862,7 @@ func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool return otherWarmIPs < warmIPTarget } -// IsRequiredForMinimumIPTarget determines if this ENI is necessary to fulfill whatever MINIMUM_IP_TARGET is -// set to. +// IsRequiredForMinimumIPTarget determines if this ENI is necessary to fulfill whatever MINIMUM_IP_TARGET is set to. func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENI) bool { otherIPs := 0 for _, other := range ds.eniPool { @@ -885,8 +883,7 @@ func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENI) return otherIPs < minimumIPTarget } -// IsRequiredForWarmPrefixTarget determines if this ENI is necessary to fulfill whatever WARM_PREFIX_TARGET is -// set to. +// IsRequiredForWarmPrefixTarget determines if this ENI is necessary to fulfill whatever WARM_PREFIX_TARGET is set to. func (ds *DataStore) isRequiredForWarmPrefixTarget(warmPrefixTarget int, eni *ENI) bool { freePrefixes := 0 for _, other := range ds.eniPool { @@ -1007,7 +1004,6 @@ func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget, minimumIPTarget, war } removableENI := deletableENI.ID - for _, availableCidr := range ds.eniPool[removableENI].AvailableIPv4Cidrs { ds.total -= availableCidr.Size() if availableCidr.IsPrefix { @@ -1277,7 +1273,6 @@ func (ds *DataStore) GetFreePrefixes() int { freePrefixes++ } } - } return freePrefixes } diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index eeeac95d8a..5e1925fdc5 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -1024,18 +1024,21 @@ func TestWarmENIInteractions(t *testing.T) { _ = ds.AddENI("eni-2", 2, false, false, false) _ = ds.AddENI("eni-3", 3, false, false, false) + // Add an IP address to ENI 1 and assign it to a pod ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) key1 := IPAMKey{"net0", "sandbox-1", "eth0"} _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) + // Add another IP address to ENI 1 and assign a pod ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) key2 := IPAMKey{"net0", "sandbox-2", "eth0"} _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) + // Add two IP addresses to ENI 2 and one IP address to ENI 3 ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.2.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-2", ipv4Addr, false) ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.2.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} @@ -1043,44 +1046,76 @@ func TestWarmENIInteractions(t *testing.T) { ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.3.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-3", ipv4Addr, false) - noWarmIPTarget := 0 - ds.eniPool["eni-2"].createTime = time.Time{} ds.eniPool["eni-3"].createTime = time.Time{} - // We have three ENIs, 5 IPs and two pods on ENI 1. Each ENI can handle two pods. + // We have 3 ENIs, 5 IPs and 2 pods on ENI 1. + // ENI 1: 2 IPs allocated, 2 IPs in use + // ENI 2: 2 IPs allocated, 0 IPs in use + // ENI 3: 1 IP allocated, 0 IPs in use + // => 3 free IPs // We should not be able to remove any ENIs if either warmIPTarget >= 3 or minimumWarmIPTarget >= 5 + + // WARM IP TARGET=3, MINIMUM_IP_TARGET=1 => no ENI should be removed eni := ds.RemoveUnusedENIFromStore(3, 1, 0) assert.Equal(t, "", eni) - // Should not be able to free this ENI because we want at least 5 IPs, which requires at least three ENIs + + // WARM IP TARGET=1, MINIMUM_IP_TARGET=5 => no ENI should be removed eni = ds.RemoveUnusedENIFromStore(1, 5, 0) assert.Equal(t, "", eni) - // Should be able to free an ENI because both warmIPTarget and minimumWarmIPTarget are both effectively 4 + + // WARM IP TARGET=2, MINIMUM_IP_TARGET=4 => ENI 3 should be removed as we only need 2 free IPs, which ENI 2 has removedEni := ds.RemoveUnusedENIFromStore(2, 4, 0) - assert.Contains(t, []string{"eni-2", "eni-3"}, removedEni) + assert.Equal(t, "eni-3", removedEni) - // Should not be able to free an ENI because minimumWarmIPTarget requires at least two ENIs and no warm IP target - eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, 3, 0) - assert.Equal(t, "", eni) - // Should be able to free an ENI because one ENI can provide a minimum count of 2 IPs - secondRemovedEni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, 2, 0) - assert.Contains(t, []string{"eni-2", "eni-3"}, secondRemovedEni) + // We have 2 ENIs, 4 IPs and 2 pods on ENI 1. + // ENI 1: 2 IPs allocated, 2 IPs in use + // ENI 2: 2 IPs allocated, 0 IPs in use + // => 2 free IPs - assert.NotEqual(t, removedEni, secondRemovedEni, "The two removed ENIs should not be the same ENI.") + // WARM IP TARGET=0, MINIMUM_IP_TARGET=3 => no ENI should be removed + eni = ds.RemoveUnusedENIFromStore(0, 3, 0) + assert.Equal(t, "", eni) - _ = ds.AddENI("eni-4", 3, false, true, false) - _ = ds.AddENI("eni-5", 3, false, false, true) + // WARM IP TARGET=0, MINIMUM_IP_TARGET=2 => ENI 2 should be removed as ENI 1 covers the requirements + removedEni = ds.RemoveUnusedENIFromStore(0, 2, 0) + assert.Contains(t, "eni-2", removedEni) + // Add 2 more ENIs to the datastore and add 1 IP address to each of them + ds.AddENI("eni-4", 4, false, true, false) // trunk ENI + ds.AddENI("eni-5", 5, false, false, true) // EFA ENI ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.4.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} - _ = ds.AddIPv4CidrToStore("eni-4", ipv4Addr, false) + ds.AddIPv4CidrToStore("eni-4", ipv4Addr, false) ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.5.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} - _ = ds.AddIPv4CidrToStore("eni-5", ipv4Addr, false) - + ds.AddIPv4CidrToStore("eni-5", ipv4Addr, false) ds.eniPool["eni-4"].createTime = time.Time{} ds.eniPool["eni-5"].createTime = time.Time{} - thirdRemovedEni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, 2, 0) - // None of the others can be removed... - assert.Equal(t, "", thirdRemovedEni) + + // We have 3 ENIs, 4 IPs and 2 pods on ENI 1. + // ENI 1: 2 IPs allocated, 2 IPs in use + // ENI 4: 1 IPs allocated, 0 IPs in use + // ENI 5: 1 IPs allocated, 0 IPs in use + // => 2 free IPs + + // WARM IP TARGET=0, MINIMUM_IP_TARGET=2 => no ENI can be removed because ENI 4 is a trunk ENI and ENI 5 is an EFA ENI + removedEni = ds.RemoveUnusedENIFromStore(0, 2, 0) + assert.Equal(t, "", removedEni) + assert.Equal(t, 3, ds.GetENIs()) + + // Add 1 more normal ENI to the datastore + ds.AddENI("eni-6", 6, false, false, false) // trunk ENI + ds.eniPool["eni-6"].createTime = time.Time{} + + // We have 4 ENIs, 4 IPs and 2 pods on ENI 1. + // ENI 1: 2 IPs allocated, 2 IPs in use + // ENI 4: 1 IPs allocated, 0 IPs in use + // ENI 5: 1 IPs allocated, 0 IPs in use + // ENI 6: 0 IPs allocated, 0 IPs in use + // => 2 free IPs + + // WARM IP TARGET=0, MINIMUM_IP_TARGET=2 => ENI 6 can be removed + removedEni = ds.RemoveUnusedENIFromStore(0, 2, 0) + assert.Equal(t, "eni-6", removedEni) assert.Equal(t, 3, ds.GetENIs()) } diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 7f958647e8..6533149900 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -154,10 +154,10 @@ const ( ipV4AddrFamily = "4" ipV6AddrFamily = "6" - //insufficientCidrErrorCooldown is the amount of time reconciler will wait before trying to fetch - //more IPs/prefixes for an ENI. With InsufficientCidr we know the subnet doesn't have enough IPs so - //instead of retrying every 5s which would lead to increase in EC2 AllocIPAddress calls, we wait for - //120 seconds for a retry. + // insufficientCidrErrorCooldown is the amount of time reconciler will wait before trying to fetch + // more IPs/prefixes for an ENI. With InsufficientCidr we know the subnet doesn't have enough IPs so + // instead of retrying every 5s which would lead to increase in EC2 AllocIPAddress calls, we wait for + // 120 seconds for a retry. insufficientCidrErrorCooldown = 120 * time.Second // envManageUntaggedENI is used to determine if untagged ENIs should be managed or unmanaged @@ -216,6 +216,7 @@ type IPAMContext struct { lastInsufficientCidrError time.Time enableManageUntaggedMode bool enablePodIPAnnotation bool + maxPods int // maximum number of pods that can be scheduled on the node } // setUnmanagedENIs will rebuild the set of ENI IDs for ENIs tagged as "no_manage" @@ -499,17 +500,26 @@ func (c *IPAMContext) nodeInit() error { }, 30*time.Second) } + // Make a k8s client request for the current node so that max pods can be derived + node, err := k8sapi.GetNode(ctx, c.k8sClient) + if err != nil { + log.Errorf("Failed to get node", err) + podENIErrInc("nodeInit") + return err + } + + maxPods, isInt64 := node.Status.Capacity.Pods().AsInt64() + if !isInt64 { + log.Errorf("Failed to parse max pods: %s", node.Status.Capacity.Pods().String) + podENIErrInc("nodeInit") + return errors.New("error while trying to determine max pods") + } + c.maxPods = int(maxPods) + if c.useCustomNetworking { // When custom networking is enabled and a valid ENIConfig is found, IPAMD patches the CNINode // resource for this instance. The operation is safe as enabling/disabling custom networking // requires terminating the previous instance. - node, err := k8sapi.GetNode(ctx, c.k8sClient) - if err != nil { - log.Errorf("Failed to get node", err) - podENIErrInc("nodeInit") - return err - } - eniConfigName, err := eniconfig.GetNodeSpecificENIConfigName(node) if err == nil && eniConfigName != "default" { // If Security Groups for Pods is enabled, the VPC Resource Controller must also know that Custom Networking is enabled @@ -528,7 +538,8 @@ func (c *IPAMContext) nodeInit() error { } // On node init, check if datastore pool needs to be increased. If so, attach CIDRs from existing ENIs and attach new ENIs. - if !c.disableENIProvisioning && c.isDatastorePoolTooLow() { + datastorePoolTooLow, _ := c.isDatastorePoolTooLow() + if !c.disableENIProvisioning && datastorePoolTooLow { if err := c.increaseDatastorePool(ctx); err != nil { // Note that the only error currently returned by increaseDatastorePool is an error attaching CIDRs (other than insufficient IPs) podENIErrInc("nodeInit") @@ -616,6 +627,8 @@ func (c *IPAMContext) StartNodeIPPoolManager() { return } + log.Infof("IP pool manager - max pods: %d, warm IP target: %d, warm prefix target: %d, warm ENI target: %d, minimum IP target: %d", + c.maxPods, c.warmIPTarget, c.warmPrefixTarget, c.warmENITarget, c.minimumIPTarget) sleepDuration := ipPoolMonitorInterval / 2 ctx := context.Background() for { @@ -634,9 +647,17 @@ func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) { c.tryEnableSecurityGroupsForPods(ctx) } - if c.isDatastorePoolTooLow() { - c.increaseDatastorePool(ctx) - } else if c.isDatastorePoolTooHigh() { + datastorePoolTooLow, stats := c.isDatastorePoolTooLow() + // Each iteration, log the current datastore IP stats + log.Debugf("IP stats - total IPs: %d, assigned IPs: %d, cooldown IPs: %d", stats.TotalIPs, stats.AssignedIPs, stats.CooldownIPs) + + if datastorePoolTooLow { + // Allow for rapid scale up to decrease time it takes for pod to retrieve an ip + // but conservative scale down to account for pod churn + for datastorePoolStillTooLow := datastorePoolTooLow; datastorePoolStillTooLow; datastorePoolStillTooLow, _ = c.isDatastorePoolTooLow() { + c.increaseDatastorePool(ctx) + } + } else if c.isDatastorePoolTooHigh(stats) { c.decreaseDatastorePool(decreaseIPPoolInterval) } if c.shouldRemoveExtraENIs() { @@ -646,6 +667,7 @@ func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) { // decreaseDatastorePool runs every `interval` and attempts to return unused ENIs and IPs func (c *IPAMContext) decreaseDatastorePool(interval time.Duration) { + log.Debug("Starting to decrease pool size") prometheusmetrics.IpamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Add(float64(1)) defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Sub(float64(1)) @@ -658,7 +680,6 @@ func (c *IPAMContext) decreaseDatastorePool(interval time.Duration) { log.Debugf("Starting to decrease Datastore pool") c.tryUnassignCidrsFromAll() - c.lastDecreaseIPPool = now c.lastNodeIPPoolAction = now @@ -693,12 +714,11 @@ func (c *IPAMContext) tryFreeENI() { } -// tryUnassignIPsorPrefixesFromAll determines if there are IPs to free when we have extra IPs beyond the target and warmIPTargetDefined -// is enabled, deallocate extra IP addresses +// When warm IP/prefix targets are defined, free extra IPs func (c *IPAMContext) tryUnassignCidrsFromAll() { - _, over, warmTargetDefined := c.datastoreTargetState() + _, over, warmIPTargetsDefined := c.datastoreTargetState(nil) // If WARM IP targets are not defined, check if WARM_PREFIX_TARGET is defined. - if !warmTargetDefined { + if !warmIPTargetsDefined { over = c.computeExtraPrefixesOverWarmTarget() } @@ -746,25 +766,12 @@ func (c *IPAMContext) tryUnassignCidrsFromAll() { } } +// PRECONDITION: isDatastorePoolTooLow returned true func (c *IPAMContext) increaseDatastorePool(ctx context.Context) error { log.Debug("Starting to increase pool size") prometheusmetrics.IpamdActionsInprogress.WithLabelValues("increaseDatastorePool").Add(float64(1)) defer prometheusmetrics.IpamdActionsInprogress.WithLabelValues("increaseDatastorePool").Sub(float64(1)) - short, _, warmIPTargetDefined := c.datastoreTargetState() - if warmIPTargetDefined && short == 0 { - log.Debugf("Skipping increase Datastore pool, warm target reached") - return nil - } - - if !warmIPTargetDefined { - shortPrefix, warmTargetDefined := c.datastorePrefixTargetState() - if warmTargetDefined && shortPrefix == 0 { - log.Debugf("Skipping increase Datastore pool, warm prefix target reached") - return nil - } - } - if c.isTerminating() { log.Debug("AWS CNI is terminating, will not try to attach any new IPs or ENIs right now") return nil @@ -811,11 +818,6 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) error { func (c *IPAMContext) updateLastNodeIPPoolAction() { c.lastNodeIPPoolAction = time.Now() stats := c.dataStore.GetIPStats(ipV4AddrFamily) - if !c.enablePrefixDelegation { - log.Debugf("Successfully increased IP pool: %s", stats) - } else { - log.Debugf("Successfully increased Prefix pool: %s", stats) - } c.logPoolStats(stats) } @@ -873,42 +875,28 @@ func (c *IPAMContext) tryAllocateENI(ctx context.Context) error { return nil } -// For an ENI, try to fill in missing IPs on an existing ENI with PD disabled -// try to fill in missing Prefixes on an existing ENI with PD enabled +// For an ENI, fill in missing IPs or prefixes. +// PRECONDITION: isDatastorePoolTooLow returned true func (c *IPAMContext) tryAssignCidrs() (increasedPool bool, err error) { - short, _, warmIPTargetDefined := c.datastoreTargetState() - if warmIPTargetDefined && short == 0 { - log.Infof("Warm IP target set and short is 0 so not assigning Cidrs (IPs or Prefixes)") - return false, nil - } - - if !warmIPTargetDefined { - shortPrefix, warmTargetDefined := c.datastorePrefixTargetState() - if warmTargetDefined && shortPrefix == 0 { - log.Infof("Warm prefix target set and short is 0 so not assigning Cidrs (Prefixes)") - return false, nil - } - } - - if !c.enablePrefixDelegation { - return c.tryAssignIPs() - } else { + if c.enablePrefixDelegation { return c.tryAssignPrefixes() + } else { + return c.tryAssignIPs() } } -// For an ENI, try to fill in missing IPs on an existing ENI +// For an ENI, try to fill in missing IPs on an existing ENI. +// PRECONDITION: isDatastorePoolTooLow returned true func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) { // If WARM_IP_TARGET is set, only proceed if we are short of target - short, _, warmIPTargetDefined := c.datastoreTargetState() - if warmIPTargetDefined && short == 0 { + short, _, warmIPTargetsDefined := c.datastoreTargetState(nil) + if warmIPTargetsDefined && short == 0 { return false, nil } - // If WARM_IP_TARGET is set we only want to allocate up to that target - // to avoid overallocating and releasing + // If WARM_IP_TARGET is set we only want to allocate up to that target to avoid overallocating and releasing toAllocate := c.maxIPsPerENI - if warmIPTargetDefined { + if warmIPTargetsDefined { toAllocate = short } @@ -995,6 +983,7 @@ func (c *IPAMContext) assignIPv6Prefix(eniID string) (err error) { return nil } +// PRECONDITION: isDatastorePoolTooLow returned true func (c *IPAMContext) tryAssignPrefixes() (increasedPool bool, err error) { toAllocate := c.getPrefixesNeeded() // Returns an ENI which has space for more prefixes to be attached, but this @@ -1233,14 +1222,13 @@ func (c *IPAMContext) tryEnableSecurityGroupsForPods(ctx context.Context) { } } -// shouldRemoveExtraENIs returns true if we should attempt to find an ENI to free. When WARM_IP_TARGET is set, we -// always check and do verification in getDeletableENI() -// PD enabled : If the WARM_PREFIX_TARGET is spread across ENIs and we have more than needed then this function will return true. -// but if the number of prefixes are on just one ENI and is more than available even then it returns true so getDeletableENI will +// shouldRemoveExtraENIs returns true if we should attempt to find an ENI to free +// PD enabled: If the WARM_PREFIX_TARGET is spread across ENIs and we have more than needed, this function will return true. +// If the number of prefixes are on just one ENI, and there are more than available, it returns true so getDeletableENI will // recheck if we need the ENI for prefix target. func (c *IPAMContext) shouldRemoveExtraENIs() bool { - _, _, warmTargetDefined := c.datastoreTargetState() - if warmTargetDefined { + // When WARM_IP_TARGET is set, return true as verification is always done in getDeletableENI() + if c.warmIPTargetsDefined() { return true } @@ -1256,29 +1244,27 @@ func (c *IPAMContext) shouldRemoveExtraENIs() bool { } shouldRemoveExtra = available >= (warmTarget)*c.maxIPsPerENI - if shouldRemoveExtra { c.logPoolStats(stats) log.Debugf("It might be possible to remove extra ENIs because available (%d) >= (ENI/Prefix target + 1 (%d) + 1) * addrsPerENI (%d)", available, warmTarget, c.maxIPsPerENI) } else if c.enablePrefixDelegation { - // When prefix target count is reduced, datastorehigh would have deleted extra prefixes over the warm prefix target. - // Hence available will be less than (warmTarget)*c.maxIPsPerENI but there can be some extra ENIs which are not used hence see if we can clean it up. + // When prefix target count is reduced, datastore would have deleted extra prefixes over the warm prefix target. + // Hence available will be less than (warmTarget)*c.maxIPsPerENI, but there can be some extra ENIs which are not used hence see if we can clean it up. shouldRemoveExtra = c.dataStore.CheckFreeableENIexists() } return shouldRemoveExtra } func (c *IPAMContext) computeExtraPrefixesOverWarmTarget() int { - over := 0 if !c.warmPrefixTargetDefined() { - return over + return 0 } freePrefixes := c.dataStore.GetFreePrefixes() - over = max(freePrefixes-c.warmPrefixTarget, 0) + over := max(freePrefixes-c.warmPrefixTarget, 0) stats := c.dataStore.GetIPStats(ipV4AddrFamily) - log.Debugf("computeExtraPrefixesOverWarmTarget available %d over %d warm_prefix_target %d", stats.AvailableAddresses(), over, c.warmPrefixTarget) + log.Debugf("computeExtraPrefixesOverWarmTarget - available: %d, over: %d, warm_prefix_target: %d", stats.AvailableAddresses(), over, c.warmPrefixTarget) c.logPoolStats(stats) return over } @@ -1666,6 +1652,11 @@ func (c *IPAMContext) verifyAndAddPrefixesToDatastore(eni string, attachedENIPre return seenIPs } +// return true when WARM_IP_TARGET or MINIMUM_IP_TARGET is defined +func (c *IPAMContext) warmIPTargetsDefined() bool { + return c.warmIPTarget != noWarmIPTarget || c.minimumIPTarget != noMinimumIPTarget +} + // UseCustomNetworkCfg returns whether Pods needs to use pod specific configuration or not. func UseCustomNetworkCfg() bool { return parseBoolEnvVar(envCustomNetworkCfg, false) @@ -1696,7 +1687,6 @@ func dsBackingStorePath() string { func getWarmIPTarget() int { inputStr, found := os.LookupEnv(envWarmIPTarget) - if !found { return noWarmIPTarget } @@ -1712,7 +1702,6 @@ func getWarmIPTarget() int { func getMinimumIPTarget() int { inputStr, found := os.LookupEnv(envMinimumIPTarget) - if !found { return noMinimumIPTarget } @@ -1791,16 +1780,18 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil return ret } -// datastoreTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET, -// accounting for the MINIMUM_IP_TARGET -// With prefix delegation this function determines the number of Prefixes `short` or `over` -func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool) { - if c.warmIPTarget == noWarmIPTarget && c.minimumIPTarget == noMinimumIPTarget { +// datastoreTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET, accounting for the MINIMUM_IP_TARGET. +// With prefix delegation, this function determines the number of Prefixes `short` or `over` +func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats) (short int, over int, enabled bool) { + if !c.warmIPTargetsDefined() { // there is no WARM_IP_TARGET defined and no MINIMUM_IP_TARGET, fallback to use all IP addresses on ENI return 0, 0, false } - stats := c.dataStore.GetIPStats(ipV4AddrFamily) + // Calculating DataStore stats can be expensive, so allow the caller to optionally pass stats it already calculated + if stats == nil { + stats = c.dataStore.GetIPStats(ipV4AddrFamily) + } available := stats.AvailableAddresses() // short is greater than 0 when we have fewer available IPs than the warm IP target @@ -1834,11 +1825,8 @@ func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool) freePrefixes := c.dataStore.GetFreePrefixes() overPrefix := max(min(freePrefixes, stats.TotalPrefixes-prefixNeededForWarmIP), 0) overPrefix = max(min(overPrefix, stats.TotalPrefixes-prefixNeededForMinIP), 0) - log.Debugf("Current warm IP stats : target: %d, short(prefixes): %d, over(prefixes): %d, stats: %s", c.warmIPTarget, shortPrefix, overPrefix, stats) return shortPrefix, overPrefix, true } - log.Debugf("Current warm IP stats : target: %d, short: %d, over: %d, stats: %s", c.warmIPTarget, short, over, stats) - return short, over, true } @@ -2005,7 +1993,7 @@ func (c *IPAMContext) AnnotatePod(podName string, podNamespace string, key strin } func (c *IPAMContext) tryUnassignIPsFromENIs() { - log.Debugf("In tryUnassignIPsFromENIs") + log.Debugf("tryUnassignIPsFromENIs") eniInfos := c.dataStore.GetENIInfos() for eniID := range eniInfos.ENIs { c.tryUnassignIPFromENI(eniID) @@ -2014,7 +2002,6 @@ func (c *IPAMContext) tryUnassignIPsFromENIs() { func (c *IPAMContext) tryUnassignIPFromENI(eniID string) { freeableIPs := c.dataStore.FreeableIPs(eniID) - if len(freeableIPs) == 0 { log.Debugf("No freeable IPs") return @@ -2044,6 +2031,7 @@ func (c *IPAMContext) tryUnassignIPFromENI(eniID string) { } func (c *IPAMContext) tryUnassignPrefixesFromENIs() { + log.Debugf("tryUnassignPrefixesFromENIs") eniInfos := c.dataStore.GetENIInfos() for eniID := range eniInfos.ENIs { c.tryUnassignPrefixFromENI(eniID) @@ -2080,14 +2068,14 @@ func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string) { func (c *IPAMContext) GetENIResourcesToAllocate() int { var resourcesToAllocate int - if !c.enablePrefixDelegation { + if c.enablePrefixDelegation { + resourcesToAllocate = min(c.getPrefixesNeeded(), c.maxPrefixesPerENI) + } else { resourcesToAllocate = c.maxIPsPerENI - short, _, warmTargetDefined := c.datastoreTargetState() + short, _, warmTargetDefined := c.datastoreTargetState(nil) if warmTargetDefined { resourcesToAllocate = min(short, c.maxIPsPerENI) } - } else { - resourcesToAllocate = min(c.getPrefixesNeeded(), c.maxPrefixesPerENI) } return resourcesToAllocate } @@ -2122,44 +2110,48 @@ func (c *IPAMContext) hasRoomForEni() bool { return c.dataStore.GetENIs() < (c.maxENI - c.unmanagedENI - trunkEni) } -func (c *IPAMContext) isDatastorePoolTooLow() bool { - short, _, warmTargetDefined := c.datastoreTargetState() - if warmTargetDefined { - return short > 0 +func (c *IPAMContext) isDatastorePoolTooLow() (bool, *datastore.DataStoreStats) { + stats := c.dataStore.GetIPStats(ipV4AddrFamily) + // If max pods has been reached, pool is not too low + if stats.TotalIPs >= c.maxPods { + return false, stats } - stats := c.dataStore.GetIPStats(ipV4AddrFamily) - available := stats.AvailableAddresses() + short, _, warmTargetDefined := c.datastoreTargetState(stats) + if warmTargetDefined { + return short > 0, stats + } warmTarget := c.warmENITarget totalIPs := c.maxIPsPerENI - if c.enablePrefixDelegation { warmTarget = c.warmPrefixTarget _, maxIpsPerPrefix, _ := datastore.GetPrefixDelegationDefaults() totalIPs = maxIpsPerPrefix } + available := stats.AvailableAddresses() poolTooLow := available < totalIPs*warmTarget || (warmTarget == 0 && available == 0) if poolTooLow { log.Debugf("IP pool is too low: available (%d) < ENI target (%d) * addrsPerENI (%d)", available, warmTarget, totalIPs) c.logPoolStats(stats) } - return poolTooLow + return poolTooLow, stats } -func (c *IPAMContext) isDatastorePoolTooHigh() bool { - _, over, warmTargetDefined := c.datastoreTargetState() +func (c *IPAMContext) isDatastorePoolTooHigh(stats *datastore.DataStoreStats) bool { + // NOTE: IPs may be allocated in chunks (full ENIs of prefixes), so the "too-high" condition does not check max pods. The limit is enforced on the allocation side. + _, over, warmTargetDefined := c.datastoreTargetState(stats) if warmTargetDefined { return over > 0 } - //For the existing ENIs check if we can cleanup prefixes + // For the existing ENIs check if we can cleanup prefixes if c.warmPrefixTargetDefined() { freePrefixes := c.dataStore.GetFreePrefixes() poolTooHigh := freePrefixes > c.warmPrefixTarget if poolTooHigh { - log.Debugf("Prefix pool is high so might be able to deallocate : free prefixes %d and warm prefix target %d", freePrefixes, c.warmPrefixTarget) + log.Debugf("Prefix pool is high so might be able to deallocate - free prefixes: %d, warm prefix target: %d", freePrefixes, c.warmPrefixTarget) } return poolTooHigh } @@ -2208,11 +2200,11 @@ func (c *IPAMContext) getPrefixesNeeded() int { // TODO - post GA we can evaluate to see if these two calls can be merged. // datastoreTargetState already has complex math so adding Prefix target will make it even more complex. - short, _, warmIPTargetDefined := c.datastoreTargetState() + short, _, warmIPTargetsDefined := c.datastoreTargetState(nil) shortPrefixes, warmPrefixTargetDefined := c.datastorePrefixTargetState() // WARM_IP_TARGET takes precendence over WARM_PREFIX_TARGET - if warmIPTargetDefined { + if warmIPTargetsDefined { toAllocate = max(toAllocate, short) } else if warmPrefixTargetDefined { toAllocate = max(toAllocate, shortPrefixes) diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index cd6bee4a70..8c360edd42 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -32,6 +32,7 @@ import ( "github.com/vishvananda/netlink" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -174,11 +175,16 @@ func TestNodeInit(t *testing.T) { m.network.EXPECT().GetExternalServiceCIDRs().Return(nil) m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()) + maxPods, _ := resource.ParseQuantity("500") fakeNode := v1.Node{ TypeMeta: metav1.TypeMeta{Kind: "Node"}, ObjectMeta: metav1.ObjectMeta{Name: myNodeName}, Spec: v1.NodeSpec{}, - Status: v1.NodeStatus{}, + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourcePods: maxPods, + }, + }, } m.k8sClient.Create(ctx, &fakeNode) @@ -258,11 +264,16 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { m.network.EXPECT().GetExternalServiceCIDRs().Return(nil) m.network.EXPECT().UpdateExternalServiceIpRules(gomock.Any(), gomock.Any()) + maxPods, _ := resource.ParseQuantity("500") fakeNode := v1.Node{ TypeMeta: metav1.TypeMeta{Kind: "Node"}, ObjectMeta: metav1.ObjectMeta{Name: myNodeName}, Spec: v1.NodeSpec{}, - Status: v1.NodeStatus{}, + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourcePods: maxPods, + }, + }, } m.k8sClient.Create(ctx, &fakeNode) @@ -499,7 +510,6 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool, unschedulabeNode bool) primaryIP: make(map[string]string), terminating: int32(0), } - mockContext.dataStore = testDatastore() primary := true @@ -775,14 +785,14 @@ func TestDecreaseIPPool(t *testing.T) { m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), gomock.Any()).Times(1) m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), gomock.Any()).Times(1) - short, over, enabled := mockContext.datastoreTargetState() + short, over, enabled := mockContext.datastoreTargetState(nil) assert.Equal(t, 0, short) // there would not be any shortage assert.Equal(t, 1, over) // out of 4 IPs we have 2 IPs assigned, warm IP target is 1, so over is 1 assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1 mockContext.decreaseDatastorePool(10 * time.Second) - short, over, enabled = mockContext.datastoreTargetState() + short, over, enabled = mockContext.datastoreTargetState(nil) assert.Equal(t, 0, short) // there would not be any shortage assert.Equal(t, 0, over) // after the above deallocation this should be zero assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1 @@ -790,7 +800,7 @@ func TestDecreaseIPPool(t *testing.T) { //make another call just to ensure that more deallocations do not happen mockContext.decreaseDatastorePool(10 * time.Second) - short, over, enabled = mockContext.datastoreTargetState() + short, over, enabled = mockContext.datastoreTargetState(nil) assert.Equal(t, 0, short) // there would not be any shortage assert.Equal(t, 0, over) // after the above deallocation this should be zero assert.Equal(t, true, enabled) // there is warm ip target enabled with the value of 1 @@ -1115,14 +1125,13 @@ func TestGetWarmIPTargetState(t *testing.T) { primaryIP: make(map[string]string), terminating: int32(0), } - mockContext.dataStore = testDatastore() - _, _, warmIPTargetDefined := mockContext.datastoreTargetState() + _, _, warmIPTargetDefined := mockContext.datastoreTargetState(nil) assert.False(t, warmIPTargetDefined) mockContext.warmIPTarget = 5 - short, over, warmIPTargetDefined := mockContext.datastoreTargetState() + short, over, warmIPTargetDefined := mockContext.datastoreTargetState(nil) assert.True(t, warmIPTargetDefined) assert.Equal(t, 5, short) assert.Equal(t, 0, over) @@ -1134,7 +1143,7 @@ func TestGetWarmIPTargetState(t *testing.T) { ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", ipv4Addr, false) - short, over, warmIPTargetDefined = mockContext.datastoreTargetState() + short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil) assert.True(t, warmIPTargetDefined) assert.Equal(t, 3, short) assert.Equal(t, 0, over) @@ -1147,13 +1156,13 @@ func TestGetWarmIPTargetState(t *testing.T) { ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.5"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", ipv4Addr, false) - short, over, warmIPTargetDefined = mockContext.datastoreTargetState() + short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil) assert.True(t, warmIPTargetDefined) assert.Equal(t, 0, short) assert.Equal(t, 0, over) } -func TestGetWarmIPTargetStatewithPDenabled(t *testing.T) { +func TestGetWarmIPTargetStateWithPDenabled(t *testing.T) { m := setup(t) defer m.ctrl.Finish() @@ -1167,11 +1176,11 @@ func TestGetWarmIPTargetStatewithPDenabled(t *testing.T) { mockContext.dataStore = testDatastorewithPrefix() - _, _, warmIPTargetDefined := mockContext.datastoreTargetState() + _, _, warmIPTargetDefined := mockContext.datastoreTargetState(nil) assert.False(t, warmIPTargetDefined) mockContext.warmIPTarget = 5 - short, over, warmIPTargetDefined := mockContext.datastoreTargetState() + short, over, warmIPTargetDefined := mockContext.datastoreTargetState(nil) assert.True(t, warmIPTargetDefined) assert.Equal(t, 1, short) assert.Equal(t, 0, over) @@ -1184,7 +1193,7 @@ func TestGetWarmIPTargetStatewithPDenabled(t *testing.T) { _, ipnet, _ = net.ParseCIDR("20.1.1.0/28") _ = mockContext.dataStore.AddIPv4CidrToStore("eni-1", *ipnet, true) - short, over, warmIPTargetDefined = mockContext.datastoreTargetState() + short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil) assert.True(t, warmIPTargetDefined) assert.Equal(t, 0, short) assert.Equal(t, 1, over) @@ -1193,7 +1202,7 @@ func TestGetWarmIPTargetStatewithPDenabled(t *testing.T) { _, ipnet, _ = net.ParseCIDR("20.1.1.0/28") _ = mockContext.dataStore.DelIPv4CidrFromStore("eni-1", *ipnet, true) - short, over, warmIPTargetDefined = mockContext.datastoreTargetState() + short, over, warmIPTargetDefined = mockContext.datastoreTargetState(nil) assert.True(t, warmIPTargetDefined) assert.Equal(t, 0, short) assert.Equal(t, 0, over) @@ -1209,6 +1218,7 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { warmENITarget int warmIPTarget int datastore *datastore.DataStore + maxPods int } tests := []struct { @@ -1216,14 +1226,15 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { fields fields want bool }{ - {"Test new ds, all defaults", fields{14, 4, 1, 0, testDatastore()}, true}, - {"Test new ds, 0 ENIs", fields{14, 4, 0, 0, testDatastore()}, true}, - {"Test new ds, 3 warm IPs", fields{14, 4, 0, 3, testDatastore()}, true}, - {"Test 3 unused IPs, 1 warm", fields{3, 4, 1, 1, datastoreWith3FreeIPs()}, false}, - {"Test 1 used, 1 warm ENI", fields{3, 4, 1, 0, datastoreWith1Pod1()}, true}, - {"Test 1 used, 0 warm ENI", fields{3, 4, 0, 0, datastoreWith1Pod1()}, false}, - {"Test 3 used, 1 warm ENI", fields{3, 4, 1, 0, datastoreWith3Pods()}, true}, - {"Test 3 used, 0 warm ENI", fields{3, 4, 0, 0, datastoreWith3Pods()}, true}, + {"Test new ds, all defaults", fields{14, 4, 1, 0, testDatastore(), 500}, true}, + {"Test new ds, 0 ENIs", fields{14, 4, 0, 0, testDatastore(), 500}, true}, + {"Test new ds, 3 warm IPs", fields{14, 4, 0, 3, testDatastore(), 500}, true}, + {"Test 3 unused IPs, 1 warm", fields{3, 4, 1, 1, datastoreWith3FreeIPs(), 500}, false}, + {"Test 1 used, 1 warm ENI", fields{3, 4, 1, 0, datastoreWith1Pod1(), 500}, true}, + {"Test 1 used, 0 warm ENI", fields{3, 4, 0, 0, datastoreWith1Pod1(), 500}, false}, + {"Test 3 used, 1 warm ENI", fields{3, 4, 1, 0, datastoreWith3Pods(), 500}, true}, + {"Test 3 used, 0 warm ENI", fields{3, 4, 0, 0, datastoreWith3Pods(), 500}, true}, + {"Test max pods exceeded", fields{3, 4, 0, 5, datastoreWith3Pods(), 3}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1237,8 +1248,9 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { warmENITarget: tt.fields.warmENITarget, warmIPTarget: tt.fields.warmIPTarget, enablePrefixDelegation: false, + maxPods: tt.fields.maxPods, } - if got := c.isDatastorePoolTooLow(); got != tt.want { + if got, _ := c.isDatastorePoolTooLow(); got != tt.want { t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want) } }) @@ -1255,6 +1267,7 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) { maxPrefixesPerENI int warmPrefixTarget int datastore *datastore.DataStore + maxPods int } tests := []struct { @@ -1262,13 +1275,14 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) { fields fields want bool }{ - {"Test new ds, all defaults", fields{256, 4, 16, 1, testDatastore()}, true}, - {"Test new ds, 0 ENIs", fields{256, 4, 16, 0, testDatastore()}, true}, - {"Test 3 unused IPs, 1 warm", fields{256, 4, 16, 1, datastoreWithFreeIPsFromPrefix()}, false}, - {"Test 1 used, 1 warm Prefix", fields{256, 4, 16, 1, datastoreWith1Pod1FromPrefix()}, true}, - {"Test 1 used, 0 warm Prefix", fields{256, 4, 16, 0, datastoreWith1Pod1FromPrefix()}, false}, - {"Test 3 used, 1 warm Prefix", fields{256, 4, 16, 1, datastoreWith3PodsFromPrefix()}, true}, - {"Test 3 used, 0 warm Prefix", fields{256, 4, 16, 0, datastoreWith3PodsFromPrefix()}, false}, + {"Test new ds, all defaults", fields{256, 4, 16, 1, testDatastore(), 500}, true}, + {"Test new ds, 0 ENIs", fields{256, 4, 16, 0, testDatastore(), 500}, true}, + {"Test 3 unused IPs, 1 warm", fields{256, 4, 16, 1, datastoreWithFreeIPsFromPrefix(), 500}, false}, + {"Test 1 used, 1 warm Prefix", fields{256, 4, 16, 1, datastoreWith1Pod1FromPrefix(), 500}, true}, + {"Test 1 used, 0 warm Prefix", fields{256, 4, 16, 0, datastoreWith1Pod1FromPrefix(), 500}, false}, + {"Test 3 used, 1 warm Prefix", fields{256, 4, 16, 1, datastoreWith3PodsFromPrefix(), 500}, true}, + {"Test 3 used, 0 warm Prefix", fields{256, 4, 16, 0, datastoreWith3PodsFromPrefix(), 500}, false}, + {"Test max pods exceeded", fields{256, 4, 16, 1, datastoreWith3PodsFromPrefix(), 4}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1282,8 +1296,9 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) { maxENI: tt.fields.maxEni, warmPrefixTarget: tt.fields.warmPrefixTarget, enablePrefixDelegation: true, + maxPods: tt.fields.maxPods, } - if got := c.isDatastorePoolTooLow(); got != tt.want { + if got, _ := c.isDatastorePoolTooLow(); got != tt.want { t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want) } }) diff --git a/pkg/rpcwrapper/mocks/rpcwrapper_mocks.go b/pkg/rpcwrapper/mocks/rpcwrapper_mocks.go index 82b01877c5..08fa3b9f5e 100644 --- a/pkg/rpcwrapper/mocks/rpcwrapper_mocks.go +++ b/pkg/rpcwrapper/mocks/rpcwrapper_mocks.go @@ -26,30 +26,30 @@ import ( grpc "google.golang.org/grpc" ) -// MockRPC is a mock of RPC interface +// MockRPC is a mock of RPC interface. type MockRPC struct { ctrl *gomock.Controller recorder *MockRPCMockRecorder } -// MockRPCMockRecorder is the mock recorder for MockRPC +// MockRPCMockRecorder is the mock recorder for MockRPC. type MockRPCMockRecorder struct { mock *MockRPC } -// NewMockRPC creates a new mock instance +// NewMockRPC creates a new mock instance. func NewMockRPC(ctrl *gomock.Controller) *MockRPC { mock := &MockRPC{ctrl: ctrl} mock.recorder = &MockRPCMockRecorder{mock} return mock } -// EXPECT returns an object that allows the caller to indicate expected use +// EXPECT returns an object that allows the caller to indicate expected use. func (m *MockRPC) EXPECT() *MockRPCMockRecorder { return m.recorder } -// NewCNIBackendClient mocks base method +// NewCNIBackendClient mocks base method. func (m *MockRPC) NewCNIBackendClient(arg0 *grpc.ClientConn) rpc.CNIBackendClient { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NewCNIBackendClient", arg0) @@ -57,7 +57,7 @@ func (m *MockRPC) NewCNIBackendClient(arg0 *grpc.ClientConn) rpc.CNIBackendClien return ret0 } -// NewCNIBackendClient indicates an expected call of NewCNIBackendClient +// NewCNIBackendClient indicates an expected call of NewCNIBackendClient. func (mr *MockRPCMockRecorder) NewCNIBackendClient(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewCNIBackendClient", reflect.TypeOf((*MockRPC)(nil).NewCNIBackendClient), arg0) diff --git a/rpc/mocks/rpc_mocks.go b/rpc/mocks/rpc_mocks.go index 4211551cce..1f8e29b949 100644 --- a/rpc/mocks/rpc_mocks.go +++ b/rpc/mocks/rpc_mocks.go @@ -27,30 +27,30 @@ import ( grpc "google.golang.org/grpc" ) -// MockCNIBackendClient is a mock of CNIBackendClient interface +// MockCNIBackendClient is a mock of CNIBackendClient interface. type MockCNIBackendClient struct { ctrl *gomock.Controller recorder *MockCNIBackendClientMockRecorder } -// MockCNIBackendClientMockRecorder is the mock recorder for MockCNIBackendClient +// MockCNIBackendClientMockRecorder is the mock recorder for MockCNIBackendClient. type MockCNIBackendClientMockRecorder struct { mock *MockCNIBackendClient } -// NewMockCNIBackendClient creates a new mock instance +// NewMockCNIBackendClient creates a new mock instance. func NewMockCNIBackendClient(ctrl *gomock.Controller) *MockCNIBackendClient { mock := &MockCNIBackendClient{ctrl: ctrl} mock.recorder = &MockCNIBackendClientMockRecorder{mock} return mock } -// EXPECT returns an object that allows the caller to indicate expected use +// EXPECT returns an object that allows the caller to indicate expected use. func (m *MockCNIBackendClient) EXPECT() *MockCNIBackendClientMockRecorder { return m.recorder } -// AddNetwork mocks base method +// AddNetwork mocks base method. func (m *MockCNIBackendClient) AddNetwork(arg0 context.Context, arg1 *rpc.AddNetworkRequest, arg2 ...grpc.CallOption) (*rpc.AddNetworkReply, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0, arg1} @@ -63,14 +63,14 @@ func (m *MockCNIBackendClient) AddNetwork(arg0 context.Context, arg1 *rpc.AddNet return ret0, ret1 } -// AddNetwork indicates an expected call of AddNetwork +// AddNetwork indicates an expected call of AddNetwork. func (mr *MockCNIBackendClientMockRecorder) AddNetwork(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddNetwork", reflect.TypeOf((*MockCNIBackendClient)(nil).AddNetwork), varargs...) } -// DelNetwork mocks base method +// DelNetwork mocks base method. func (m *MockCNIBackendClient) DelNetwork(arg0 context.Context, arg1 *rpc.DelNetworkRequest, arg2 ...grpc.CallOption) (*rpc.DelNetworkReply, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0, arg1} @@ -83,7 +83,7 @@ func (m *MockCNIBackendClient) DelNetwork(arg0 context.Context, arg1 *rpc.DelNet return ret0, ret1 } -// DelNetwork indicates an expected call of DelNetwork +// DelNetwork indicates an expected call of DelNetwork. func (mr *MockCNIBackendClientMockRecorder) DelNetwork(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() varargs := append([]interface{}{arg0, arg1}, arg2...) diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index eed86e2226..6f8092d34d 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1,14 +1,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.23.0 -// protoc v3.17.2 +// protoc-gen-go v1.31.0 +// protoc v3.15.8 // source: rpc.proto package rpc import ( context "context" - proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -25,10 +24,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type AddNetworkRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/scripts/generate-cni-yaml.sh b/scripts/generate-cni-yaml.sh index c45f7c50af..be238a5b99 100755 --- a/scripts/generate-cni-yaml.sh +++ b/scripts/generate-cni-yaml.sh @@ -4,7 +4,7 @@ set -euo pipefail SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" PLATFORM=$(uname | tr '[:upper:]' '[:lower:]') -HELM_VERSION="3.13.2" +HELM_VERSION="3.14.0" NAMESPACE="kube-system" MAKEFILEPATH=$SCRIPTPATH/../Makefile diff --git a/scripts/run-static-canary.sh b/scripts/run-static-canary.sh index 59621474fd..e70b23fb9f 100755 --- a/scripts/run-static-canary.sh +++ b/scripts/run-static-canary.sh @@ -29,6 +29,7 @@ function run_ginkgo_test() { --ng-name-label-key="kubernetes.io/os" \ --ng-name-label-val="linux" \ --test-image-registry=$TEST_IMAGE_REGISTRY \ + --publish-cw-metrics=true \ $ENDPOINT_OPTION) } diff --git a/test/agent/Dockerfile b/test/agent/Dockerfile index 82ddfa2daf..4b828713f6 100644 --- a/test/agent/Dockerfile +++ b/test/agent/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/eks-distro-build-tooling/golang:1.21.4-5-gcc-al2 as builder +FROM public.ecr.aws/eks-distro-build-tooling/golang:1.21.6-7-gcc-al2 as builder WORKDIR /workspace ENV GOPROXY direct diff --git a/test/agent/go.mod b/test/agent/go.mod index ebad42c41d..97fa95a625 100644 --- a/test/agent/go.mod +++ b/test/agent/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/coreos/go-iptables v0.7.0 github.com/vishvananda/netlink v1.1.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.16.0 ) require github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect diff --git a/test/agent/go.sum b/test/agent/go.sum index 8c226d5bed..44a0ebee54 100644 --- a/test/agent/go.sum +++ b/test/agent/go.sum @@ -5,5 +5,5 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/test/framework/options.go b/test/framework/options.go index 058bcb381e..d99eda6c2d 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -48,6 +48,7 @@ type Options struct { PublicRouteTableID string NgK8SVersion string TestImageRegistry string + PublishCWMetrics bool } func (options *Options) BindFlags() { @@ -72,6 +73,7 @@ func (options *Options) BindFlags() { flag.StringVar(&options.PublicRouteTableID, "public-route-table-id", "", "Public route table ID (optional, if specified you must specify all of public/private-subnets, public-route-table-id, and availability-zones)") flag.StringVar(&options.NgK8SVersion, "ng-kubernetes-version", "1.25", `Kubernetes version for self-managed node groups (optional, default is "1.25")`) flag.StringVar(&options.TestImageRegistry, "test-image-registry", "617930562442.dkr.ecr.us-west-2.amazonaws.com", `AWS registry where the e2e test images are stored`) + flag.BoolVar(&options.PublishCWMetrics, "publish-cw-metrics", false, "Option to publish cloudwatch metrics from the test.") } func (options *Options) Validate() error { diff --git a/test/framework/resources/aws/services/cloudwatch.go b/test/framework/resources/aws/services/cloudwatch.go index 297baf9904..9680ad1bb4 100644 --- a/test/framework/resources/aws/services/cloudwatch.go +++ b/test/framework/resources/aws/services/cloudwatch.go @@ -21,6 +21,7 @@ import ( type CloudWatch interface { GetMetricStatistics(getMetricStatisticsInput *cloudwatch.GetMetricStatisticsInput) (*cloudwatch.GetMetricStatisticsOutput, error) + PutMetricData(input *cloudwatch.PutMetricDataInput) (*cloudwatch.PutMetricDataOutput, error) } type defaultCloudWatch struct { @@ -36,3 +37,7 @@ func NewCloudWatch(session *session.Session) CloudWatch { func (d *defaultCloudWatch) GetMetricStatistics(getMetricStatisticsInput *cloudwatch.GetMetricStatisticsInput) (*cloudwatch.GetMetricStatisticsOutput, error) { return d.CloudWatchAPI.GetMetricStatistics(getMetricStatisticsInput) } + +func (d *defaultCloudWatch) PutMetricData(input *cloudwatch.PutMetricDataInput) (*cloudwatch.PutMetricDataOutput, error) { + return d.CloudWatchAPI.PutMetricData(input) +} diff --git a/test/framework/resources/aws/services/ec2.go b/test/framework/resources/aws/services/ec2.go index 3a5d3ff1f5..4c7f7a4d00 100644 --- a/test/framework/resources/aws/services/ec2.go +++ b/test/framework/resources/aws/services/ec2.go @@ -47,6 +47,8 @@ type EC2 interface { DeleteKey(keyName string) error DescribeKey(keyName string) (*ec2.DescribeKeyPairsOutput, error) ModifyNetworkInterfaceSecurityGroups(securityGroupIds []*string, networkInterfaceId *string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) + + DescribeAvailabilityZones() (*ec2.DescribeAvailabilityZonesOutput, error) } type defaultEC2 struct { @@ -67,6 +69,11 @@ func (d *defaultEC2) DescribeInstanceType(instanceType string) ([]*ec2.InstanceT return describeInstanceOp.InstanceTypes, nil } +func (d *defaultEC2) DescribeAvailabilityZones() (*ec2.DescribeAvailabilityZonesOutput, error) { + describeAvailabilityZonesInput := &ec2.DescribeAvailabilityZonesInput{} + return d.EC2API.DescribeAvailabilityZones(describeAvailabilityZonesInput) +} + func (d *defaultEC2) ModifyNetworkInterfaceSecurityGroups(securityGroupIds []*string, networkInterfaceId *string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) { return d.EC2API.ModifyNetworkInterfaceAttribute(&ec2.ModifyNetworkInterfaceAttributeInput{ NetworkInterfaceId: networkInterfaceId, diff --git a/test/framework/resources/k8s/resources/daemonset.go b/test/framework/resources/k8s/resources/daemonset.go index e6a9c2eca9..ad41920b4d 100644 --- a/test/framework/resources/k8s/resources/daemonset.go +++ b/test/framework/resources/k8s/resources/daemonset.go @@ -52,7 +52,7 @@ func (d *defaultDaemonSetManager) CreateAndWaitTillDaemonSetIsReady(daemonSet *v } // Allow for the cache to sync - time.Sleep(utils.PollIntervalShort) + time.Sleep(utils.PollIntervalLong) err = d.CheckIfDaemonSetIsReady(daemonSet.Namespace, daemonSet.Name) if err != nil { diff --git a/test/integration/cni/pod_traffic_across_az_test.go b/test/integration/cni/pod_traffic_across_az_test.go index 753744a55f..082d47db7e 100644 --- a/test/integration/cni/pod_traffic_across_az_test.go +++ b/test/integration/cni/pod_traffic_across_az_test.go @@ -4,6 +4,9 @@ import ( "fmt" "strconv" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" @@ -19,6 +22,46 @@ var ( retries = 3 ) +const MetricNamespace = "NetworkingAZConnectivity" + +var _ = Describe("[STATIC_CANARY] AZ Node Presence", FlakeAttempts(retries), func() { + + Context("While testing AZ availability", func() { + It("Ensure nodes are present across AZs and returned by the k8s client.", func() { + nodePresenceInAZ := make(map[string]bool) + + nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + Expect(err).ToNot(HaveOccurred()) + + describeAZOutput, err := f.CloudServices.EC2().DescribeAvailabilityZones() + Expect(err).ToNot(HaveOccurred()) + + for _, az := range describeAZOutput.AvailabilityZones { + nodePresenceInAZ[*az.ZoneName] = false + } + for i := range nodes.Items { + // node label key "topology.kubernetes.io/zone" is well known label populated by cloud controller manager + // guaranteed to be present and represent the AZ name + // Ref https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone + azName := nodes.Items[i].ObjectMeta.Labels["topology.kubernetes.io/zone"] + nodePresenceInAZ[azName] = true + } + + // If a node is not present or reported in an AZ, fail this test for coverage. + var nodeNotInAZ []string + for az, present := range nodePresenceInAZ { + if !present { + nodeNotInAZ = append(nodeNotInAZ, az) + } + } + if len(nodeNotInAZ) > 0 { + Fail(fmt.Sprintf("nodes not detected in AZs: %v", nodeNotInAZ)) + } + + }) + }) +}) + // Tests pod networking across AZs. It similar to pod connectivity test, but launches a daemonset, so that // there is a pod on each node across AZs. It then tests connectivity between pods on different nodes across AZs. var _ = Describe("[STATIC_CANARY] test pod networking", FlakeAttempts(retries), func() { @@ -50,6 +93,9 @@ var _ = Describe("[STATIC_CANARY] test pod networking", FlakeAttempts(retries), // Map of AZ name, string to pod of testDaemonSet azToTestPod map[string]coreV1.Pod + + // Map of AZ name, string to AZ ID for the account. + azToazID map[string]string ) JustBeforeEach(func() { @@ -88,7 +134,7 @@ var _ = Describe("[STATIC_CANARY] test pod networking", FlakeAttempts(retries), Expect(err).ToNot(HaveOccurred()) - azToTestPod = GetAZtoPod(nodes) + azToTestPod, azToazID = GetAZMappings(nodes) }) JustAfterEach(func() { @@ -128,14 +174,31 @@ var _ = Describe("[STATIC_CANARY] test pod networking", FlakeAttempts(retries), }) It("Should allow TCP traffic across AZs.", func() { - CheckConnectivityBetweenPods(azToTestPod, serverPort, testerExpectedStdOut, testerExpectedStdErr, testConnectionCommandFunc) + CheckConnectivityBetweenPods(azToTestPod, azToazID, serverPort, testerExpectedStdOut, testerExpectedStdErr, testConnectionCommandFunc) }) }) }) -func GetAZtoPod(nodes coreV1.NodeList) map[string]coreV1.Pod { +// Function for AZ to Pod mapping and AZ to AZ ID mapping +func GetAZMappings(nodes coreV1.NodeList) (map[string]coreV1.Pod, map[string]string) { // Map of AZ name to Pod from Daemonset running on nodes azToPod := make(map[string]coreV1.Pod) + // Map of AZ name to AZ ID + azToazID := make(map[string]string) + + describeAZOutput, err := f.CloudServices.EC2().DescribeAvailabilityZones() + + // iterate describe AZ output and populate AZ name to AZ ID mapping + for _, az := range describeAZOutput.AvailabilityZones { + azToazID[*az.ZoneName] = *az.ZoneId + } + + if err != nil { + // Don't fail the test if we can't describe AZs. The failure will be caught by the test + // We use describe AZs to get the AZ ID for metrics. + fmt.Println("Error while describing AZs", err) + } + for i := range nodes.Items { // node label key "topology.kubernetes.io/zone" is well known label populated by cloud controller manager // guaranteed to be present and represent the AZ name @@ -149,18 +212,20 @@ func GetAZtoPod(nodes coreV1.NodeList) map[string]coreV1.Pod { if len(interfaceToPodList.PodsOnPrimaryENI) > 0 { azToPod[azName] = interfaceToPodList.PodsOnPrimaryENI[0] } + } - return azToPod + return azToPod, azToazID } -var _ = Describe("[STATIC_CANARY2] API Server Connectivity from AZs", FlakeAttempts(retries), func() { +var _ = Describe("[STATIC_CANARY] API Server Connectivity from AZs", FlakeAttempts(retries), func() { var ( err error testDaemonSet *v1.DaemonSet // Map of AZ name to Pod of testDaemonSet running on nodes - azToPod map[string]coreV1.Pod + azToPod map[string]coreV1.Pod + azToazID map[string]string ) JustBeforeEach(func() { @@ -190,7 +255,8 @@ var _ = Describe("[STATIC_CANARY2] API Server Connectivity from AZs", FlakeAttem nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) Expect(err).ToNot(HaveOccurred()) - azToPod = GetAZtoPod(nodes) + azToPod, azToazID = GetAZMappings(nodes) + }) JustAfterEach(func() { @@ -208,21 +274,22 @@ var _ = Describe("[STATIC_CANARY2] API Server Connectivity from AZs", FlakeAttem APIServerNLBEndpoint := fmt.Sprintf("%s/api", *describeClusterOutput.Cluster.Endpoint) APIServerInternalEndpoint := "https://kubernetes.default.svc/api" - CheckAPIServerConnectivityFromPods(azToPod, APIServerInternalEndpoint) + CheckAPIServerConnectivityFromPods(azToPod, azToazID, APIServerInternalEndpoint) - CheckAPIServerConnectivityFromPods(azToPod, APIServerNLBEndpoint) + CheckAPIServerConnectivityFromPods(azToPod, azToazID, APIServerNLBEndpoint) }) }) }) -func CheckAPIServerConnectivityFromPods(azToPod map[string]coreV1.Pod, api_server_url string) { +func CheckAPIServerConnectivityFromPods(azToPod map[string]coreV1.Pod, azToazId map[string]string, api_server_url string) { // Standard paths for SA token, CA cert and API Server URL token_path := "/var/run/secrets/kubernetes.io/serviceaccount/token" cacert := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + const MetricName = "APIServerConnectivity" for az := range azToPod { - fmt.Printf("Testing API Server %s Connectivity from AZ %s \n", api_server_url, az) + fmt.Printf("Testing API Server %s Connectivity from AZ %s AZID %s \n", api_server_url, az, azToazId[az]) sa_token := []string{"cat", token_path} token_value, _, err := RunCommandOnPod(azToPod[az], sa_token) @@ -240,19 +307,63 @@ func CheckAPIServerConnectivityFromPods(azToPod map[string]coreV1.Pod, api_serve Expect(api_server_stdout).ToNot(BeEmpty()) Expect(api_server_stdout).To(ContainSubstring("APIVersions")) fmt.Printf("API Server %s Connectivity from AZ %s was successful.\n", api_server_url, az) + + if f.Options.PublishCWMetrics { + putmetricData := cloudwatch.PutMetricDataInput{ + Namespace: aws.String(MetricNamespace), + MetricData: []*cloudwatch.MetricDatum{ + { + MetricName: aws.String(MetricName), + Unit: aws.String("Count"), + Value: aws.Float64(1), + Dimensions: []*cloudwatch.Dimension{ + { + Name: aws.String("AZID"), + Value: aws.String(azToazId[az]), + }, + }, + }, + }, + } + + _, err = f.CloudServices.CloudWatch().PutMetricData(&putmetricData) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error while putting metric data for API Server Connectivity from %s", az)) + } } } -func CheckConnectivityBetweenPods(azToPod map[string]coreV1.Pod, port int, testerExpectedStdOut string, testerExpectedStdErr string, getTestCommandFunc func(serverPod coreV1.Pod, port int) []string) { +func CheckConnectivityBetweenPods(azToPod map[string]coreV1.Pod, azToazId map[string]string, port int, testerExpectedStdOut string, testerExpectedStdErr string, getTestCommandFunc func(serverPod coreV1.Pod, port int) []string) { + const MetricName = "InterAZConnectivity" By("checking connection on same node, primary to primary") for az1 := range azToPod { for az2 := range azToPod { if az1 != az2 { - fmt.Printf("Testing Connectivity from Pod IP1 %s (%s) to Pod IP2 %s (%s) \n", - azToPod[az1].Status.PodIP, az1, azToPod[az2].Status.PodIP, az2) + fmt.Printf("Testing Connectivity from Pod IP1 %s (%s, %s) to Pod IP2 %s (%s, %s) \n", + azToPod[az1].Status.PodIP, az1, azToazId[az1], azToPod[az2].Status.PodIP, az2, azToazId[az2]) testConnectivity(azToPod[az1], azToPod[az2], testerExpectedStdOut, testerExpectedStdErr, port, getTestCommandFunc) + + if f.Options.PublishCWMetrics { + putmetricData := cloudwatch.PutMetricDataInput{ + Namespace: aws.String(MetricNamespace), + MetricData: []*cloudwatch.MetricDatum{ + { + MetricName: aws.String(MetricName), + Unit: aws.String("Count"), + Value: aws.Float64(1), + Dimensions: []*cloudwatch.Dimension{ + { + Name: aws.String("AZID"), + Value: aws.String(azToazId[az1]), + }, + }, + }, + }, + } + _, err := f.CloudServices.CloudWatch().PutMetricData(&putmetricData) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error while putting metric data for API Server Connectivity from %s", azToazId[az1])) + } } } } diff --git a/testdata/deploy-130-pods.yaml b/testdata/deploy-130-pods.yaml index 48537f46eb..7d6c86bc14 100644 --- a/testdata/deploy-130-pods.yaml +++ b/testdata/deploy-130-pods.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: hello - image: registry.k8s.io/pause:latest + image: public.ecr.aws/eks-distro/kubernetes/pause:3.9 ports: - name: http containerPort: 80 diff --git a/testdata/deploy-5000-pods.yaml b/testdata/deploy-5000-pods.yaml index 5534fa5965..a5929079aa 100644 --- a/testdata/deploy-5000-pods.yaml +++ b/testdata/deploy-5000-pods.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: hello - image: registry.k8s.io/pause:latest + image: public.ecr.aws/eks-distro/kubernetes/pause:3.9 ports: - name: http containerPort: 80 diff --git a/testdata/deploy-730-pods.yaml b/testdata/deploy-730-pods.yaml index 0d9d1d9a83..74bdcb0791 100644 --- a/testdata/deploy-730-pods.yaml +++ b/testdata/deploy-730-pods.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: hello - image: registry.k8s.io/pause:latest + image: public.ecr.aws/eks-distro/kubernetes/pause:3.9 ports: - name: http containerPort: 80