Add configurability for HTTP requests per IO cycle (projectcontour#5827)

An additional mitigation to CVE-2023-44487 available in Envoy 1.27.1. This change allows configuring the http.max_requests_per_io_cycle Envoy runtime setting via Contour configuration to allow administrators of Contour to prevent abusive connections from starving resources from others. The default is left as the existing behavior, that is no limit, so as not to impact existing valid traffic. See the Envoy release notes for more information: https://www.envoyproxy.io/docs/envoy/v1.27.1/version_history/v1.27/v1.27.1 Signed-off-by: Sunjay Bhatia <[email protected]>
sunjayBhatia · Oct 13, 2023 · cda0703 · cda0703
1 parent d9b2a55
commit cda0703
Show file tree

Hide file tree

Showing 20 changed files with 328 additions and 40 deletions.
diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go
@@ -391,6 +391,16 @@ type EnvoyListenerConfig struct {
 	// Single set of options are applied to all listeners.
 	// +optional
 	SocketOptions *SocketOptions `json:"socketOptions,omitempty"`
+
+	// Defines the limit on number of HTTP requests that Envoy will process from a single
+	// connection in a single I/O cycle. Requests over this limit are processed in subsequent
+	// I/O cycles. Can be used as a mitigation for CVE-2023-44487 when abusive traffic is
+	// detected. Configures the http.max_requests_per_io_cycle Envoy runtime setting. The default
+	// value when this is not set is no limit.
+	//
+	// +kubebuilder:validation:Minimum=1
+	// +optional
+	MaxRequestsPerIOCycle *uint32 `json:"maxRequestsPerIOCycle"`
 }
 
 // SocketOptions defines configurable socket options for Envoy listeners.

diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go
diff --git a/changelogs/unreleased/5827-sunjayBhatia-minor.md b/changelogs/unreleased/5827-sunjayBhatia-minor.md
@@ -0,0 +1,16 @@
+## Max HTTP requests per IO cycle is configurable as an additional mitigation for HTTP/2 CVE-2023-44487
+
+Envoy v1.27.1 mitigates CVE-2023-44487 with some default runtime settings, however the `http.max_requests_per_io_cycle` does not have a default value.
+This change allows configuring this runtime setting via Contour configuration to allow administrators of Contour to prevent abusive connections from starving resources from other valid connections.
+The default is left as the existing behavior (no limit) so as not to impact existing valid traffic.
+
+The Contour ConfigMap can be modified similar to the following (and Contour restarted) to set this value:
+
+```
+listener:
+  max-requests-per-io-cycle: 10
+```
+
+(Note this can be used in addition to the existing Listener configuration field `listener.max-requests-per-connection` which is used primarily for HTTP/1.1 connections and is an approximate limit for HTTP/2)
+
+See the [Envoy release notes](https://www.envoyproxy.io/docs/envoy/v1.27.1/version_history/v1.27/v1.27.1) for more details.
diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go
@@ -458,7 +458,9 @@ func (s *Server) doServe() error {
 		&xdscache_v3.RouteCache{},
 		&xdscache_v3.ClusterCache{},
 		endpointHandler,
-		&xdscache_v3.RuntimeCache{},
+		xdscache_v3.NewRuntimeCache(xdscache_v3.ConfigurableRuntimeSettings{
+			MaxRequestsPerIOCycle: contourConfiguration.Envoy.Listener.MaxRequestsPerIOCycle,
+		}),
 	}
 
 	// snapshotHandler is used to produce new snapshots when the internal state changes for any xDS resource.

diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go
@@ -528,6 +528,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha
 				ConnectionBalancer:            ctx.Config.Listener.ConnectionBalancer,
 				PerConnectionBufferLimitBytes: ctx.Config.Listener.PerConnectionBufferLimitBytes,
 				MaxRequestsPerConnection:      ctx.Config.Listener.MaxRequestsPerConnection,
+				MaxRequestsPerIOCycle:         ctx.Config.Listener.MaxRequestsPerIOCycle,
 				TLS: &contour_api_v1alpha1.EnvoyTLS{
 					MinimumProtocolVersion: ctx.Config.TLS.MinimumProtocolVersion,
 					MaximumProtocolVersion: ctx.Config.TLS.MaximumProtocolVersion,

diff --git a/cmd/contour/servecontext_test.go b/cmd/contour/servecontext_test.go
@@ -874,6 +874,16 @@ func TestConvertServeContext(t *testing.T) {
 				return cfg
 			},
 		},
+		"envoy listener settings": {
+			getServeContext: func(ctx *serveContext) *serveContext {
+				ctx.Config.Listener.MaxRequestsPerIOCycle = ref.To(uint32(10))
+				return ctx
+			},
+			getContourConfiguration: func(cfg contour_api_v1alpha1.ContourConfigurationSpec) contour_api_v1alpha1.ContourConfigurationSpec {
+				cfg.Envoy.Listener.MaxRequestsPerIOCycle = ref.To(uint32(10))
+				return cfg
+			},
+		},
 	}
 
 	for name, tc := range cases {

diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml
@@ -203,6 +203,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3634,6 +3645,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml
@@ -422,6 +422,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3853,6 +3864,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml
@@ -217,6 +217,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3648,6 +3659,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml
@@ -428,6 +428,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3859,6 +3870,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml
@@ -422,6 +422,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3853,6 +3864,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

diff --git a/internal/envoy/v3/runtime.go b/internal/envoy/v3/runtime.go
@@ -24,20 +24,24 @@ const (
 	maxRegexProgramSizeWarn  = 1000
 )
 
-func RuntimeLayers() []*envoy_service_runtime_v3.Runtime {
+func RuntimeLayers(configurableRuntimeFields map[string]*structpb.Value) []*envoy_service_runtime_v3.Runtime {
+	baseLayer := baseRuntimeLayer()
+	for k, v := range configurableRuntimeFields {
+		baseLayer.Fields[k] = v
+	}
 	return []*envoy_service_runtime_v3.Runtime{
 		{
 			Name:  DynamicRuntimeLayerName,
-			Layer: baseRuntimeLayer(),
+			Layer: baseLayer,
 		},
 	}
 }
 
 func baseRuntimeLayer() *structpb.Struct {
 	return &structpb.Struct{
 		Fields: map[string]*structpb.Value{
-			"re2.max_program_size.error_level": {Kind: &structpb.Value_NumberValue{NumberValue: maxRegexProgramSizeError}},
-			"re2.max_program_size.warn_level":  {Kind: &structpb.Value_NumberValue{NumberValue: maxRegexProgramSizeWarn}},
+			"re2.max_program_size.error_level": structpb.NewNumberValue(maxRegexProgramSizeError),
+			"re2.max_program_size.warn_level":  structpb.NewNumberValue(maxRegexProgramSizeWarn),
 		},
 	}
 }
diff --git a/internal/envoy/v3/runtime_test.go b/internal/envoy/v3/runtime_test.go
@@ -22,15 +22,38 @@ import (
 )
 
 func TestRuntimeLayers(t *testing.T) {
-	require.Equal(t, []*envoy_service_runtime_v3.Runtime{
-		{
-			Name: "dynamic",
-			Layer: &structpb.Struct{
-				Fields: map[string]*structpb.Value{
-					"re2.max_program_size.error_level": {Kind: &structpb.Value_NumberValue{NumberValue: 1 << 20}},
-					"re2.max_program_size.warn_level":  {Kind: &structpb.Value_NumberValue{NumberValue: 1000}},
-				},
+	testCases := map[string]struct {
+		configurableFields map[string]*structpb.Value
+	}{
+		"nil configurable fields": {},
+		"empty configurable fields": {
+			configurableFields: map[string]*structpb.Value{},
+		},
+		"some configurable fields": {
+			configurableFields: map[string]*structpb.Value{
+				"some.value1": structpb.NewBoolValue(true),
+				"some.value2": structpb.NewNumberValue(1000),
 			},
 		},
-	}, RuntimeLayers())
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			expectedFields := map[string]*structpb.Value{
+				"re2.max_program_size.error_level": structpb.NewNumberValue(1 << 20),
+				"re2.max_program_size.warn_level":  structpb.NewNumberValue(1000),
+			}
+			for k, v := range tc.configurableFields {
+				expectedFields[k] = v
+			}
+			layers := RuntimeLayers(tc.configurableFields)
+			require.Equal(t, []*envoy_service_runtime_v3.Runtime{
+				{
+					Name: "dynamic",
+					Layer: &structpb.Struct{
+						Fields: expectedFields,
+					},
+				},
+			}, layers)
+		})
+	}
 }
diff --git a/internal/xdscache/v3/runtime.go b/internal/xdscache/v3/runtime.go
@@ -20,30 +20,46 @@ import (
 	envoy_v3 "github.com/projectcontour/contour/internal/envoy/v3"
 	"github.com/projectcontour/contour/internal/protobuf"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
+type ConfigurableRuntimeSettings struct {
+	MaxRequestsPerIOCycle *uint32
+}
+
 // RuntimeCache manages the contents of the gRPC RTDS cache.
-type RuntimeCache struct {
+type runtimeCache struct {
 	contour.Cond
+	runtimeKV map[string]*structpb.Value
+}
+
+// NewRuntimeCache builds a RuntimeCache with the provided runtime
+// settings that will be set in the runtime layer configured by Contour.
+func NewRuntimeCache(runtimeSettings ConfigurableRuntimeSettings) *runtimeCache {
+	runtimeKV := make(map[string]*structpb.Value)
+	if runtimeSettings.MaxRequestsPerIOCycle != nil && *runtimeSettings.MaxRequestsPerIOCycle > 0 {
+		runtimeKV["http.max_requests_per_io_cycle"] = structpb.NewNumberValue(float64(*runtimeSettings.MaxRequestsPerIOCycle))
+	}
+	return &runtimeCache{runtimeKV: runtimeKV}
 }
 
 // Contents returns all Runtime layers.
-func (c *RuntimeCache) Contents() []proto.Message {
-	return protobuf.AsMessages(envoy_v3.RuntimeLayers())
+func (c *runtimeCache) Contents() []proto.Message {
+	return protobuf.AsMessages(envoy_v3.RuntimeLayers(c.runtimeKV))
 }
 
 // Query returns only the "dynamic" layer if requested, otherwise empty.
-func (c *RuntimeCache) Query(names []string) []proto.Message {
+func (c *runtimeCache) Query(names []string) []proto.Message {
 	for _, name := range names {
 		if name == envoy_v3.DynamicRuntimeLayerName {
-			return protobuf.AsMessages(envoy_v3.RuntimeLayers())
+			return protobuf.AsMessages(envoy_v3.RuntimeLayers(c.runtimeKV))
 		}
 	}
 	return []proto.Message{}
 }
 
-func (*RuntimeCache) TypeURL() string { return resource.RuntimeType }
+func (*runtimeCache) TypeURL() string { return resource.RuntimeType }
 
-func (c *RuntimeCache) OnChange(root *dag.DAG) {
+func (c *runtimeCache) OnChange(root *dag.DAG) {
 	// DAG changes do not affect runtime layers at the moment.
 }