rpc: add option to whitelist ips in rate limiting (#3701)

This PR adds two new CLI options to disable rate limiting for certain ip
addresses and whether to trust "proxy header".
After going back in forth I decided to use ip addr instead host because
we don't want rely on the host header which can be spoofed but another
solution is to resolve the ip addr from the socket to host name.

Example:

```bash
$ polkadot --rpc-rate-limit 10 --rpc-rate-limit-whitelisted-ips 127.0.0.1/8 --rpc-rate-limit-trust-proxy-headers
```

The ip addr is read from the HTTP proxy headers `Forwarded`,
`X-Forwarded-For` `X-Real-IP` if `--rpc-rate-limit-trust-proxy-headers`
is enabled if that is not enabled or the headers are not found then the
ip address is read from the socket.

//cc @BulatSaif can you test this and give some feedback on it?

cumulus/test/service/src/lib.rs

@@ -824,6 +824,8 @@ pub fn node_config(
 		rpc_message_buffer_capacity: Default::default(),
 		rpc_batch_config: RpcBatchRequestConfig::Unlimited,
 		rpc_rate_limit: None,
+		rpc_rate_limit_whitelisted_ips: Default::default(),
+		rpc_rate_limit_trust_proxy_headers: Default::default(),
 		prometheus_config: None,
 		telemetry_endpoints: None,
 		default_heap_pages: None,

polkadot/node/test/service/src/lib.rs

@@ -216,6 +216,8 @@ pub fn node_config(
 		rpc_message_buffer_capacity: Default::default(),
 		rpc_batch_config: RpcBatchRequestConfig::Unlimited,
 		rpc_rate_limit: None,
+		rpc_rate_limit_whitelisted_ips: Default::default(),
+		rpc_rate_limit_trust_proxy_headers: Default::default(),
 		prometheus_config: None,
 		telemetry_endpoints: None,
 		default_heap_pages: None,

prdoc/pr_3701.prdoc

@@ -0,0 +1,11 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: add option to whitelist peers in rpc rate limiting
+
+doc:
+  - audience: Node Operator
+    description: |
+      This PR adds two new CLI options to disable rate limiting for certain ip addresses and whether to trust "proxy headers".
+
+crates: [ ]

substrate/bin/node/cli/benches/block_production.rs

@@ -86,6 +86,8 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		rpc_message_buffer_capacity: Default::default(),
 		rpc_batch_config: RpcBatchRequestConfig::Unlimited,
 		rpc_rate_limit: None,
+		rpc_rate_limit_whitelisted_ips: Default::default(),
+		rpc_rate_limit_trust_proxy_headers: Default::default(),
 		prometheus_config: None,
 		telemetry_endpoints: None,
 		default_heap_pages: None,

substrate/bin/node/cli/benches/transaction_pool.rs

@@ -82,6 +82,8 @@ fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase {
 		rpc_message_buffer_capacity: Default::default(),
 		rpc_batch_config: RpcBatchRequestConfig::Unlimited,
 		rpc_rate_limit: None,
+		rpc_rate_limit_whitelisted_ips: Default::default(),
+		rpc_rate_limit_trust_proxy_headers: Default::default(),
 		prometheus_config: None,
 		telemetry_endpoints: None,
 		default_heap_pages: None,

substrate/client/cli/src/commands/run_cmd.rs

@@ -30,7 +30,9 @@ use crate::{
 use clap::Parser;
 use regex::Regex;
 use sc_service::{
-	config::{BasePath, PrometheusConfig, RpcBatchRequestConfig, TransactionPoolOptions},
+	config::{
+		BasePath, IpNetwork, PrometheusConfig, RpcBatchRequestConfig, TransactionPoolOptions,
+	},
 	ChainSpec, Role,
 };
 use sc_telemetry::TelemetryEndpoints;
@@ -94,6 +96,22 @@ pub struct RunCmd {
 	#[arg(long)]
 	pub rpc_rate_limit: Option<NonZeroU32>,
 
+	/// Disable RPC rate limiting for certain ip addresses.
+	///
+	/// Each IP address must be in CIDR notation such as `1.2.3.4/24`.
+	#[arg(long, num_args = 1..)]
+	pub rpc_rate_limit_whitelisted_ips: Vec<IpNetwork>,
+
+	/// Trust proxy headers for disable rate limiting.
+	///
+	/// By default the rpc server will not trust headers such `X-Real-IP`, `X-Forwarded-For` and
+	/// `Forwarded` and this option will make the rpc server to trust these headers.
+	///
+	/// For instance this may be secure if the rpc server is behind a reverse proxy and that the
+	/// proxy always sets these headers.
+	#[arg(long)]
+	pub rpc_rate_limit_trust_proxy_headers: bool,
+
 	/// Set the maximum RPC request payload size for both HTTP and WS in megabytes.
 	#[arg(long, default_value_t = RPC_DEFAULT_MAX_REQUEST_SIZE_MB)]
 	pub rpc_max_request_size: u32,
@@ -439,6 +457,14 @@ impl CliConfiguration for RunCmd {
 		Ok(self.rpc_rate_limit)
 	}
 
+	fn rpc_rate_limit_whitelisted_ips(&self) -> Result<Vec<IpNetwork>> {
+		Ok(self.rpc_rate_limit_whitelisted_ips.clone())
+	}
+
+	fn rpc_rate_limit_trust_proxy_headers(&self) -> Result<bool> {
+		Ok(self.rpc_rate_limit_trust_proxy_headers)
+	}
+
 	fn transaction_pool(&self, is_dev: bool) -> Result<TransactionPoolOptions> {
 		Ok(self.pool_config.transaction_pool(is_dev))
 	}

substrate/client/cli/src/config.rs

@@ -26,7 +26,7 @@ use log::warn;
 use names::{Generator, Name};
 use sc_service::{
 	config::{
-		BasePath, Configuration, DatabaseSource, KeystoreConfig, NetworkConfiguration,
+		BasePath, Configuration, DatabaseSource, IpNetwork, KeystoreConfig, NetworkConfiguration,
 		NodeKeyConfig, OffchainWorkerConfig, OutputFormat, PrometheusConfig, PruningMode, Role,
 		RpcBatchRequestConfig, RpcMethods, TelemetryEndpoints, TransactionPoolOptions,
 		WasmExecutionMethod,
@@ -349,6 +349,16 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		Ok(None)
 	}
 
+	/// RPC rate limit whitelisted ip addresses.
+	fn rpc_rate_limit_whitelisted_ips(&self) -> Result<Vec<IpNetwork>> {
+		Ok(vec![])
+	}
+
+	/// RPC rate limit trust proxy headers.
+	fn rpc_rate_limit_trust_proxy_headers(&self) -> Result<bool> {
+		Ok(false)
+	}
+
 	/// Get the prometheus configuration (`None` if disabled)
 	///
 	/// By default this is `None`.
@@ -523,6 +533,8 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			rpc_message_buffer_capacity: self.rpc_buffer_capacity_per_connection()?,
 			rpc_batch_config: self.rpc_batch_config()?,
 			rpc_rate_limit: self.rpc_rate_limit()?,
+			rpc_rate_limit_whitelisted_ips: self.rpc_rate_limit_whitelisted_ips()?,
+			rpc_rate_limit_trust_proxy_headers: self.rpc_rate_limit_trust_proxy_headers()?,
 			prometheus_config: self
 				.prometheus_config(DCV::prometheus_listen_port(), &chain_spec)?,
 			telemetry_endpoints,

substrate/client/cli/src/runner.rs

@@ -273,6 +273,8 @@ mod tests {
 				rpc_port: 9944,
 				rpc_batch_config: sc_service::config::RpcBatchRequestConfig::Unlimited,
 				rpc_rate_limit: None,
+				rpc_rate_limit_whitelisted_ips: Default::default(),
+				rpc_rate_limit_trust_proxy_headers: Default::default(),
 				prometheus_config: None,
 				telemetry_endpoints: None,
 				default_heap_pages: None,

substrate/client/rpc-servers/Cargo.toml

@@ -16,14 +16,16 @@ workspace = true
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+forwarded-header-value = "0.1.1"
+futures = "0.3.30"
+governor = "0.6.0"
+http = "0.2.8"
+hyper = "0.14.27"
+ip_network = "0.4.1"
 jsonrpsee = { version = "0.22", features = ["server"] }
 log = { workspace = true, default-features = true }
+prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" }
 serde_json = { workspace = true, default-features = true }
 tokio = { version = "1.22.0", features = ["parking_lot"] }
-prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" }
-tower-http = { version = "0.4.0", features = ["cors"] }
 tower = { version = "0.4.13", features = ["util"] }
-http = "0.2.8"
-hyper = "0.14.27"
-futures = "0.3.30"
-governor = "0.6.0"
+tower-http = { version = "0.4.0", features = ["cors"] }

substrate/client/rpc-servers/src/lib.rs

@@ -21,27 +21,28 @@
 #![warn(missing_docs)]
 
 pub mod middleware;
+pub mod utils;
 
 use std::{
 	convert::Infallible, error::Error as StdError, net::SocketAddr, num::NonZeroU32, time::Duration,
 };
 
-use http::header::HeaderValue;
 use hyper::{
 	server::conn::AddrStream,
 	service::{make_service_fn, service_fn},
 };
 use jsonrpsee::{
 	server::{
-		middleware::http::{HostFilterLayer, ProxyGetRequestLayer},
-		stop_channel, ws, PingConfig, StopHandle, TowerServiceBuilder,
+		middleware::http::ProxyGetRequestLayer, stop_channel, ws, PingConfig, StopHandle,
+		TowerServiceBuilder,
 	},
 	Methods, RpcModule,
 };
 use tokio::net::TcpListener;
 use tower::Service;
-use tower_http::cors::{AllowOrigin, CorsLayer};
+use utils::{build_rpc_api, format_cors, get_proxy_ip, host_filtering, try_into_cors};
 
+pub use ip_network::IpNetwork;
 pub use jsonrpsee::{
 	core::{
 		id_providers::{RandomIntegerIdProvider, RandomStringIdProvider},
@@ -85,6 +86,10 @@ pub struct Config<'a, M: Send + Sync + 'static> {
 	pub batch_config: BatchRequestConfig,
 	/// Rate limit calls per minute.
 	pub rate_limit: Option<NonZeroU32>,
+	/// Disable rate limit for certain ips.
+	pub rate_limit_whitelisted_ips: Vec<IpNetwork>,
+	/// Trust proxy headers for rate limiting.
+	pub rate_limit_trust_proxy_headers: bool,
 }
 
 #[derive(Debug, Clone)]
@@ -117,11 +122,13 @@ where
 		tokio_handle,
 		rpc_api,
 		rate_limit,
+		rate_limit_whitelisted_ips,
+		rate_limit_trust_proxy_headers,
 	} = config;
 
 	let std_listener = TcpListener::bind(addrs.as_slice()).await?.into_std()?;
 	let local_addr = std_listener.local_addr().ok();
-	let host_filter = hosts_filtering(cors.is_some(), local_addr);
+	let host_filter = host_filtering(cors.is_some(), local_addr);
 
 	let http_middleware = tower::ServiceBuilder::new()
 		.option_layer(host_filter)
@@ -160,20 +167,39 @@ where
 		stop_handle: stop_handle.clone(),
 	};
 
-	let make_service = make_service_fn(move |_conn: &AddrStream| {
+	let make_service = make_service_fn(move |addr: &AddrStream| {
 		let cfg = cfg.clone();
+		let rate_limit_whitelisted_ips = rate_limit_whitelisted_ips.clone();
+		let ip = addr.remote_addr().ip();
 
 		async move {
 			let cfg = cfg.clone();
+			let rate_limit_whitelisted_ips = rate_limit_whitelisted_ips.clone();
 
 			Ok::<_, Infallible>(service_fn(move |req| {
+				let proxy_ip =
+					if rate_limit_trust_proxy_headers { get_proxy_ip(&req) } else { None };
+
+				let rate_limit_cfg = if rate_limit_whitelisted_ips
+					.iter()
+					.any(|ips| ips.contains(proxy_ip.unwrap_or(ip)))
+				{
+					log::debug!(target: "rpc", "ip={ip}, proxy_ip={:?} is trusted, disabling rate-limit", proxy_ip);
+					None
+				} else {
+					if !rate_limit_whitelisted_ips.is_empty() {
+						log::debug!(target: "rpc", "ip={ip}, proxy_ip={:?} is not trusted, rate-limit enabled", proxy_ip);
+					}
+					rate_limit
+				};
+
 				let PerConnection { service_builder, metrics, tokio_handle, stop_handle, methods } =
 					cfg.clone();
 
 				let is_websocket = ws::is_upgrade_request(&req);
 				let transport_label = if is_websocket { "ws" } else { "http" };
 
-				let middleware_layer = match (metrics, rate_limit) {
+				let middleware_layer = match (metrics, rate_limit_cfg) {
 					(None, None) => None,
 					(Some(metrics), None) => Some(
 						MiddlewareLayer::new().with_metrics(Metrics::new(metrics, transport_label)),
@@ -227,57 +253,3 @@ where
 
 	Ok(server_handle)
 }
-
-fn hosts_filtering(enabled: bool, addr: Option<SocketAddr>) -> Option<HostFilterLayer> {
-	// If the local_addr failed, fallback to wildcard.
-	let port = addr.map_or("*".to_string(), |p| p.port().to_string());
-
-	if enabled {
-		// NOTE: The listening addresses are whitelisted by default.
-		let hosts =
-			[format!("localhost:{port}"), format!("127.0.0.1:{port}"), format!("[::1]:{port}")];
-		Some(HostFilterLayer::new(hosts).expect("Valid hosts; qed"))
-	} else {
-		None
-	}
-}
-
-fn build_rpc_api<M: Send + Sync + 'static>(mut rpc_api: RpcModule<M>) -> RpcModule<M> {
-	let mut available_methods = rpc_api.method_names().collect::<Vec<_>>();
-	// The "rpc_methods" is defined below and we want it to be part of the reported methods.
-	available_methods.push("rpc_methods");
-	available_methods.sort();
-
-	rpc_api
-		.register_method("rpc_methods", move |_, _| {
-			serde_json::json!({
-				"methods": available_methods,
-			})
-		})
-		.expect("infallible all other methods have their own address space; qed");
-
-	rpc_api
-}
-
-fn try_into_cors(
-	maybe_cors: Option<&Vec<String>>,
-) -> Result<CorsLayer, Box<dyn StdError + Send + Sync>> {
-	if let Some(cors) = maybe_cors {
-		let mut list = Vec::new();
-		for origin in cors {
-			list.push(HeaderValue::from_str(origin)?);
-		}
-		Ok(CorsLayer::new().allow_origin(AllowOrigin::list(list)))
-	} else {
-		// allow all cors
-		Ok(CorsLayer::permissive())
-	}
-}
-
-fn format_cors(maybe_cors: Option<&Vec<String>>) -> String {
-	if let Some(cors) = maybe_cors {
-		format!("{:?}", cors)
-	} else {
-		format!("{:?}", ["*"])
-	}
-}