feat: configure http cache size through upstream setting (#1988)

Co-authored-by: Tushar Mathur <[email protected]>
tailcallhq · May 22, 2024 · b048f7c · b048f7c · github-actions · May 22, 2024
1 parent 677ecad
commit b048f7c
Show file tree

Hide file tree

Showing 71 changed files with 190 additions and 99 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -58,7 +58,7 @@ The below file is a standard `.graphQL` file, with a few additions such as `@ser
 ```graphql
 schema
   @server(port: 8000, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true) {
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42) {
   query: Query
 }
 

diff --git a/benches/http_execute_bench.rs b/benches/http_execute_bench.rs
@@ -8,7 +8,7 @@ pub fn benchmark_http_execute_method(c: &mut Criterion) {
     let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
 
     let mut blueprint = Blueprint::default();
-    blueprint.upstream.http_cache = true; // allow http caching for bench test.
+    blueprint.upstream.http_cache = 42; // allow http caching for bench test.
     let native_http = NativeHttp::init(&blueprint.upstream, &blueprint.telemetry);
     let request_url = String::from("http://jsonplaceholder.typicode.com/users");
 

diff --git a/benches/impl_path_string_for_evaluation_context.rs b/benches/impl_path_string_for_evaluation_context.rs
@@ -57,10 +57,10 @@ impl Http {
 
         let mut client = ClientBuilder::new(builder.build().expect("Failed to build client"));
 
-        if upstream.http_cache {
+        if upstream.http_cache > 0 {
             client = client.with(Cache(HttpCache {
                 mode: CacheMode::Default,
-                manager: HttpCacheManager::default(),
+                manager: HttpCacheManager::new(upstream.http_cache),
                 options: HttpCacheOptions::default(),
             }))
         }

diff --git a/examples/auth.graphql b/examples/auth.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000)
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true)
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42)
   @link(id: "auth-basic", type: Htpasswd, src: ".htpasswd")
   @link(id: "auth-jwt", type: Jwks, src: ".jwks") {
   query: Query

diff --git a/examples/graphql-composition.graphql b/examples/graphql-composition.graphql
@@ -4,7 +4,7 @@
 
 schema
   @server(port: 8001, queryValidation: false, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://localhost:8000/graphql", httpCache: true, batch: {delay: 1}) {
+  @upstream(baseURL: "http://localhost:8000/graphql", httpCache: 42, batch: {delay: 1}) {
   query: Query
 }
 

diff --git a/examples/grpc-reflection.graphql b/examples/grpc-reflection.graphql
@@ -1,7 +1,7 @@
 # for test upstream server see [repo](https://github.com/tailcallhq/tailcall/tree/main/tailcall-upstream-grpc)
 schema
   @server(port: 8000)
-  @upstream(baseURL: "http://localhost:50051", httpCache: true, batch: {delay: 10})
+  @upstream(baseURL: "http://localhost:50051", httpCache: 42, batch: {delay: 10})
   @link(src: "http://localhost:50051", type: Grpc) {
   query: Query
 }

diff --git a/examples/grpc.graphql b/examples/grpc.graphql
@@ -1,7 +1,7 @@
 # for test upstream server see [repo](https://github.com/tailcallhq/rust-grpc)
 schema
   @server(port: 8000)
-  @upstream(baseURL: "http://localhost:50051", httpCache: true, batch: {delay: 10})
+  @upstream(baseURL: "http://localhost:50051", httpCache: 42, batch: {delay: 10})
   @link(id: "news", src: "../tailcall-fixtures/fixtures/protobuf/news.proto", type: Protobuf) {
   query: Query
 }

diff --git a/examples/jsonplaceholder.graphql b/examples/jsonplaceholder.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000, headers: {cors: {allowOrigins: ["*"], allowHeaders: ["*"], allowMethods: [POST, GET, OPTIONS]}})
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true, batch: {delay: 100}) {
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42, batch: {delay: 100}) {
   query: Query
 }
 

diff --git a/examples/jsonplaceholder.json b/examples/jsonplaceholder.json
@@ -6,7 +6,7 @@
   },
   "upstream": {
     "baseURL": "http://jsonplaceholder.typicode.com",
-    "httpCache": true
+    "httpCache": 42
   },
   "schema": {
     "query": "Query"

diff --git a/examples/jsonplaceholder.yml b/examples/jsonplaceholder.yml
@@ -3,7 +3,7 @@ server:
   port: 8000
 upstream:
   baseURL: http://jsonplaceholder.typicode.com
-  httpCache: true
+  httpCache: 42
 schema:
   query: Query
 types:

diff --git a/examples/jsonplaceholder_batch.graphql b/examples/jsonplaceholder_batch.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000)
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true, batch: {delay: 1, maxSize: 1000}) {
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42, batch: {delay: 1, maxSize: 1000}) {
   query: Query
 }
 

diff --git a/examples/jsonplaceholder_batch.json b/examples/jsonplaceholder_batch.json
@@ -4,7 +4,7 @@
   },
   "upstream": {
     "baseURL": "http://jsonplaceholder.typicode.com",
-    "httpCache": true,
+    "httpCache": 42,
     "batch": {
       "maxSize": 1000,
       "delay": 1,

diff --git a/examples/jsonplaceholder_batch.yml b/examples/jsonplaceholder_batch.yml
@@ -2,7 +2,7 @@ server:
   port: 8000
 upstream:
   baseURL: http://jsonplaceholder.typicode.com
-  httpCache: true
+  httpCache: 42
   batch:
     maxSize: 1000
     delay: 1

diff --git a/examples/jsonplaceholder_http_2.graphql b/examples/jsonplaceholder_http_2.graphql
@@ -2,7 +2,7 @@ schema
   @server(port: 3000, queryValidation: false, hostname: "0.0.0.0", version: HTTP2)
   @link(type: Cert, src: "./example.crt")
   @link(type: Key, src: "./example.key")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true) {
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42) {
   query: Query
 }
 

diff --git a/examples/jsonplaceholder_script.graphql b/examples/jsonplaceholder_script.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true)
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42)
   @link(type: Script, src: "scripts/echo.js") {
   query: Query
 }

diff --git a/examples/rest-api.graphql b/examples/rest-api.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true)
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42)
   @link(type: Operation, src: "operations/routes.graphql") {
   query: Query
 }

diff --git a/examples/telemetry-otlp.graphql b/examples/telemetry-otlp.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true)
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42)
   @link(type: Operation, src: "operations/routes.graphql")
   @link(id: "news", src: "../tailcall-fixtures/fixtures/protobuf/news.proto", type: Protobuf)
   @telemetry(

diff --git a/examples/telemetry-prometheus.graphql b/examples/telemetry-prometheus.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true)
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42)
   @telemetry(export: {prometheus: {path: "/metrics"}}) {
   query: Query
 }

diff --git a/examples/telemetry-stdout.graphql b/examples/telemetry-stdout.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true)
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42)
   @telemetry(export: {stdout: {pretty: true}}) {
   query: Query
 }

diff --git a/generated/.tailcallrc.graphql b/generated/.tailcallrc.graphql
@@ -362,11 +362,11 @@ directive @upstream(
   """
   http2Only: Boolean
   """
-  Activating this enables Tailcall's HTTP caching, adhering to the [HTTP Caching RFC](https://tools.ietf.org/html/rfc7234), 
-  to enhance performance by minimizing redundant data fetches. Defaults to `false` 
-  if unspecified.
+  Providing httpCache size enables Tailcall's HTTP caching, adhering to the [HTTP Caching 
+  RFC](https://tools.ietf.org/html/rfc7234), to enhance performance by minimizing redundant 
+  data fetches. Defaults to `0` if unspecified.
   """
-  httpCache: Boolean
+  httpCache: Int
   """
   The time in seconds between each keep-alive message sent to maintain the connection.
   """

diff --git a/generated/.tailcallrc.schema.json b/generated/.tailcallrc.schema.json
@@ -1366,11 +1366,13 @@
           ]
         },
         "httpCache": {
-          "description": "Activating this enables Tailcall's HTTP caching, adhering to the [HTTP Caching RFC](https://tools.ietf.org/html/rfc7234), to enhance performance by minimizing redundant data fetches. Defaults to `false` if unspecified.",
+          "description": "Providing httpCache size enables Tailcall's HTTP caching, adhering to the [HTTP Caching RFC](https://tools.ietf.org/html/rfc7234), to enhance performance by minimizing redundant data fetches. Defaults to `0` if unspecified.",
           "type": [
-            "boolean",
+            "integer",
             "null"
-          ]
+          ],
+          "format": "uint64",
+          "minimum": 0.0
         },
         "keepAliveInterval": {
           "description": "The time in seconds between each keep-alive message sent to maintain the connection.",

diff --git a/src/cli/runtime/http.rs b/src/cli/runtime/http.rs
@@ -112,10 +112,10 @@ impl NativeHttp {
 
         let mut client = ClientBuilder::new(builder.build().expect("Failed to build client"));
 
-        if upstream.http_cache {
+        if upstream.http_cache > 0 {
             client = client.with(Cache(HttpCache {
                 mode: CacheMode::Default,
-                manager: HttpCacheManager::default(),
+                manager: HttpCacheManager::new(upstream.http_cache),
                 options: HttpCacheOptions::default(),
             }))
         }
@@ -190,13 +190,20 @@ mod tests {
     use tokio;
 
     use super::*;
+    use crate::core::http::Response;
 
     fn start_mock_server() -> httpmock::MockServer {
         httpmock::MockServer::start()
     }
 
+    async fn make_request(request_url: &str, native_http: &NativeHttp) -> Response<Bytes> {
+        let request = reqwest::Request::new(Method::GET, request_url.parse().unwrap());
+        let result = native_http.execute(request).await;
+        result.unwrap()
+    }
+
     #[tokio::test]
-    async fn test_native_http_get_request() {
+    async fn test_native_http_get_request_without_cache() {
         let server = start_mock_server();
 
         let header_serv = server.mock(|when, then| {
@@ -208,17 +215,70 @@ mod tests {
         let port = server.port();
         // Build a GET request to the mock server
         let request_url = format!("http://localhost:{}/test", port);
-        let request = reqwest::Request::new(Method::GET, request_url.parse().unwrap());
+        let response = make_request(&request_url, &native_http).await;
 
-        // Execute the request
-        let result = native_http.execute(request).await;
+        // Assert the response is as expected
+        assert_eq!(response.status, reqwest::StatusCode::OK);
+        assert_eq!(response.body, Bytes::from("Hello"));
+        assert!(response.headers.get("x-cache-lookup").is_none());
+
+        let request_url = format!("http://localhost:{}/test", port);
+        let response = make_request(&request_url, &native_http).await;
 
         // Assert the response is as expected
-        assert!(result.is_ok());
-        let response = result.unwrap();
         assert_eq!(response.status, reqwest::StatusCode::OK);
         assert_eq!(response.body, Bytes::from("Hello"));
+        assert!(response.headers.get("x-cache-lookup").is_none());
+
+        header_serv.assert_hits(2);
+    }
+
+    #[tokio::test]
+    async fn test_native_http_get_request_with_cache() {
+        let server = start_mock_server();
+
+        server.mock(|when, then| {
+            when.method(httpmock::Method::GET).path("/test-1");
+            then.status(200).body("Hello");
+        });
+
+        server.mock(|when, then| {
+            when.method(httpmock::Method::GET).path("/test-2");
+            then.status(200).body("Hello");
+        });
+
+        server.mock(|when, then| {
+            when.method(httpmock::Method::GET).path("/test-3");
+            then.status(200).body("Hello");
+        });
+
+        let upstream = Upstream { http_cache: 2, ..Default::default() };
+        let native_http = NativeHttp::init(&upstream, &Default::default());
+        let port = server.port();
+
+        let url1 = format!("http://localhost:{}/test-1", port);
+        let resp = make_request(&url1, &native_http).await;
+        assert_eq!(resp.headers.get("x-cache-lookup").unwrap(), "MISS");
+
+        let resp = make_request(&url1, &native_http).await;
+        assert_eq!(resp.headers.get("x-cache-lookup").unwrap(), "HIT");
+
+        let url2 = format!("http://localhost:{}/test-2", port);
+        let resp = make_request(&url2, &native_http).await;
+        assert_eq!(resp.headers.get("x-cache-lookup").unwrap(), "MISS");
+
+        let resp = make_request(&url2, &native_http).await;
+        assert_eq!(resp.headers.get("x-cache-lookup").unwrap(), "HIT");
+
+        // now cache is full, let's make 3rd request and cache it and evict url1.
+        let url3 = format!("http://localhost:{}/test-3", port);
+        let resp = make_request(&url3, &native_http).await;
+        assert_eq!(resp.headers.get("x-cache-lookup").unwrap(), "MISS");
+
+        let resp = make_request(&url3, &native_http).await;
+        assert_eq!(resp.headers.get("x-cache-lookup").unwrap(), "HIT");
 
-        header_serv.assert();
+        let resp = make_request(&url1, &native_http).await;
+        assert_eq!(resp.headers.get("x-cache-lookup").unwrap(), "MISS");
     }
 }
diff --git a/src/core/blueprint/upstream.rs b/src/core/blueprint/upstream.rs
@@ -24,7 +24,7 @@ pub struct Upstream {
     pub user_agent: String,
     pub allowed_headers: BTreeSet<String>,
     pub base_url: Option<String>,
-    pub http_cache: bool,
+    pub http_cache: u64,
     pub batch: Option<Batch>,
     pub http2_only: bool,
     pub dedupe: bool,
@@ -78,7 +78,7 @@ impl TryFrom<&ConfigModule> for Upstream {
                 user_agent: (config_upstream).get_user_agent(),
                 allowed_headers,
                 base_url,
-                http_cache: (config_upstream).get_enable_http_cache(),
+                http_cache: (config_upstream).get_http_cache_size(),
                 batch,
                 http2_only: (config_upstream).get_http_2_only(),
                 dedupe: (config_upstream).get_dedupe(),

diff --git a/src/core/config/upstream.rs b/src/core/config/upstream.rs
@@ -79,8 +79,8 @@ pub struct Upstream {
     pub connect_timeout: Option<u64>,
 
     #[serde(default, skip_serializing_if = "is_default")]
-    /// Activating this enables Tailcall's HTTP caching, adhering to the [HTTP Caching RFC](https://tools.ietf.org/html/rfc7234), to enhance performance by minimizing redundant data fetches. Defaults to `false` if unspecified.
-    pub http_cache: Option<bool>,
+    /// Providing httpCache size enables Tailcall's HTTP caching, adhering to the [HTTP Caching RFC](https://tools.ietf.org/html/rfc7234), to enhance performance by minimizing redundant data fetches. Defaults to `0` if unspecified.
+    pub http_cache: Option<u64>,
 
     #[setters(strip_option)]
     #[serde(rename = "http2Only", default, skip_serializing_if = "is_default")]
@@ -172,8 +172,8 @@ impl Upstream {
             .clone()
             .unwrap_or("Tailcall/1.0".to_string())
     }
-    pub fn get_enable_http_cache(&self) -> bool {
-        self.http_cache.unwrap_or(false)
+    pub fn get_http_cache_size(&self) -> u64 {
+        self.http_cache.unwrap_or(0)
     }
     pub fn get_allowed_headers(&self) -> BTreeSet<String> {
         self.allowed_headers.clone().unwrap_or_default()

diff --git a/src/core/runtime.rs b/src/core/runtime.rs
@@ -95,10 +95,10 @@ pub mod test {
 
             let mut client = ClientBuilder::new(builder.build().expect("Failed to build client"));
 
-            if upstream.http_cache {
+            if upstream.http_cache > 0 {
                 client = client.with(Cache(HttpCache {
                     mode: CacheMode::Default,
-                    manager: HttpCacheManager::default(),
+                    manager: HttpCacheManager::new(upstream.http_cache),
                     options: HttpCacheOptions::default(),
                 }))
             }

diff --git a/tailcall-fixtures/fixtures/configs/user-posts.graphql b/tailcall-fixtures/fixtures/configs/user-posts.graphql
@@ -1,6 +1,6 @@
 schema
   @server(port: 8000, hostname: "0.0.0.0")
-  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: true) {
+  @upstream(baseURL: "http://jsonplaceholder.typicode.com", httpCache: 42) {
   query: Query
 }
 

diff --git a/tailcall-http-cache/Cargo.toml b/tailcall-http-cache/Cargo.toml
@@ -6,6 +6,9 @@ publish = false
 
 [dependencies]
 http-cache-reqwest = { version = "0.13.0", default-features = false, features = ["manager-moka"] }
+moka = { version = "0.12.7", default-features = false, features = [
+    "future",
+]}
 http-cache-semantics = { version = "1.0.1", default-features = false, features = ["with_serde", "reqwest"]}
 serde = "1.0.202"
 async-trait = "0.1.80"
Thread Stats	Avg	Stdev	Max	+/- Stdev
Latency	6.75ms	3.23ms	120.83ms	75.30%
Req/Sec	3.76k	148.53	4.32k	90.00%