feat: implement upstream base url generator. (#2124)

Co-authored-by: Ranjit Mahadik <[email protected]>
Co-authored-by: Tushar Mathur <[email protected]>

src/core/config/transformer/consolidate_url/consolidate_url.rs

@@ -0,0 +1,143 @@
+use std::collections::HashSet;
+
+use super::max_value_map::MaxValueMap;
+use crate::core::config::transformer::Transform;
+use crate::core::config::Config;
+use crate::core::valid::Valid;
+
+struct UrlTypeMapping {
+    /// maintains the url to it's frequency mapping.
+    url_to_frequency_map: MaxValueMap<String, u32>,
+    /// maintains the types that contains the base_url in it's fields.
+    visited_type_set: HashSet<String>,
+}
+
+impl UrlTypeMapping {
+    fn new() -> Self {
+        Self {
+            url_to_frequency_map: Default::default(),
+            visited_type_set: Default::default(),
+        }
+    }
+
+    /// Populates the URL type mapping based on the given configuration.
+    fn populate_url_frequency_map(&mut self, config: &Config) {
+        for (type_name, type_) in config.types.iter() {
+            for field_ in type_.fields.values() {
+                if let Some(http_directive) = &field_.http {
+                    if let Some(base_url) = &http_directive.base_url {
+                        self.url_to_frequency_map.increment(base_url.to_owned(), 1);
+                        self.visited_type_set.insert(type_name.to_owned());
+                    }
+                }
+            }
+        }
+    }
+
+    /// Finds the most common URL that meets the threshold.
+    fn find_common_url(&self, threshold: f32) -> Option<String> {
+        if let Some((common_url, frequency)) = self.url_to_frequency_map.get_max_pair() {
+            if *frequency >= (self.url_to_frequency_map.len() as f32 * threshold).ceil() as u32 {
+                return Some(common_url.to_owned());
+            }
+        }
+        None
+    }
+}
+
+pub struct ConsolidateURL {
+    threshold: f32,
+}
+
+impl ConsolidateURL {
+    pub fn new(threshold: f32) -> Self {
+        let mut validated_thresh = threshold;
+        if !(0.0..=1.0).contains(&threshold) {
+            validated_thresh = 1.0;
+            tracing::warn!(
+                "Invalid threshold value ({:.2}), reverting to default threshold ({:.2}). allowed range is [0.0 - 1.0] inclusive",
+                threshold,
+                validated_thresh
+            );
+        }
+        Self { threshold: validated_thresh }
+    }
+
+    fn generate_base_url(&self, mut config: Config) -> Config {
+        let mut url_type_mapping = UrlTypeMapping::new();
+        url_type_mapping.populate_url_frequency_map(&config);
+
+        if let Some(common_url) = url_type_mapping.find_common_url(self.threshold) {
+            config.upstream.base_url = Some(common_url.to_owned());
+
+            for type_name in url_type_mapping.visited_type_set {
+                if let Some(type_) = config.types.get_mut(&type_name) {
+                    for field_ in type_.fields.values_mut() {
+                        if let Some(htto_directive) = &mut field_.http {
+                            if let Some(base_url) = &htto_directive.base_url {
+                                if *base_url == common_url {
+                                    htto_directive.base_url = None;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        } else {
+            tracing::warn!(
+                "Threshold matching base url not found, transformation cannot be performed."
+            );
+        }
+
+        config
+    }
+}
+
+impl Transform for ConsolidateURL {
+    fn transform(&self, config: Config) -> Valid<Config, String> {
+        let config = self.generate_base_url(config);
+        Valid::succeed(config)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::fs;
+
+    use tailcall_fixtures::configs;
+
+    use super::*;
+    use crate::core::config::transformer::Transform;
+    use crate::core::config::Config;
+    use crate::core::valid::Validator;
+
+    fn read_fixture(path: &str) -> String {
+        fs::read_to_string(path).unwrap()
+    }
+
+    #[test]
+    fn should_generate_correct_upstream_when_multiple_base_urls_present() {
+        let config = Config::from_sdl(read_fixture(configs::MULTI_URL_CONFIG).as_str())
+            .to_result()
+            .unwrap();
+
+        let transformed_config = ConsolidateURL::new(0.5)
+            .transform(config)
+            .to_result()
+            .unwrap();
+        insta::assert_snapshot!(transformed_config.to_sdl());
+    }
+
+    #[test]
+    fn should_not_generate_upstream_when_threshold_is_not_matched() {
+        let config = Config::from_sdl(read_fixture(configs::MULTI_URL_CONFIG).as_str())
+            .to_result()
+            .unwrap();
+
+        let transformed_config = ConsolidateURL::new(0.9)
+            .transform(config)
+            .to_result()
+            .unwrap();
+        insta::assert_snapshot!(transformed_config.to_sdl());
+    }
+}

src/core/config/transformer/consolidate_url/max_value_map.rs

@@ -0,0 +1,90 @@
+use std::collections::HashMap;
+use std::hash::Hash;
+
+/// A data structure that holds K and V, and allows query the max valued key.
+pub struct MaxValueMap<K, V> {
+    map: HashMap<K, V>,
+    max_valued_key: Option<K>,
+}
+
+impl<K, V> Default for MaxValueMap<K, V>
+where
+    K: Eq + Hash + Clone,
+    V: PartialOrd + Clone + std::ops::AddAssign,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<K, V> MaxValueMap<K, V>
+where
+    K: Eq + Hash + Clone,
+    V: PartialOrd + Clone + std::ops::AddAssign,
+{
+    pub fn new() -> Self {
+        MaxValueMap { map: HashMap::new(), max_valued_key: None }
+    }
+
+    pub fn insert(&mut self, key: K, value: V) {
+        if let Some((_, max_value)) = self.get_max_pair() {
+            if *max_value < value {
+                self.max_valued_key = Some(key.to_owned());
+            }
+        } else {
+            self.max_valued_key = Some(key.to_owned());
+        }
+        self.map.insert(key, value);
+    }
+
+    pub fn increment(&mut self, key: K, increment_by: V)
+    where
+        V: Clone + std::ops::Add<Output = V>,
+    {
+        if let Some(value) = self.map.get(&key) {
+            self.insert(key, value.to_owned() + increment_by);
+        } else {
+            self.insert(key, increment_by);
+        }
+    }
+
+    pub fn get_max_pair(&self) -> Option<(&K, &V)> {
+        if let Some(ref key) = self.max_valued_key {
+            return self.map.get_key_value(key);
+        }
+        None
+    }
+
+    pub fn len(&self) -> usize {
+        self.map.len()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_insert() {
+        let mut max_value_map = MaxValueMap::new();
+        max_value_map.insert("a", 10);
+        max_value_map.insert("b", 20);
+
+        assert_eq!(max_value_map.get_max_pair(), Some((&"b", &20)));
+    }
+
+    #[test]
+    fn test_increment() {
+        let mut max_value_map = MaxValueMap::new();
+        max_value_map.insert("a", 10);
+        max_value_map.increment("a", 15); // "a" becomes 25
+
+        assert_eq!(max_value_map.get_max_pair(), Some((&"a", &25)));
+    }
+
+    #[test]
+    fn test_get_max_pair_empty() {
+        let max_value_map: MaxValueMap<String, i32> = MaxValueMap::new();
+        assert_eq!(max_value_map.get_max_pair(), None);
+    }
+}

src/core/config/transformer/consolidate_url/mod.rs

@@ -0,0 +1,4 @@
+mod consolidate_url;
+mod max_value_map;
+
+pub use consolidate_url::ConsolidateURL;

src/core/config/transformer/consolidate_url/snapshots/tailcall__core__config__transformer__consolidate_url__consolidate_url__test__should_generate_correct_upstream_when_multiple_base_urls_present.snap

@@ -0,0 +1,31 @@
+---
+source: src/core/config/transformer/consolidate_url/consolidate_url.rs
+expression: transformed_config.to_sdl()
+---
+schema @server(hostname: "0.0.0.0", port: 8000) @upstream(baseURL: "http://jsonplaceholder-1.typicode.com", httpCache: 42) {
+  query: Query
+}
+
+type Post {
+  body: String!
+  id: Int!
+  title: String!
+  user: User @http(baseURL: "http://jsonplaceholder-2.typicode.com", path: "/users/{{.value.userId}}")
+  userId: Int!
+}
+
+type Query {
+  post(id: Int!): Post @http(path: "/posts/{{.args.id}}")
+  posts: [Post] @http(path: "/posts")
+  user(id: Int!): User @http(baseURL: "http://jsonplaceholder-3.typicode.com", path: "/users/{{.args.id}}")
+  users: [User] @http(baseURL: "http://jsonplaceholder-2.typicode.com", path: "/users")
+}
+
+type User {
+  email: String!
+  id: Int!
+  name: String!
+  phone: String
+  username: String!
+  website: String @expr(body: "/users/website/{{.value.username}}")
+}

src/core/config/transformer/consolidate_url/snapshots/tailcall__core__config__transformer__consolidate_url__consolidate_url__test__should_not_generate_upstream_when_threshold_is_not_matched.snap

@@ -0,0 +1,31 @@
+---
+source: src/core/config/transformer/consolidate_url/consolidate_url.rs
+expression: transformed_config.to_sdl()
+---
+schema @server(hostname: "0.0.0.0", port: 8000) @upstream(httpCache: 42) {
+  query: Query
+}
+
+type Post {
+  body: String!
+  id: Int!
+  title: String!
+  user: User @http(baseURL: "http://jsonplaceholder-2.typicode.com", path: "/users/{{.value.userId}}")
+  userId: Int!
+}
+
+type Query {
+  post(id: Int!): Post @http(baseURL: "http://jsonplaceholder-1.typicode.com", path: "/posts/{{.args.id}}")
+  posts: [Post] @http(baseURL: "http://jsonplaceholder-1.typicode.com", path: "/posts")
+  user(id: Int!): User @http(baseURL: "http://jsonplaceholder-3.typicode.com", path: "/users/{{.args.id}}")
+  users: [User] @http(baseURL: "http://jsonplaceholder-2.typicode.com", path: "/users")
+}
+
+type User {
+  email: String!
+  id: Int!
+  name: String!
+  phone: String
+  username: String!
+  website: String @expr(body: "/users/website/{{.value.username}}")
+}

src/core/config/transformer/mod.rs

@@ -1,8 +1,10 @@
 mod ambiguous_type;
+mod consolidate_url;
 mod remove_unused;
 mod type_merger;
 
 pub use ambiguous_type::{AmbiguousType, Resolution};
+pub use consolidate_url::ConsolidateURL;
 pub use remove_unused::RemoveUnused;
 pub use type_merger::TypeMerger;
 

src/core/generator/from_json.rs

@@ -4,7 +4,9 @@ use url::Url;
 use super::json::{
     FieldBaseUrlGenerator, NameGenerator, QueryGenerator, SchemaGenerator, TypesGenerator,
 };
-use crate::core::config::transformer::{RemoveUnused, Transform, TransformerOps, TypeMerger};
+use crate::core::config::transformer::{
+    ConsolidateURL, RemoveUnused, Transform, TransformerOps, TypeMerger,
+};
 use crate::core::config::Config;
 use crate::core::valid::Validator;
 
@@ -41,5 +43,7 @@ pub fn from_json(
             .to_result()?;
     }
 
+    let config = ConsolidateURL::new(0.5).transform(config).to_result()?;
+
     Ok(config)
 }

src/core/generator/snapshots/tailcall__core__generator__generator__test__read_all_with_different_domain_rest_api_gen.snap

@@ -2,13 +2,13 @@
 source: src/core/generator/generator.rs
 expression: config.to_sdl()
 ---
-schema @server @upstream {
+schema @server @upstream(baseURL: "https://jsonplaceholder.typicode.com") {
   query: Query
 }
 
 type Query {
-  f1(p1: Int!): [T1] @http(baseURL: "https://jsonplaceholder.typicode.com", path: "/posts/{{.args.p1}}/comments")
-  f2(p1: Int!): T2 @http(baseURL: "https://jsonplaceholder.typicode.com", path: "/posts/{{.args.p1}}")
+  f1(p1: Int!): [T1] @http(path: "/posts/{{.args.p1}}/comments")
+  f2(p1: Int!): T2 @http(path: "/posts/{{.args.p1}}")
   f3(q: String): T19 @http(baseURL: "https://dummyjson.com", path: "/products/search", query: [{key: "q", value: "{{.args.q}}"}])
 }
 

src/core/generator/snapshots/tailcall__core__generator__generator__test__read_all_with_rest_api_gen.snap

@@ -2,7 +2,7 @@
 source: src/core/generator/generator.rs
 expression: config.to_sdl()
 ---
-schema @server @upstream {
+schema @server @upstream(baseURL: "http://jsonplaceholder.typicode.com") {
   query: Query
 }
 
@@ -37,6 +37,6 @@ type M4 {
 }
 
 type Query {
-  f1: [M4] @http(baseURL: "http://jsonplaceholder.typicode.com", path: "/users")
-  f2(p1: Int!): M4 @http(baseURL: "http://jsonplaceholder.typicode.com", path: "/users/{{.args.p1}}")
+  f1: [M4] @http(path: "/users")
+  f2(p1: Int!): M4 @http(path: "/users/{{.args.p1}}")
 }

src/core/generator/tests/snapshots/json_to_config_spec__boolean.json.snap

@@ -2,10 +2,10 @@
 source: src/core/generator/tests/json_to_config_spec.rs
 expression: config.to_sdl()
 ---
-schema @server @upstream {
+schema @server @upstream(baseURL: "https://example.com") {
   query: Query
 }
 
 type Query {
-  f1(p1: Int!): Boolean @http(baseURL: "https://example.com", path: "/user/{{.args.p1}}/online")
+  f1(p1: Int!): Boolean @http(path: "/user/{{.args.p1}}/online")
 }

src/core/generator/tests/snapshots/json_to_config_spec__empty_list.json.snap

@@ -2,12 +2,12 @@
 source: src/core/generator/tests/json_to_config_spec.rs
 expression: config.to_sdl()
 ---
-schema @server @upstream {
+schema @server @upstream(baseURL: "https://example.com") {
   query: Query
 }
 
 scalar Any
 
 type Query {
-  f1: [Any] @http(baseURL: "https://example.com", path: "/users")
+  f1: [Any] @http(path: "/users")
 }