From 529f1a76be2e10759b44e6cfb21a7d43bbc53109 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar <83278309+tsdk02@users.noreply.github.com> Date: Fri, 8 Nov 2024 15:35:24 +0530 Subject: [PATCH] feat(opensearch): refactor global search querybuilder and add case insensitivity opensearch filters (#6476) --- crates/analytics/src/opensearch.rs | 223 ++++++++++++++++++----------- crates/analytics/src/search.rs | 12 +- 2 files changed, 151 insertions(+), 84 deletions(-) diff --git a/crates/analytics/src/opensearch.rs b/crates/analytics/src/opensearch.rs index a6e6c486ebe1..84a2b9db3d4e 100644 --- a/crates/analytics/src/opensearch.rs +++ b/crates/analytics/src/opensearch.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use api_models::{ analytics::search::SearchIndex, errors::types::{ApiError, ApiErrorResponse}, @@ -456,7 +458,8 @@ pub struct OpenSearchQueryBuilder { pub count: Option, pub filters: Vec<(String, Vec)>, pub time_range: Option, - pub search_params: Vec, + search_params: Vec, + case_sensitive_fields: HashSet<&'static str>, } impl OpenSearchQueryBuilder { @@ -469,6 +472,12 @@ impl OpenSearchQueryBuilder { count: Default::default(), filters: Default::default(), time_range: Default::default(), + case_sensitive_fields: HashSet::from([ + "customer_email.keyword", + "search_tags.keyword", + "card_last_4.keyword", + "payment_id.keyword", + ]), } } @@ -490,48 +499,16 @@ impl OpenSearchQueryBuilder { pub fn get_status_field(&self, index: &SearchIndex) -> &str { match index { - SearchIndex::Refunds => "refund_status.keyword", - SearchIndex::Disputes => "dispute_status.keyword", + SearchIndex::Refunds | SearchIndex::SessionizerRefunds => "refund_status.keyword", + SearchIndex::Disputes | SearchIndex::SessionizerDisputes => "dispute_status.keyword", _ => "status.keyword", } } - pub fn replace_status_field(&self, filters: &[Value], index: &SearchIndex) -> Vec { - filters - .iter() - .map(|filter| { - if let Some(terms) = filter.get("terms").and_then(|v| v.as_object()) { - let mut new_filter = filter.clone(); - if let Some(new_terms) = - new_filter.get_mut("terms").and_then(|v| v.as_object_mut()) - { - let key = "status.keyword"; - if let Some(status_terms) = terms.get(key) { - new_terms.remove(key); - new_terms.insert( - self.get_status_field(index).to_string(), - status_terms.clone(), - ); - } - } - new_filter - } else { - filter.clone() - } - }) - .collect() - } - - /// # Panics - /// - /// This function will panic if: - /// - /// * The structure of the JSON query is not as expected (e.g., missing keys or incorrect types). - /// - /// Ensure that the input data and the structure of the query are valid and correctly handled. - pub fn construct_payload(&self, indexes: &[SearchIndex]) -> QueryResult> { - let mut query_obj = Map::new(); - let mut bool_obj = Map::new(); + pub fn build_filter_array( + &self, + case_sensitive_filters: Vec<&(String, Vec)>, + ) -> Vec { let mut filter_array = Vec::new(); filter_array.push(json!({ @@ -542,13 +519,12 @@ impl OpenSearchQueryBuilder { } })); - let mut filters = self - .filters - .iter() + let case_sensitive_json_filters = case_sensitive_filters + .into_iter() .map(|(k, v)| json!({"terms": {k: v}})) .collect::>(); - filter_array.append(&mut filters); + filter_array.extend(case_sensitive_json_filters); if let Some(ref time_range) = self.time_range { let range = json!(time_range); @@ -559,8 +535,72 @@ impl OpenSearchQueryBuilder { })); } - let should_array = self - .search_params + filter_array + } + + pub fn build_case_insensitive_filters( + &self, + mut payload: Value, + case_insensitive_filters: &[&(String, Vec)], + auth_array: Vec, + index: &SearchIndex, + ) -> Value { + let mut must_array = case_insensitive_filters + .iter() + .map(|(k, v)| { + let key = if *k == "status.keyword" { + self.get_status_field(index).to_string() + } else { + k.clone() + }; + json!({ + "bool": { + "must": [ + { + "bool": { + "should": v.iter().map(|value| { + json!({ + "term": { + format!("{}", key): { + "value": value, + "case_insensitive": true + } + } + }) + }).collect::>(), + "minimum_should_match": 1 + } + } + ] + } + }) + }) + .collect::>(); + + must_array.push(json!({ "bool": { + "must": [ + { + "bool": { + "should": auth_array, + "minimum_should_match": 1 + } + } + ] + }})); + + if let Some(query) = payload.get_mut("query") { + if let Some(bool_obj) = query.get_mut("bool") { + if let Some(bool_map) = bool_obj.as_object_mut() { + bool_map.insert("must".to_string(), Value::Array(must_array)); + } + } + } + + payload + } + + pub fn build_auth_array(&self) -> Vec { + self.search_params .iter() .map(|user_level| match user_level { AuthInfo::OrgLevel { org_id } => { @@ -579,11 +619,17 @@ impl OpenSearchQueryBuilder { }) } AuthInfo::MerchantLevel { - org_id: _, + org_id, merchant_ids, } => { let must_clauses = vec![ - // TODO: Add org_id field to the filters + json!({ + "term": { + "organization_id.keyword": { + "value": org_id + } + } + }), json!({ "terms": { "merchant_id.keyword": merchant_ids @@ -598,12 +644,18 @@ impl OpenSearchQueryBuilder { }) } AuthInfo::ProfileLevel { - org_id: _, + org_id, merchant_id, profile_ids, } => { let must_clauses = vec![ - // TODO: Add org_id field to the filters + json!({ + "term": { + "organization_id.keyword": { + "value": org_id + } + } + }), json!({ "term": { "merchant_id.keyword": { @@ -625,55 +677,60 @@ impl OpenSearchQueryBuilder { }) } }) - .collect::>(); + .collect::>() + } + + /// # Panics + /// + /// This function will panic if: + /// + /// * The structure of the JSON query is not as expected (e.g., missing keys or incorrect types). + /// + /// Ensure that the input data and the structure of the query are valid and correctly handled. + pub fn construct_payload(&self, indexes: &[SearchIndex]) -> QueryResult> { + let mut query_obj = Map::new(); + let mut bool_obj = Map::new(); + + let (case_sensitive_filters, case_insensitive_filters): (Vec<_>, Vec<_>) = self + .filters + .iter() + .partition(|(k, _)| self.case_sensitive_fields.contains(k.as_str())); + + let filter_array = self.build_filter_array(case_sensitive_filters); if !filter_array.is_empty() { bool_obj.insert("filter".to_string(), Value::Array(filter_array)); } + + let should_array = self.build_auth_array(); + if !bool_obj.is_empty() { query_obj.insert("bool".to_string(), Value::Object(bool_obj)); } - let mut query = Map::new(); - query.insert("query".to_string(), Value::Object(query_obj)); + let mut sort_obj = Map::new(); + sort_obj.insert( + "@timestamp".to_string(), + json!({ + "order": "desc" + }), + ); Ok(indexes .iter() .map(|index| { - let updated_query = query - .get("query") - .and_then(|q| q.get("bool")) - .and_then(|b| b.get("filter")) - .and_then(|f| f.as_array()) - .map(|filters| self.replace_status_field(filters, index)) - .unwrap_or_default(); - let mut final_bool_obj = Map::new(); - if !updated_query.is_empty() { - final_bool_obj.insert("filter".to_string(), Value::Array(updated_query)); - } - if !should_array.is_empty() { - final_bool_obj.insert("should".to_string(), Value::Array(should_array.clone())); - final_bool_obj - .insert("minimum_should_match".to_string(), Value::Number(1.into())); - } - let mut final_query = Map::new(); - if !final_bool_obj.is_empty() { - final_query.insert("bool".to_string(), Value::Object(final_bool_obj)); - } - - let mut sort_obj = Map::new(); - sort_obj.insert( - "@timestamp".to_string(), - json!({ - "order": "desc" - }), - ); - let payload = json!({ - "query": Value::Object(final_query), + let mut payload = json!({ + "query": query_obj.clone(), "sort": [ - Value::Object(sort_obj) + Value::Object(sort_obj.clone()) ] }); + payload = self.build_case_insensitive_filters( + payload, + &case_insensitive_filters, + should_array.clone(), + index, + ); payload }) .collect::>()) diff --git a/crates/analytics/src/search.rs b/crates/analytics/src/search.rs index f53b07b12321..9be0200030d2 100644 --- a/crates/analytics/src/search.rs +++ b/crates/analytics/src/search.rs @@ -190,7 +190,17 @@ pub async fn search_results( search_params: Vec, ) -> CustomResult { let search_req = req.search_req; - + if search_req.query.trim().is_empty() + && search_req + .filters + .as_ref() + .map_or(true, |filters| filters.is_all_none()) + { + return Err(OpenSearchError::BadRequestError( + "Both query and filters are empty".to_string(), + ) + .into()); + } let mut query_builder = OpenSearchQueryBuilder::new( OpenSearchQuery::Search(req.index), search_req.query,