diff --git a/test/protoClientTest/README b/test/protoClientTest/README new file mode 100644 index 0000000000000..f144b322fcbf6 --- /dev/null +++ b/test/protoClientTest/README @@ -0,0 +1,269 @@ +Test with Proto client: +1. add same proto definition +2. generate proto_pb2.py + + +Test case: +Note: +1. disable cahe +2. change filter_term_count to generate target # of terms. (will be repeated) + +Index settings: +{ + "settings": { + "index": { + "fielddata": { + "cache": "none" + }, + "requests": { + "cache": { + "enable": false + } + } + } + }, + "mappings" : { + "properties" : { + "category" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "currency" : { + "type" : "keyword" + }, + "customer_birth_date" : { + "type" : "date" + }, + "customer_first_name" : { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "customer_full_name" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "customer_gender" : { + "type" : "keyword" + }, + "customer_id" : { + "type" : "keyword" + }, + "customer_last_name" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "customer_phone" : { + "type" : "keyword" + }, + "day_of_week" : { + "type" : "keyword" + }, + "day_of_week_i" : { + "type" : "integer" + }, + "email" : { + "type" : "keyword" + }, + "event" : { + "properties" : { + "dataset" : { + "type" : "keyword" + } + } + }, + "geoip" : { + "properties" : { + "city_name" : { + "type" : "keyword" + }, + "continent_name" : { + "type" : "keyword" + }, + "country_iso_code" : { + "type" : "keyword" + }, + "location" : { + "type" : "geo_point" + }, + "region_name" : { + "type" : "keyword" + } + } + }, + "manufacturer" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "order_date" : { + "type" : "date" + }, + "order_id" : { + "type" : "keyword" + }, + "products" : { + "properties" : { + "_id" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "base_price" : { + "type" : "half_float" + }, + "base_unit_price" : { + "type" : "half_float" + }, + "category" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "created_on" : { + "type" : "date" + }, + "discount_amount" : { + "type" : "half_float" + }, + "discount_percentage" : { + "type" : "half_float" + }, + "manufacturer" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "min_price" : { + "type" : "half_float" + }, + "price" : { + "type" : "half_float" + }, + "product_id" : { + "type" : "long" + }, + "product_name" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + }, + "analyzer" : "english" + }, + "quantity" : { + "type" : "integer" + }, + "sku" : { + "type" : "keyword" + }, + "tax_amount" : { + "type" : "half_float" + }, + "taxful_price" : { + "type" : "half_float" + }, + "taxless_price" : { + "type" : "half_float" + }, + "unit_discount_amount" : { + "type" : "half_float" + } + } + }, + "sku" : { + "type" : "keyword" + }, + "taxful_total_price" : { + "type" : "half_float" + }, + "taxless_total_price" : { + "type" : "half_float" + }, + "total_quantity" : { + "type" : "integer" + }, + "total_unique_products" : { + "type" : "integer" + }, + "type" : { + "type" : "keyword" + }, + "user" : { + "type" : "keyword" + } + } + } +} + +Query: +{ + "query": { + "bool": { + "must": [ + { + "match": { + "customer_first_name": "Eddie" + } + }, + { + "match": { + "customer_last_name": "Underwood" + } + }, + { + "match": { + "email": "eddie@underwood-family.zzz" + } + } + ], + "filter": [ + { + "term": { + "day_of_week": "Monday" + } + }, + { + "term": { + "customer_gender": "MALE" + } + }, + { + "term": { + "currency": "EUR" + } + } + ] + } diff --git a/test/protoClientTest/TestJsonSearchAPI.py b/test/protoClientTest/TestJsonSearchAPI.py new file mode 100644 index 0000000000000..2ab623a1c1cd4 --- /dev/null +++ b/test/protoClientTest/TestJsonSearchAPI.py @@ -0,0 +1,125 @@ +import requests +import json +from concurrent.futures import ThreadPoolExecutor, as_completed +import time +import sys +import random +import concurrent + + +url = "http://localhost:5436/ecommerce/_search" +headers = { + "Content-Type": "application/json", + "RPC-Service":"XX" +} + + +MOCK_INDEX_NAME = "ecommerce" + +MOCK_INDEX_NAME = "ecommerce" +MOCK_MATCH_FIELD_NAME1 = "customer_first_name" +MOCK_MATCH_VALUE1 = "Eddie" +MOCK_MATCH_FIELD_NAME2 = "customer_last_name" +MOCK_MATCH_VALUE2 = "Underwood" +MOCK_MATCH_FIELD_NAME3 = "email" +MOCK_MATCH_VALUE3 = "eddie@underwood-family.zzz" +MOCK_TERM_FIELD_NAME1 = "day_of_week" +MOCK_TERM_VALUE1 = "Monday" +MOCK_TERM_FIELD_NAME2 = "customer_gender" +MOCK_TERM_VALUE2 = "MALE" +MOCK_TERM_FIELD_NAME3 = "currency" +MOCK_TERM_VALUE3 = "EUR" + +term_map = { + MOCK_TERM_FIELD_NAME1: MOCK_TERM_VALUE1, + MOCK_TERM_FIELD_NAME2: MOCK_TERM_VALUE2, + MOCK_TERM_FIELD_NAME3: MOCK_TERM_VALUE3 + } + +def get_random_item_from_dict(my_dict): + key = random.choice(list(my_dict.keys())) + return key, my_dict[key] + +def generate_filter_term(): + filter_name, value_name = get_random_item_from_dict(term_map) + x = {filter_name: value_name} + return x + +def generate_all_filter_terms(count): + + result_dict = {} + + with concurrent.futures.ThreadPoolExecutor() as executor: + future_to_key = {executor.submit(generate_filter_term, i) for i in range(count)} + + for future in concurrent.futures.as_completed(future_to_key): + key, value = future.result() + result_dict[key] = value + print(result_dict) + return result_dict + +def generate_request(filter_term_count): + search_request = { + "query": { + "bool": { + "must": [ + { + "match": { + "customer_first_name": "Eddie" + } + }, + { + "match": { + "customer_last_name": "Underwood" + } + }, + { + "match": { + "email": "eddie@underwood-family.zzz" + } + } + ], + "filter": [{"term": generate_filter_term()} for i in range(filter_term_count)] + } + } + } + data = json.dumps(search_request) + return data + +def make_request(filter_term_count): + data = generate_request(filter_term_count) + payload_size = sys.getsizeof(data) + start = time.time() + response = requests.post(url, headers=headers, data=data) + end = time.time() + latency = end - start + if response.status_code == 200: + response_json = response.json() + took = response_json.get('took', 0) + print(response_json) + return took, latency + else: + print(f"Error: {response.status_code}, Content: {response.content}") + return 0, latency + +def run_multithreaded_requests(num_threads, num_calls): + filter_term_count = 200 + total_took = 0 + total_latency = 0 + with ThreadPoolExecutor(max_workers=num_threads) as executor: + + futures = [executor.submit(make_request, filter_term_count) for _ in range(num_calls)] + + for future in as_completed(futures): + took, latency = future.result() + total_latency += latency + total_took += took + + return total_took, total_latency + + +if __name__ == "__main__": + num_threads = 3 + num_calls = 1000 + run_multithreaded_requests(num_threads, num_calls) + diff --git a/test/protoClientTest/TestProtoSearchAPI.py b/test/protoClientTest/TestProtoSearchAPI.py new file mode 100644 index 0000000000000..e401a6319af03 --- /dev/null +++ b/test/protoClientTest/TestProtoSearchAPI.py @@ -0,0 +1,141 @@ +import sys +import requests +import base64 +import json +from concurrent.futures import ThreadPoolExecutor, as_completed +import time +import random +import concurrent + +sys.path.append('/home/user/IdeaProjects/searchProto/generated') + +from generated import MatchQueryProto_pb2 as MatchQueryProto +from generated import TermQueryProto_pb2 as TermQueryProto +from generated import BoolQueryProto_pb2 as BoolQueryProto +from generated import SearchRequestProto_pb2 as SearchRequestProto + +url = "http://localhost:5436/_search_proto" +headers = { + "Content-Type": "application/json", + "RPC-Service":"XX" +} + +MOCK_INDEX_NAME = "ecommerce" +MOCK_MATCH_FIELD_NAME1 = "customer_first_name" +MOCK_MATCH_VALUE1 = "Eddie" +MOCK_MATCH_FIELD_NAME2 = "customer_last_name" +MOCK_MATCH_VALUE2 = "Underwood" +MOCK_MATCH_FIELD_NAME3 = "email" +MOCK_MATCH_VALUE3 = "eddie@underwood-family.zzz" +MOCK_TERM_FIELD_NAME1 = "day_of_week" +MOCK_TERM_VALUE1 = "Monday" +MOCK_TERM_FIELD_NAME2 = "customer_gender" +MOCK_TERM_VALUE2 = "MALE" +MOCK_TERM_FIELD_NAME3 = "currency" +MOCK_TERM_VALUE3 = "EUR" + +def get_random_item_from_dict(my_dict): + key = random.choice(list(my_dict.keys())) + return key, my_dict[key] + +def generate_filter_term(i): + term_map = { + MOCK_TERM_FIELD_NAME1: MOCK_TERM_VALUE1, + MOCK_TERM_FIELD_NAME2: MOCK_TERM_VALUE2, + MOCK_TERM_FIELD_NAME3: MOCK_TERM_VALUE3 + } + filter_name, value_name = get_random_item_from_dict(term_map) + term_value = value_name + term_query = TermQueryProto.TermQuery( + fieldName=filter_name, + value=term_value + ) + return term_query + +def generate_all_filter_terms(count): + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [executor.submit(generate_filter_term, i) for i in range(count)] + results = [future.result() for future in concurrent.futures.as_completed(futures)] + return results + +def create_search_request(filter_term_count): + mock_match_query1 = MatchQueryProto.MatchQuery( + fieldName=MOCK_MATCH_FIELD_NAME1, + value=MOCK_MATCH_VALUE1 + ) + + mock_match_query2 = MatchQueryProto.MatchQuery( + fieldName=MOCK_MATCH_FIELD_NAME2, + value=MOCK_MATCH_VALUE2 + ) + + mock_match_query3 = MatchQueryProto.MatchQuery( + fieldName=MOCK_MATCH_FIELD_NAME3, + value=MOCK_MATCH_VALUE3 + ) + + filters = generate_all_filter_terms(filter_term_count) + + bool_query_builder = BoolQueryProto.BoolQuery( + must=[mock_match_query1, mock_match_query2, mock_match_query3], + filter=filters + ) + + query_builder = SearchRequestProto.SearchRequest.SourceBuilder.QueryBuilder( + bool=bool_query_builder + ) + + source_builder = SearchRequestProto.SearchRequest.SourceBuilder( + query=query_builder + ) + + search_request_builder = SearchRequestProto.SearchRequest( + indices=[MOCK_INDEX_NAME], + sourceBuilder=source_builder, + routing="null", + preference="null" + ) + + search_request_bytes = search_request_builder.SerializeToString() + return search_request_bytes + + +def make_request(filter_term_count): + data = create_search_request() + payload_size = sys.getsizeof(data) + start = time.time() + response = requests.get(url, headers=headers, data=data) + end = time.time() + latency = end - start + if response.status_code == 200: + response_json = response.json() + took = response_json.get('took', 0) + print(response_json) + return took, latency + else: + print(f"Error: {response.status_code}, Content: {response.content}") + return 0, latency + +def run_multithreaded_requests(num_threads, num_calls): + filter_term_count = 200 + total_took = 0 + total_latency = 0 + + with ThreadPoolExecutor(max_workers=num_threads) as executor: + + futures = [executor.submit(make_request, filter_term_count) for _ in range(num_calls)] + + for future in as_completed(futures): + took, latency = future.result() + total_latency += latency + total_took += took + + return total_took, total_latency + + +if __name__ == "__main__": + num_threads = 3 + num_calls = 1000 + run_multithreaded_requests(num_threads, num_calls) +