From 6d50ed0983e5bb295bc547aee42949a21d009bf7 Mon Sep 17 00:00:00 2001 From: khaledk2 Date: Sat, 28 Sep 2024 20:37:36 +0100 Subject: [PATCH] fix issue of returning a query within a container --- .../api/v1/resources/query_handler.py | 24 +++--- .../api/v1/resources/resource_analyser.py | 81 ++++--------------- omero_search_engine/api/v1/resources/urls.py | 3 +- omero_search_engine/api/v1/resources/utils.py | 5 +- .../validation/results_validator.py | 8 +- 5 files changed, 37 insertions(+), 84 deletions(-) diff --git a/omero_search_engine/api/v1/resources/query_handler.py b/omero_search_engine/api/v1/resources/query_handler.py index 03d4c7e..8d30cb8 100644 --- a/omero_search_engine/api/v1/resources/query_handler.py +++ b/omero_search_engine/api/v1/resources/query_handler.py @@ -44,13 +44,17 @@ def check_get_names(idr_, resource, attribute, return_exact=False): if idr_: idr_ = idr_.strip() pr_names = get_resource_names(resource) + all_act_names=[] if pr_names: if not return_exact: - act_name = [ - name["id"] - for name in pr_names - if name[attribute] and idr_.lower() in name[attribute].lower() - ] + for data_source, pr_names_ in pr_names.items(): + act_name = [ + name["id"] + for name in pr_names_ + if name[attribute] and idr_.lower() in name[attribute].lower() + ] + print (act_name, data_source) + all_act_names=all_act_names+act_name else: # This should be modified to query specific data source specific for data_source, pr_names_ in pr_names.items(): @@ -59,7 +63,8 @@ def check_get_names(idr_, resource, attribute, return_exact=False): for name in pr_names_ if name[attribute] and idr_.lower() == name[attribute].lower() ] - return act_name + all_act_names = all_act_names + act_name + return all_act_names class QueryItem(object): @@ -459,13 +464,13 @@ def search_query( search_omero_app.logger.info( "-------------------------------------------------" ) # noqa - search_omero_app.logger.info(query) - search_omero_app.logger.info(main_attributes) + # search_omero_app.logger.info("1. query: %s" % query) + search_omero_app.logger.info("2. main_attributes: %s "%main_attributes) search_omero_app.logger.info(resource) search_omero_app.logger.info( "-------------------------------------------------" ) # noqa - search_omero_app.logger.info(("%s, %s") % (resource, query)) + search_omero_app.logger.info(("2: resource: %s, 2: query: %s") % (resource, query)) if not main_attributes: q_data = {"query": {"query_details": query}} elif resource == "image": @@ -668,7 +673,6 @@ def determine_search_results_( if main_attributes: res_and_main_attributes = main_attributes.get("and_main_attributes") res_or_main_attributes = main_attributes.get("or_main_attributes") - columns_def = query_.get("columns_def") or_query_groups = [] if and_filters and len(and_filters) > 0: diff --git a/omero_search_engine/api/v1/resources/resource_analyser.py b/omero_search_engine/api/v1/resources/resource_analyser.py index 195e2aa..c9d351c 100644 --- a/omero_search_engine/api/v1/resources/resource_analyser.py +++ b/omero_search_engine/api/v1/resources/resource_analyser.py @@ -631,51 +631,12 @@ def search_value_for_resource( Search using key, part of the value and resource """ key_part_values_buckets_template = Template( - """ -{ - "query":{ - "bool":{ - "must":[ - { - "bool":{ - "must":[ - { - "match":{ - "Attribute.keyrnamenormalize":"$name" - } - }, - { - "wildcard":{ - "Value.keyvaluenormalize":"*$value*" - } - } - ] - } - }, - { - "bool":{ - "must":[ - { - "match":{ - "resource.keyresource":"$resource" - } - } - ] - } - }, - { - "bool":{ - "must":{ - "terms":{ - "data_source.keyvalue":$data_source - } - } - } - } - ] - } - } -}""" +"""{"query":{"bool":{"must":[{"bool":{"must":[{ +"match":{"Attribute.keyrnamenormalize":"$name"}}, +{"wildcard":{"Value.keyvaluenormalize":"*$value*"} +}]}},{"bool":{"must":[{"match":{"resource.keyresource":"$resource"} +}]}},{"bool":{"must":{"terms":{"data_source.keyvalue":$data_source}}}}]}}} +""" ) @@ -697,8 +658,7 @@ def search_value_for_resource( """ {"query":{"bool":{"must":[{"bool":{ "must":{"match":{"Value.keyvalue":"$value"}}}},{ -"bool": {"must": {"match": -{"resource.keyresource": "$resource"}}}}]}},"size": 9999}""" +"bool": {"must": {"match":{"resource.keyresource": "$resource"}}}}]}},"size": 9999}""" ) """ @@ -717,44 +677,31 @@ def search_value_for_resource( {"query":{"bool":{"must":[{"bool":{ "must":{"wildcard":{"Value.keyvaluenormalize":"*$value*"}}}},{"bool":{ "must":{"terms":{"data_source.keyvalue":$data_source}}}},{ -"bool": {"must": {"match": -{"resource.keyresource": "$resource"}}}}]}}}""" +"bool": {"must": {"match":{"resource.keyresource": "$resource"}}}}]}}}""" ) resource_key_values_buckets_template = Template( """ {"query":{"bool":{"must":[{"bool":{ "must":{"wildcard":{"Value.keyvaluenormalize":"*$value*"}}}},{ -"bool": {"must": {"match": -{"resource.keyresource": "$resource"}}}}, -{"bool":{ -"must":{"terms":{ "data_source.keyvalue":$data_source}}}} -]}}, -"size": 9999, "sort":[{ "_script": { - "script": "doc['Value.keyvaluenormalize'].value.length()", - "type": "number", - "order": "asc" - }},{"items_in_the_bucket": "desc"}, {"id": "asc"}]}""" +"bool": {"must": {"match":{"resource.keyresource": "$resource"}}}},{"bool":{ +"must":{"terms":{ "data_source.keyvalue":$data_source}}}}]}}, +"size": 9999, "sort":[{ "_script": {"script": "doc['Value.keyvaluenormalize'].value.length()","type": "number","order": "asc"}},{"items_in_the_bucket": "desc"}, {"id": "asc"}]}""" ) key_values_buckets_template_2 = Template( """ -{"query":{"bool":{"must":[{"bool":{"must":{"match":{ -"resource.keyresource":"$resource"}}}}]}}} """ +{"query":{"bool":{"must":[{"bool":{"must":{"match":{"resource.keyresource":"$resource"}}}}]}}} """ ) key_values_buckets_template_with_data_source = Template( """ -{"query":{"bool":{"must":[{"bool":{"must":{"match":{ -"resource.keyresource":"$resource"}}}},{"bool": {"must": -{"match": {"data_source.keyvalue":$data_source}}}}]}}} """ +{"query":{"bool":{"must":[{"bool":{"must":{"match":{"resource.keyresource":"$resource"}}}},{"bool": {"must":{"match": {"data_source.keyvalue":$data_source}}}}]}}} """ ) key_values_buckets_template_search_name = Template( """ -{"query":{"bool":{"must":[{"bool":{"must":{"match":{ -"resource.keyresource":"$resource"}}}},{"bool": {"must": -{"wildcard": {"resourcename.keyresourcename":"*$name*"}}}}]}}} """ +{"query":{"bool":{"must":[{"bool":{"must":{"match":{"resource.keyresource":"$resource"}}}},{"bool": {"must":{"wildcard": {"resourcename.keyresourcename":"*$name*"}}}}]}}} """ ) diff --git a/omero_search_engine/api/v1/resources/urls.py b/omero_search_engine/api/v1/resources/urls.py index 3423b32..328e87a 100644 --- a/omero_search_engine/api/v1/resources/urls.py +++ b/omero_search_engine/api/v1/resources/urls.py @@ -399,8 +399,10 @@ def submit_query(): adjust_query_for_container(query) return_columns = request.args.get("return_columns") data_source = request.args.get("data_source") + data_source = check_empty_string(data_source) if data_source: data_source = data_source.strip() + if return_columns: try: return_columns = json.loads(return_columns.lower()) @@ -408,7 +410,6 @@ def submit_query(): return_columns = False validation_results = query_validator(query) if validation_results == "OK": - return jsonify( determine_search_results_( query, data_source=data_source, return_columns=return_columns diff --git a/omero_search_engine/api/v1/resources/utils.py b/omero_search_engine/api/v1/resources/utils.py index ca55cb0..78fb79f 100644 --- a/omero_search_engine/api/v1/resources/utils.py +++ b/omero_search_engine/api/v1/resources/utils.py @@ -1133,7 +1133,6 @@ def search_resource_annotation( "{table_} is not a valid resurce".format(table_=table_) ) query_details = query.get("query_details") - start_time = time.time() if not raw_elasticsearch_query: query_details = query.get("query_details") @@ -1171,12 +1170,14 @@ def search_resource_annotation( main_attributes = {} main_attributes["and_main_attributes"] = [clause] + and_filters = query_details.get("and_filters") or_filters = query_details.get("or_filters") case_sensitive = query_details.get("case_sensitive") # check and fid if possible names and values inside # filters conditions - check_filters(table_, [and_filters, or_filters], case_sensitive) + #check_filters is cpmmented as it has no actual use and take time to excute + # check_filters(table_, [and_filters, or_filters], case_sensitive) query_string = elasticsearch_query_builder( and_filters, or_filters, case_sensitive, main_attributes ) diff --git a/omero_search_engine/validation/results_validator.py b/omero_search_engine/validation/results_validator.py index 94fec68..70b51ce 100644 --- a/omero_search_engine/validation/results_validator.py +++ b/omero_search_engine/validation/results_validator.py @@ -1020,10 +1020,10 @@ def check_number_images_sql_containers_using_ids(data_source): if ress["id"] not in test_array: print(ress["id"]) search_omero_app.logger.info("ERROR: Not equal results") - print(sql, query_data) - print("searchengine_results:", searchengine_results) - print("postgres_results: ", postgres_results) - print("==============-=======") + # print(sql, query_data) + # print("searchengine_results:", searchengine_results) + # print("postgres_results: ", postgres_results) + # print("==============-=======") print( "Error checking %s name: %s, id: %s" % (resource, res_name, res_id)