From 5ff232bd5e96f3b9d4e43d53c36517c5ced5284b Mon Sep 17 00:00:00 2001 From: khaledk2 Date: Fri, 20 Sep 2024 14:09:30 +0100 Subject: [PATCH] fix --- manage.py | 9 +++++---- omero_search_engine/__init__.py | 2 ++ .../api/v1/resources/query_handler.py | 3 --- .../api/v1/resources/resource_analyser.py | 15 ++++++--------- omero_search_engine/api/v1/resources/utils.py | 10 +++------- .../elasticsearch/transform_data.py | 12 +++++++----- 6 files changed, 23 insertions(+), 28 deletions(-) diff --git a/manage.py b/manage.py index 4f33f61..cf66edb 100644 --- a/manage.py +++ b/manage.py @@ -175,22 +175,23 @@ def get_index_data_from_database(resource="all", source="all", backup="True"): if resource.lower() != "all" and resource.lower() != res.lower(): continue get_insert_data_to_index(sql_st, res, data_source, clean_index) - save_key_value_buckets( resource_table_=None, data_source=data_source, clean_index=clean_index, only_values=False, ) + print("!Done for data_source: %s from %s" % (data_source, search_omero_app.config.database_connectors.keys())) + if clean_index: + clean_index = False - if clean_index: - clean_index = False # validat ethe indexing #test_indexing_search_query( # source=data_source, deep_check=False, check_studies=True #) - # backup the index data + + #backup the index data # if backup: # backup_elasticsearch_data() diff --git a/omero_search_engine/__init__.py b/omero_search_engine/__init__.py index 40ebca6..b617199 100644 --- a/omero_search_engine/__init__.py +++ b/omero_search_engine/__init__.py @@ -82,6 +82,7 @@ def create_app(config_name="development"): scheme="https", http_auth=("elastic", ELASTIC_PASSWORD), ) + search_omero_app.config.database_connectors = app_config.database_connectors print(search_omero_app.config.database_connectors) search_omero_app.config["es_connector"] = es_connector @@ -104,6 +105,7 @@ def create_app(config_name="development"): search_omero_app.logger.setLevel(logging.INFO) search_omero_app.logger.info("app assistant startup") + return search_omero_app diff --git a/omero_search_engine/api/v1/resources/query_handler.py b/omero_search_engine/api/v1/resources/query_handler.py index 208e9b6..983ffa4 100644 --- a/omero_search_engine/api/v1/resources/query_handler.py +++ b/omero_search_engine/api/v1/resources/query_handler.py @@ -206,7 +206,6 @@ def __init__( self.return_columns = return_columns self.return_containers = return_containers self.data_source=data_source - print ("TOOOOOOOOOOOOOOOOOOZ::::::::", self.data_source) def get_image_non_image_query(self): res = None @@ -429,7 +428,6 @@ def run_query(self, query_, resource): data_source=self.data_source ) else: - print("TOOOOOOOOOOOOOOOOZ2:",self.data_source) res = search_query( query, resource, @@ -467,7 +465,6 @@ def search_query( search_omero_app.logger.info( "-------------------------------------------------" ) # noqa - print ("TOOOOOOOOOOZZ33::::", data_source) search_omero_app.logger.info(("%s, %s") % (resource, query)) if not main_attributes: q_data = {"query": {"query_details": query}} diff --git a/omero_search_engine/api/v1/resources/resource_analyser.py b/omero_search_engine/api/v1/resources/resource_analyser.py index f1b4033..e138792 100644 --- a/omero_search_engine/api/v1/resources/resource_analyser.py +++ b/omero_search_engine/api/v1/resources/resource_analyser.py @@ -41,7 +41,7 @@ }}}}}}}}""" ) key_number_search_template = Template( - """ + r""" { "size":0, "query":{ "bool" : {"must": { @@ -261,7 +261,11 @@ def get_number_of_buckets(key, data_source, res_index): def get_all_values_for_a_key(table_, data_source, key): res_index = resource_elasticsearchindex.get(table_) query = key_number_search_template.substitute(key=key, data_source=data_source) - res = search_index_for_value(res_index, query) + try: + res = search_index_for_value(res_index, query) + except Exception as ex: + print("Query: %s Error: %s"%(query,str(ex))) + raise ex number_of_buckets = ( res.get("aggregations") .get("value_search") @@ -333,7 +337,6 @@ def get_values_for_a_key(table_, key, data_source): start_time = time.time() res = search_index_for_value(res_index, query) query_time = "%.2f" % (time.time() - start_time) - print("TIME ...", query_time) returned_results = [] if res.get("aggregations"): for bucket in ( @@ -906,10 +909,6 @@ def get_resource_names(resource, name=None, description=False): ress = ["project", "screen"] for res in ress: returned_results[res] = get_the_results(res, name, description) - - print("#############################################") - print(returned_results) - print("#############################################") return returned_results @@ -965,10 +964,8 @@ def get_container_values_for_key(table_, container_name, csv, ret_data_source=No ret_data_source = [itm.strip().lower() for itm in ret_data_source.split(',')] for resourse, names_ in pr_names.items(): for data_source, names in names_.items(): - print ("====>>>>>",data_source, names) if ret_data_source: if data_source.lower() not in ret_data_source: - print ("TTTTTCONAZZZ",ret_data_source) continue act_name = [ {"id": name["id"], "name": name["name"]} diff --git a/omero_search_engine/api/v1/resources/utils.py b/omero_search_engine/api/v1/resources/utils.py index 43af70c..15aafef 100644 --- a/omero_search_engine/api/v1/resources/utils.py +++ b/omero_search_engine/api/v1/resources/utils.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import copy +import logging import os import sys import json @@ -1116,9 +1117,6 @@ def search_resource_annotation( @raw_elasticsearch_query: raw query sending directly to elasticsearch """ # try: - print ("tiiiiiiiiz") - print (data_source) - print("tiiiiiiiiz") res_index = resource_elasticsearchindex.get(table_) if not res_index: return build_error_message( @@ -1140,7 +1138,7 @@ def search_resource_annotation( or len(query_details) == 0 or isinstance(query_details, str) ): - print("Error ") + logging.info("Error ") return build_error_message( "{query} is not a valid query".format(query=query) ) @@ -1178,7 +1176,7 @@ def search_resource_annotation( if isinstance(query_string, dict): return query_string - search_omero_app.logger.info("Query %s" % query_string) + #search_omero_app.logger.info("Query %s" % query_string) query = json.loads(query_string, strict=False) raw_query_to_send_back = json.loads(query_string, strict=False) @@ -1186,7 +1184,6 @@ def search_resource_annotation( query = raw_elasticsearch_query raw_query_to_send_back = copy.copy(raw_elasticsearch_query) if return_containers: - print ("1222222222222222222222") # code to return the containers only # It will call the projects container first then # search within screens @@ -1214,7 +1211,6 @@ def search_resource_annotation( studies = res + res_2 res = {"results": studies} else: - print ("333333333333333333333") res = search_index_using_search_after( res_index, query, bookmark, pagination_dict, return_containers, data_source=data_source ) diff --git a/omero_search_engine/cache_functions/elasticsearch/transform_data.py b/omero_search_engine/cache_functions/elasticsearch/transform_data.py index d424f3a..58de101 100644 --- a/omero_search_engine/cache_functions/elasticsearch/transform_data.py +++ b/omero_search_engine/cache_functions/elasticsearch/transform_data.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import logging # Copyright (C) 2022 University of Dundee & Open Microscopy Environment. # All rights reserved. @@ -531,7 +532,7 @@ def processor_work(lock, global_counter, val): lock.acquire() global_counter.value += 1 except Exception as ex: - print("Error is %s" % ex) + print("Error %s" % ex) raise ex finally: lock.release() @@ -587,6 +588,7 @@ def insert_resource_data_from_df(df, resource, data_source, lock=None): actions.append({"_index": es_index, "_source": record}) # , es = search_omero_app.config.get("es_connector") + #logging.getLogger("elasticsearch").setLevel(logging.ERROR) search_omero_app.logger.info("Pushing the data to the Elasticsearch") try: lock.acquire() @@ -730,7 +732,7 @@ def save_key_value_buckets( push_keys_cache_index( resource_keys, resource_table, data_source, es_index_2, name_results ) - print(type(resource_keys), type(resource_table), es_index_2, type(name_results)) + logging.info(type(resource_keys), type(resource_table), es_index_2, type(name_results)) if only_values: continue search_omero_app.logger.info( @@ -759,7 +761,7 @@ def save_key_value_buckets( counter_val = manager.Value("i", 0) func = partial(save_key_value_buckets_process, lock, counter_val) res = pool.map(func, vals) - print(res) + search_omero_app.logger.info(res) finally: pool.close() @@ -806,8 +808,8 @@ def save_key_value_buckets_process(lock, global_counter, vals): search_omero_app.logger.info(helpers.bulk(es, actions)) except Exception as e: search_omero_app.logger.info("Error: %s" % str(e)) + #raise e raise e - # raise e finally: lock.release() except Exception as e: @@ -816,7 +818,7 @@ def save_key_value_buckets_process(lock, global_counter, vals): Error:%s " % (global_counter.value, str(e)) ) - raise e + #raise e if wrong_keys.get(resource_table): wrong_keys[resource_table] = wrong_keys[resource_table].append(key) else: