fix

khaledk2 · Sep 20, 2024 · 5ff232b · 5ff232b
1 parent ae6dfcb
commit 5ff232b
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 28 deletions.
diff --git a/manage.py b/manage.py
@@ -175,22 +175,23 @@ def get_index_data_from_database(resource="all", source="all", backup="True"):
             if resource.lower() != "all" and resource.lower() != res.lower():
                 continue
             get_insert_data_to_index(sql_st, res, data_source, clean_index)
-
         save_key_value_buckets(
             resource_table_=None,
             data_source=data_source,
             clean_index=clean_index,
             only_values=False,
         )
+        print("!Done for data_source: %s from %s" % (data_source, search_omero_app.config.database_connectors.keys()))
+    if clean_index:
+        clean_index = False
 
-        if clean_index:
-            clean_index = False
         # validat ethe indexing
         #test_indexing_search_query(
         #    source=data_source, deep_check=False, check_studies=True
         #)
 
-    # backup the index data
+
+     #backup the index data
     # if backup:
     #    backup_elasticsearch_data()
 

diff --git a/omero_search_engine/__init__.py b/omero_search_engine/__init__.py
@@ -82,6 +82,7 @@ def create_app(config_name="development"):
         scheme="https",
         http_auth=("elastic", ELASTIC_PASSWORD),
     )
+
     search_omero_app.config.database_connectors = app_config.database_connectors
     print(search_omero_app.config.database_connectors)
     search_omero_app.config["es_connector"] = es_connector
@@ -104,6 +105,7 @@ def create_app(config_name="development"):
 
     search_omero_app.logger.setLevel(logging.INFO)
     search_omero_app.logger.info("app assistant startup")
+
     return search_omero_app
 
 

diff --git a/omero_search_engine/api/v1/resources/query_handler.py b/omero_search_engine/api/v1/resources/query_handler.py
@@ -206,7 +206,6 @@ def __init__(
         self.return_columns = return_columns
         self.return_containers = return_containers
         self.data_source=data_source
-        print ("TOOOOOOOOOOOOOOOOOOZ::::::::", self.data_source)
 
     def get_image_non_image_query(self):
         res = None
@@ -429,7 +428,6 @@ def run_query(self, query_, resource):
                 data_source=self.data_source
             )
         else:
-            print("TOOOOOOOOOOOOOOOOZ2:",self.data_source)
             res = search_query(
                 query,
                 resource,
@@ -467,7 +465,6 @@ def search_query(
     search_omero_app.logger.info(
         "-------------------------------------------------"
     )  # noqa
-    print ("TOOOOOOOOOOZZ33::::", data_source)
     search_omero_app.logger.info(("%s, %s") % (resource, query))
     if not main_attributes:
         q_data = {"query": {"query_details": query}}

diff --git a/omero_search_engine/api/v1/resources/resource_analyser.py b/omero_search_engine/api/v1/resources/resource_analyser.py
@@ -41,7 +41,7 @@
 }}}}}}}}"""
 )
 key_number_search_template = Template(
-    """
+    r"""
 {
    "size":0,
     "query":{ "bool" : {"must": {
@@ -261,7 +261,11 @@ def get_number_of_buckets(key, data_source, res_index):
 def get_all_values_for_a_key(table_, data_source, key):
     res_index = resource_elasticsearchindex.get(table_)
     query = key_number_search_template.substitute(key=key, data_source=data_source)
-    res = search_index_for_value(res_index, query)
+    try:
+        res = search_index_for_value(res_index, query)
+    except Exception as ex:
+        print("Query: %s Error: %s"%(query,str(ex)))
+        raise ex
     number_of_buckets = (
         res.get("aggregations")
         .get("value_search")
@@ -333,7 +337,6 @@ def get_values_for_a_key(table_, key, data_source):
     start_time = time.time()
     res = search_index_for_value(res_index, query)
     query_time = "%.2f" % (time.time() - start_time)
-    print("TIME ...", query_time)
     returned_results = []
     if res.get("aggregations"):
         for bucket in (
@@ -906,10 +909,6 @@ def get_resource_names(resource, name=None, description=False):
         ress = ["project", "screen"]
         for res in ress:
             returned_results[res] = get_the_results(res, name, description)
-
-    print("#############################################")
-    print(returned_results)
-    print("#############################################")
     return returned_results
 
 
@@ -965,10 +964,8 @@ def get_container_values_for_key(table_, container_name, csv, ret_data_source=No
         ret_data_source = [itm.strip().lower() for itm in ret_data_source.split(',')]
     for resourse, names_ in pr_names.items():
         for data_source, names in names_.items():
-            print ("====>>>>>",data_source, names)
             if ret_data_source:
                 if data_source.lower() not in ret_data_source:
-                    print ("TTTTTCONAZZZ",ret_data_source)
                     continue
             act_name = [
                 {"id": name["id"], "name": name["name"]}

diff --git a/omero_search_engine/api/v1/resources/utils.py b/omero_search_engine/api/v1/resources/utils.py
@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import copy
+import logging
 import os
 import sys
 import json
@@ -1116,9 +1117,6 @@ def search_resource_annotation(
     @raw_elasticsearch_query: raw query sending directly to elasticsearch
     """
     # try:
-    print ("tiiiiiiiiz")
-    print (data_source)
-    print("tiiiiiiiiz")
     res_index = resource_elasticsearchindex.get(table_)
     if not res_index:
         return build_error_message(
@@ -1140,7 +1138,7 @@ def search_resource_annotation(
             or len(query_details) == 0
             or isinstance(query_details, str)
         ):
-            print("Error ")
+            logging.info("Error ")
             return build_error_message(
                 "{query} is not a valid query".format(query=query)
             )
@@ -1178,15 +1176,14 @@ def search_resource_annotation(
         if isinstance(query_string, dict):
             return query_string
 
-        search_omero_app.logger.info("Query %s" % query_string)
+        #search_omero_app.logger.info("Query %s" % query_string)
 
         query = json.loads(query_string, strict=False)
         raw_query_to_send_back = json.loads(query_string, strict=False)
     else:
         query = raw_elasticsearch_query
         raw_query_to_send_back = copy.copy(raw_elasticsearch_query)
     if return_containers:
-        print ("1222222222222222222222")
         # code to return the containers only
         # It will call the projects container first then
         # search within screens
@@ -1214,7 +1211,6 @@ def search_resource_annotation(
         studies = res + res_2
         res = {"results": studies}
     else:
-        print ("333333333333333333333")
         res = search_index_using_search_after(
             res_index, query, bookmark, pagination_dict, return_containers, data_source=data_source
         )

diff --git a/omero_search_engine/cache_functions/elasticsearch/transform_data.py b/omero_search_engine/cache_functions/elasticsearch/transform_data.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+import logging
 
 # Copyright (C) 2022 University of Dundee & Open Microscopy Environment.
 # All rights reserved.
@@ -531,7 +532,7 @@ def processor_work(lock, global_counter, val):
         lock.acquire()
         global_counter.value += 1
     except Exception as ex:
-        print("Error is %s" % ex)
+        print("Error %s" % ex)
         raise ex
     finally:
         lock.release()
@@ -587,6 +588,7 @@ def insert_resource_data_from_df(df, resource, data_source, lock=None):
 
         actions.append({"_index": es_index, "_source": record})  # ,
     es = search_omero_app.config.get("es_connector")
+    #logging.getLogger("elasticsearch").setLevel(logging.ERROR)
     search_omero_app.logger.info("Pushing the data to the Elasticsearch")
     try:
         lock.acquire()
@@ -730,7 +732,7 @@ def save_key_value_buckets(
         push_keys_cache_index(
             resource_keys, resource_table, data_source, es_index_2, name_results
         )
-        print(type(resource_keys), type(resource_table), es_index_2, type(name_results))
+        logging.info(type(resource_keys), type(resource_table), es_index_2, type(name_results))
         if only_values:
             continue
         search_omero_app.logger.info(
@@ -759,7 +761,7 @@ def save_key_value_buckets(
             counter_val = manager.Value("i", 0)
             func = partial(save_key_value_buckets_process, lock, counter_val)
             res = pool.map(func, vals)
-            print(res)
+            search_omero_app.logger.info(res)
         finally:
             pool.close()
 
@@ -806,8 +808,8 @@ def save_key_value_buckets_process(lock, global_counter, vals):
             search_omero_app.logger.info(helpers.bulk(es, actions))
         except Exception as e:
             search_omero_app.logger.info("Error:  %s" % str(e))
+            #raise e
             raise e
-            # raise e
         finally:
             lock.release()
     except Exception as e:
@@ -816,7 +818,7 @@ def save_key_value_buckets_process(lock, global_counter, vals):
             Error:%s "
             % (global_counter.value, str(e))
         )
-        raise e
+        #raise e
         if wrong_keys.get(resource_table):
             wrong_keys[resource_table] = wrong_keys[resource_table].append(key)
         else: