fix per commit issues

configurations/configuration.py

@@ -104,6 +104,7 @@ def update_config_file(updated_configuration, configure_database=False):
         with open(app_config.INSTANCE_CONFIG, "w") as f:
             yaml.dump(configuration, f)
 
+
 def config_database(configuration, updated_configuration):
     for data_source in configuration.get("DATA_SOURCES"):
         changed = False

manage.py

@@ -123,6 +123,7 @@ def restore_postgresql_database(source="all"):
 
     restore_database(source)
 
+
 @manager.command
 @manager.option(
     "-r",
@@ -181,7 +182,10 @@ def get_index_data_from_database(resource="all", source="all", backup="True"):
             clean_index=clean_index,
             only_values=False,
         )
-        print("!Done for data_source: %s from %s" % (data_source, search_omero_app.config.database_connectors.keys()))
+        print(
+            "!Done for data_source: %s from %s"
+            % (data_source, search_omero_app.config.database_connectors.keys())
+        )
         if clean_index:
             clean_index = False
 
@@ -190,9 +194,9 @@ def get_index_data_from_database(resource="all", source="all", backup="True"):
             source=data_source, deep_check=False, check_studies=True
         )
 
-    #backup the index data
+    # backup the index data
     if backup:
-      backup_elasticsearch_data()
+        backup_elasticsearch_data()
 
 
 # set configurations
@@ -338,7 +342,6 @@ def set_no_processes(no_processes=None):
     "--data_source",
     help="data source name, the default is all",  # noqa
 )
-
 @manager.option(
     "-r",
     "--resource",
@@ -350,15 +353,17 @@ def set_no_processes(no_processes=None):
     help="creating the elastic search index if set to True",  # noqa
 )
 @manager.option("-o", "--only_values", help="creating cached values only ")
-def cache_key_value_index(resource=None, data_source='all',create_index=None, only_values=None):
+def cache_key_value_index(
+    resource=None, data_source="all", create_index=None, only_values=None
+):
     """
     Cache the value bucket for each value for each resource
     """
     from omero_search_engine.cache_functions.elasticsearch.transform_data import (
         save_key_value_buckets,
     )
 
-    save_key_value_buckets(resource,data_source ,create_index, only_values)
+    save_key_value_buckets(resource, data_source, create_index, only_values)
 
 
 @manager.command

omero_search_engine/api/v1/resources/query_handler.py

@@ -192,7 +192,7 @@ def __init__(
         columns_def,
         return_columns,
         return_containers,
-        data_source
+        data_source,
     ):
         self.or_query_group = or_query_group
         self.and_query_group = and_query_group
@@ -205,7 +205,7 @@ def __init__(
         self.additional_image_conds = []
         self.return_columns = return_columns
         self.return_containers = return_containers
-        self.data_source=data_source
+        self.data_source = data_source
 
     def get_image_non_image_query(self):
         res = None
@@ -425,7 +425,7 @@ def run_query(self, query_, resource):
                 self.raw_elasticsearch_query,
                 main_attributes,
                 return_containers=self.return_containers,
-                data_source=self.data_source
+                data_source=self.data_source,
             )
         else:
             res = search_query(
@@ -435,7 +435,7 @@ def run_query(self, query_, resource):
                 pagination_dict,
                 self.raw_elasticsearch_query,
                 main_attributes,
-                data_source=self.data_source
+                data_source=self.data_source,
             )
 
         if resource != "image":
@@ -454,7 +454,7 @@ def search_query(
     raw_elasticsearch_query,
     main_attributes=None,
     return_containers=False,
-    data_source=None
+    data_source=None,
 ):
     search_omero_app.logger.info(
         "-------------------------------------------------"
@@ -484,16 +484,19 @@ def search_query(
                 bookmark=bookmark,
                 pagination_dict=pagination_dict,
                 return_containers=return_containers,
-                data_source=data_source
+                data_source=data_source,
             )
         else:
             # Should have a method to search the elasticsearch and
             # returns the containers only,
             # It is hard coded in the util search_annotation method.
             ress = search_resource_annotation(
-                resource, q_data.get("query"), return_containers=return_containers, data_source=data_source
+                resource,
+                q_data.get("query"),
+                return_containers=return_containers,
+                data_source=data_source,
             )
-        if type (ress) is str:
+        if type(ress) is str:
             return ress
 
         ress["Error"] = "none"
@@ -644,7 +647,9 @@ def process_search_results(results, resource, columns_def):
     return returned_results
 
 
-def determine_search_results_(query_,data_source="all", return_columns=False, return_containers=False):
+def determine_search_results_(
+    query_, data_source="all", return_columns=False, return_containers=False
+):
     from omero_search_engine.api.v1.resources.utils import build_error_message
 
     if query_.get("query_details"):
@@ -750,7 +755,7 @@ def determine_search_results_(query_,data_source="all", return_columns=False, re
         columns_def,
         return_columns,
         return_containers,
-        data_source
+        data_source,
     )
     query_results = query_runner.get_image_non_image_query()
     return query_results
@@ -787,7 +792,7 @@ def simple_search(
             {"query_details": query_details},
             bookmark=bookmark,
             return_containers=return_containers,
-            data_source=data_source
+            data_source=data_source,
         )
     else:
         and_filters.append(
@@ -798,7 +803,9 @@ def simple_search(
                 "resource": "project",
             }
         )
-        return determine_search_results_({"query_details": query_details},data_source=data_source)
+        return determine_search_results_(
+            {"query_details": query_details}, data_source=data_source
+        )
 
 
 def add_local_schemas_to(resolver, schema_folder, base_uri, schema_ext=".json"):

omero_search_engine/api/v1/resources/resource_analyser.py

@@ -264,7 +264,7 @@ def get_all_values_for_a_key(table_, data_source, key):
     try:
         res = search_index_for_value(res_index, query)
     except Exception as ex:
-        print("Query: %s Error: %s"%(query,str(ex)))
+        print("Query: %s Error: %s" % (query, str(ex)))
         raise ex
     number_of_buckets = (
         res.get("aggregations")
@@ -337,6 +337,7 @@ def get_values_for_a_key(table_, key, data_source):
     start_time = time.time()
     res = search_index_for_value(res_index, query)
     query_time = "%.2f" % (time.time() - start_time)
+    print("Query time: %s" % query_time)
     returned_results = []
     if res.get("aggregations"):
         for bucket in (
@@ -382,7 +383,7 @@ def prepare_search_results(results, size=0):
             continue
         row = {}
         returned_results.append(row)
-        row["Data Source"]=res["data_source"]
+        row["Data Source"] = res["data_source"]
         row["Key"] = res["Attribute"]
         row["Value"] = res["Value"]
         row["Number of %ss" % resource] = res.get("items_in_the_bucket")
@@ -491,13 +492,16 @@ def query_cashed_bucket_part_value_keys(
         name = name.strip()
     value = adjust_value(value)
     if data_source and data_source.strip() and data_source.lower() != "all":
-        data_source = [itm.strip().lower() for itm in data_source.split(',')]
+        data_source = [itm.strip().lower() for itm in data_source.split(",")]
     else:
         data_source = get_data_sources()
 
     if resource != "all":
         query = key_part_values_buckets_template.substitute(
-            name=name, value=value, resource=resource, data_source=json.dumps(data_source)
+            name=name,
+            value=value,
+            resource=resource,
+            data_source=json.dumps(data_source),
         )
         res = search_index_for_values_get_all_buckets(es_index, query)
         returned_results = prepare_search_results_buckets(res)
@@ -510,7 +514,10 @@ def query_cashed_bucket_part_value_keys(
             if table == "image1":
                 continue
             query = key_part_values_buckets_template.substitute(
-                name=name, value=value, resource=table, data_source=json.dumps(data_source)
+                name=name,
+                value=value,
+                resource=table,
+                data_source=json.dumps(data_source),
             )
             res = search_index_for_values_get_all_buckets(es_index, query)
             returned_results[table] = prepare_search_results_buckets(res)
@@ -522,9 +529,9 @@ def query_cashed_bucket(
 ):
     # returns possible matches for a specific resource
     if data_source and data_source.strip() and data_source.lower() != "all":
-        data_source = [itm.strip().lower() for itm in data_source.split(',')]
+        data_source = [itm.strip().lower() for itm in data_source.split(",")]
     else:
-        data_source =get_data_sources()
+        data_source = get_data_sources()
 
     if name:
         name = name.strip()
@@ -564,9 +571,9 @@ def search_value_for_resource(
     value = adjust_value(value)
 
     if data_source and data_source.lower() != "all":
-        data_source = [itm.strip().lower() for itm in data_source.split(',')]
+        data_source = [itm.strip().lower() for itm in data_source.split(",")]
     else:
-        data_source=get_data_sources()
+        data_source = get_data_sources()
 
     if table_ != "all":
         query = resource_key_values_buckets_template.substitute(
@@ -737,7 +744,7 @@ def search_value_for_resource(
 )
 
 key_values_buckets_template_with_data_source = Template(
- """
+    """
 {"query":{"bool":{"must":[{"bool":{"must":{"match":{
 "resource.keyresource":"$resource"}}}},{"bool": {"must":
 {"match": {"data_source.keyvalue":$data_source}}}}]}}} """
@@ -774,7 +781,9 @@ def get_restircted_search_terms():
     return restricted_search_terms
 
 
-def get_resource_attributes(resource, data_source=None, mode=None, es_index="key_values_resource_cach"):
+def get_resource_attributes(
+    resource, data_source=None, mode=None, es_index="key_values_resource_cach"
+):
     """
     return the available attributes for one or all resources
     """
@@ -785,34 +794,35 @@ def get_resource_attributes(resource, data_source=None, mode=None, es_index="key
             you may remove it to return all the keys."
         )
     returned_results = []
-    if data_source and data_source.lower() !="all":
-       data_source = [itm.strip().lower() for itm in data_source.split(',')]
-    all_data_sources=get_data_sources()
+    if data_source and data_source.lower() != "all":
+        data_source = [itm.strip().lower() for itm in data_source.split(",")]
+    all_data_sources = get_data_sources()
     for data_s in all_data_sources:
-        if data_source and data_source !="all" and data_s.lower() not in data_source:
+        if data_source and data_source != "all" and data_s.lower() not in data_source:
             continue
         returned_results_ = {}
         returned_results_["data_source"] = data_s
         returned_results.append(returned_results_)
         if resource != "all":
-            query = key_values_buckets_template_with_data_source.substitute(resource=resource, data_source=json.dumps(data_s))
-            #else:
+            query = key_values_buckets_template_with_data_source.substitute(
+                resource=resource, data_source=json.dumps(data_s)
+            )
+            # else:
             #    query = key_values_buckets_template_2.substitute(resource=resource)
             res = connect_elasticsearch(
                 es_index, query
             )  # es.search(index=es_index, body=query)
 
-
             hits = res["hits"]["hits"]
             if len(hits) > 0:
                 returned_results_[resource] = hits[0]["_source"]["name"]
 
-
         else:
             for table in resource_elasticsearchindex:
-                query = key_values_buckets_template_with_data_source.substitute(resource=table,
-                                                                       data_source=json.dumps(data_s))
-                #else:
+                query = key_values_buckets_template_with_data_source.substitute(
+                    resource=table, data_source=json.dumps(data_s)
+                )
+                # else:
                 #    query = key_values_buckets_template_2.substitute(resource=table)
                 res = connect_elasticsearch(
                     es_index, query
@@ -829,7 +839,7 @@ def get_resource_attributes(resource, data_source=None, mode=None, es_index="key
                 search_terms = list(set(restricted_search_terms[k]) & set(val))
                 if len(search_terms) > 0:
                     restircted_resources[k] = search_terms
-        returned_results.append( restircted_resources)
+        returned_results.append(restircted_resources)
         if "project" in returned_results:
             returned_results_["project"].append("name")
 
@@ -911,10 +921,14 @@ def get_resource_names(resource, name=None, description=False, data_source=None)
     return returned_results
 
 
-def get_the_results(resource, name, description, data_source, es_index="key_values_resource_cach"):
+def get_the_results(
+    resource, name, description, data_source, es_index="key_values_resource_cach"
+):
     returned_results = {}
     if data_source:
-        query = key_values_buckets_template_with_data_source.substitute(resource=resource, data_source=data_source)
+        query = key_values_buckets_template_with_data_source.substitute(
+            resource=resource, data_source=data_source
+        )
     else:
         query = key_values_buckets_template_2.substitute(resource=resource)
     results_ = connect_elasticsearch(
@@ -959,11 +973,13 @@ def get_the_results(resource, name, description, data_source, es_index="key_valu
     return returned_results
 
 
-def get_container_values_for_key(table_, container_name, csv, ret_data_source=None, key=None):
+def get_container_values_for_key(
+    table_, container_name, csv, ret_data_source=None, key=None
+):
     returned_results = []
     pr_names = get_resource_names("all")
     if ret_data_source:
-        ret_data_source = [itm.strip().lower() for itm in ret_data_source.split(',')]
+        ret_data_source = [itm.strip().lower() for itm in ret_data_source.split(",")]
     for resourse, names_ in pr_names.items():
         for data_source, names in names_.items():
             if ret_data_source: