Support indexing from multiple data sources andd fix unit tests and q…

…ueries

configurations/configuration.py

@@ -26,21 +26,17 @@
 def load_configuration_variables_from_file(config):
     # loading application configuration variables from a file
     print("Injecting config variables from :%s" % app_config.INSTANCE_CONFIG)
-    with open(app_config.INSTANCE_CONFIG, 'rt') as f:
+    with open(app_config.INSTANCE_CONFIG, "rt") as f:
 
-    #with open(app_config.INSTANCE_CONFIG) as f:
+        # with open(app_config.INSTANCE_CONFIG) as f:
         cofg = yaml.safe_load(f.read())
-
-
-    print (cofg)
     for x, y in cofg.items():
         setattr(config, x, y)
     if hasattr(config, "verify_certs"):
         try:
             verify_certs = json.load(config.verify_certs)
         except Exception as ex:
-            print (x)
-            print("Error %s"%str(ex))
+            print("Error %s" % str(ex))
             verify_certs = False
     else:
         verify_certs = False
@@ -61,22 +57,24 @@ def set_database_connection_variables(config):
     """
     from omero_search_engine.database.database_connector import DatabaseConnector
 
-    config.database_connectors={}
+    config.database_connectors = {}
     for source in config.DATA_SOURCES:
         if source.get("DATABASE").get("DATABASE_PORT"):
-            address = source.get("DATABASE").get("DATABASE_SERVER_URI") + ":%s" % source.get("DATABASE").get("DATABASE_PORT")
+            address = source.get("DATABASE").get(
+                "DATABASE_SERVER_URI"
+            ) + ":%s" % source.get("DATABASE").get("DATABASE_PORT")
         else:
             address = source.get("DATABASE").get("DATABASE_SERVER_URI")
         DATABASE_URI = "postgresql://%s:%s@%s/%s" % (
             source.get("DATABASE").get("DATABASE_USER"),
             source.get("DATABASE").get("DATABASE_PASSWORD"),
             address,
-            source.get("DATABASE").get("DATABASE_NAME")
+            source.get("DATABASE").get("DATABASE_NAME"),
         )
         database_connector = DatabaseConnector(
             source.get("DATABASE").get("DATABASE_NAME"), DATABASE_URI
         )
-        config.database_connectors[source.get("name")]= database_connector
+        config.database_connectors[source.get("name")] = database_connector
 
 
 def update_config_file(updated_configuration, configure_database=False):
@@ -106,22 +104,24 @@ def update_config_file(updated_configuration, configure_database=False):
         with open(app_config.INSTANCE_CONFIG, "w") as f:
             yaml.dump(configuration, f)
 
+
 def config_database(configuration, updated_configuration):
     for data_source in configuration.get("DATA_SOURCES"):
-        changed=False
+        changed = False
         Found = False
-        if data_source["name"].lower()==updated_configuration["name"].lower():
+        if data_source["name"].lower() == updated_configuration["name"].lower():
             Found = True
             for k, v in updated_configuration["DATABASE"].items():
-                if data_source["DATABASE"][k] !=v:
-                    data_source["DATABASE"][k]=v
-                    changed=True
+                if data_source["DATABASE"][k] != v:
+                    data_source["DATABASE"][k] = v
+                    changed = True
             break
     if not Found:
         configuration.get("DATA_SOURCES").append(updated_configuration)
         changed = True
     return changed
 
+
 class app_config(object):
     # the configuration can be loadd from yml file later
     home_folder = os.path.expanduser("~")

manage.py

@@ -32,9 +32,7 @@ def show_saved_indices():
     )
 
     all_indexes = get_all_indexes()
-    for index in all_indexes:
-        print("Index: ==>>>", index)
-    # return (all_indexes)
+    return all_indexes
 
 
 @manager.command
@@ -118,6 +116,8 @@ def sql_results_to_panda():
 def restore_postgresql_database():
     from omero_search_engine.database.utils import restore_database
 
+    print("HI")
+
     restore_database()
 
 
@@ -150,31 +150,47 @@ def get_index_data_from_database(resource="all", source="all", backup="True"):
         save_key_value_buckets,
     )
     import json
+
     backup = json.loads(backup.lower())
+    if not source:
+        print("Data source is required to process")
+        return
+    elif source == "all":
+        clean_index = True
+
+    else:
+        clean_index = False
+
     for data_source in search_omero_app.config.database_connectors.keys():
-        if source.lower()!="all" and data_source.lower() != source.lower():
+        if source.lower() != "all" and data_source.lower() != source.lower():
             continue
-        #if resource != "all":
+        # if resource != "all":
         #    sql_st = sqls_resources.get(resource)
         #    if not sql_st:
         #        return
-       #     get_insert_data_to_index(sql_st, resource)
-       # else:
+        #     get_insert_data_to_index(sql_st, resource)
+        # else:
         for res, sql_st in sqls_resources.items():
-            if resource.lower()!="all" and resource.lower() != res.lower():
+            if resource.lower() != "all" and resource.lower() != res.lower():
                 continue
-            get_insert_data_to_index(sql_st, res, data_source)
-
+            get_insert_data_to_index(sql_st, res, data_source, clean_index)
 
         save_key_value_buckets(
-            resource_table_=None, data_source=data_source, re_create_index=True ,only_values=False
+            resource_table_=None,
+            data_source=data_source,
+            clean_index=clean_index,
+            only_values=False,
         )
+
+        if clean_index:
+            clean_index = False
         # validat ethe indexing
-        test_indexing_search_query(source=data_source, deep_check=False, check_studies=True)
+        # test_indexing_search_query(source=data_source, deep_check=False,
+        # check_studies=True)
 
     # backup the index data
-    if backup:
-        backup_elasticsearch_data()
+    # if backup:
+    #    backup_elasticsearch_data()
 
 
 # set configurations
@@ -186,14 +202,19 @@ def get_index_data_from_database(resource="all", source="all", backup="True"):
 @manager.option("-p", "--password", help="database username password")
 @manager.option("-w", "--working_data_source", help="data source")
 def set_database_configuration(
-    working_data_source=None,url=None, server_port_number=None, database=None, name=None, password=None
+    working_data_source=None,
+    url=None,
+    server_port_number=None,
+    database=None,
+    name=None,
+    password=None,
 ):
-    if not  working_data_source:
-        print ("Data source is required to process")
+    if not working_data_source:
+        print("Data source is required to process")
     database_attrs = {}
-    databse_config={}
-    databse_config["name"]=working_data_source
-    databse_config["DATABASE"]=database_attrs
+    databse_config = {}
+    databse_config["name"] = working_data_source
+    databse_config["DATABASE"] = database_attrs
     if database:
         database_attrs["DATABASE_NAME"] = database
     if url:
@@ -342,7 +363,10 @@ def cache_key_value_index(resource=None, create_index=None, only_values=None):
     help="data source name, ndexeing all the data sources is the default",  # noqa
 )
 def test_indexing_search_query(
-    json_file="app_data/test_index_data.json", source=None, deep_check=False, check_studies=False
+    json_file="app_data/test_index_data.json",
+    source=None,
+    deep_check=False,
+    check_studies=False,
 ):
     """
     test the indexing and the searchengine query functions
@@ -360,8 +384,9 @@ def test_indexing_search_query(
         get_omero_stats,
         get_no_images_sql_containers,
     )
+
     if not source:
-        print ("Data source is required to process ")
+        print("Data source is required to process ")
         return
 
     validate_queries(json_file, source, deep_check)

omero_search_engine/__init__.py

@@ -22,7 +22,8 @@
 import logging
 from elasticsearch import Elasticsearch
 from flasgger import Swagger, LazyString, LazyJSONEncoder
-from omero_search_engine.database.database_connector import DatabaseConnector
+
+# from omero_search_engine.database.database_connector import DatabaseConnector
 from configurations.configuration import (
     configLooader,
     load_configuration_variables_from_file,
@@ -62,8 +63,7 @@ def create_app(config_name="development"):
     app_config = configLooader.get(config_name)
     load_configuration_variables_from_file(app_config)
     set_database_connection_variables(app_config)
-    print ("config.database_connectors::::::", app_config.database_connectors)
-    #atabase_connector = DatabaseConnector(
+    # atabase_connector = DatabaseConnector(
     #   app_config.DATABASE_NAME, app_config.DATABASE_URI
     #
     search_omero_app.config.from_object(app_config)
@@ -82,7 +82,7 @@ def create_app(config_name="development"):
         scheme="https",
         http_auth=("elastic", ELASTIC_PASSWORD),
     )
-    search_omero_app.config.database_connectors= app_config.database_connectors
+    search_omero_app.config.database_connectors = app_config.database_connectors
     print(search_omero_app.config.database_connectors)
     search_omero_app.config["es_connector"] = es_connector
     log_folder = os.path.join(os.path.expanduser("~"), "logs")
@@ -117,6 +117,7 @@ def create_app(config_name="development"):
     resources_routers_blueprint_v1, url_prefix="/api/v1/resources"
 )
 
+
 # add it to account for CORS
 @search_omero_app.after_request
 def after_request(response):

omero_search_engine/api/v1/resources/query_handler.py

@@ -110,7 +110,7 @@ def adjust_resource(self):
                     )
                     if ac_value and len(ac_value) == 1:
                         self.value = ac_value[0]
-                    elif not ac_value  or len(ac_value) == 0:
+                    elif not ac_value or len(ac_value) == 0:
                         self.value = -1
                     else:
                         self.value = ac_value