Update the app configuration and test to work with more than one data…

… source

configurations/app_config.yml

@@ -15,3 +15,11 @@ ELASTICSEARCH_BACKUP_FOLDER: "path/to/elasticsearch/backup/folder"
 verify_certs: False
 ELASTIC_PASSWORD: elasticsearch_user_password
 BASE_FOLDER: /etc/searchengine/
+DATA_SOURCES:
+    - name: omero1
+      DATABASE:
+          DATABASE_NAME: omero
+          DATABSE_USER: khaledk
+          DATABASE_PASSWORD: khaled
+          DATABASE_SERVER_URI: 192.168.0.190
+          DATABASE_PORT: 5432

configurations/configuration.py

@@ -26,15 +26,21 @@
 def load_configuration_variables_from_file(config):
     # loading application configuration variables from a file
     print("Injecting config variables from :%s" % app_config.INSTANCE_CONFIG)
-    with open(app_config.INSTANCE_CONFIG) as f:
-        cofg = yaml.load(f)
+    with open(app_config.INSTANCE_CONFIG, 'rt') as f:
+
+    #with open(app_config.INSTANCE_CONFIG) as f:
+        cofg = yaml.safe_load(f.read())
+
+
+    print (cofg)
     for x, y in cofg.items():
         setattr(config, x, y)
     if hasattr(config, "verify_certs"):
         try:
             verify_certs = json.load(config.verify_certs)
         except Exception as ex:
-            print(str(ex))
+            print (x)
+            print("Error %s"%str(ex))
             verify_certs = False
     else:
         verify_certs = False
@@ -53,43 +59,68 @@ def set_database_connection_variables(config):
     :param database: databse name
     :return:
     """
-    if hasattr(config, "DATABASE_PORT"):
-        address = config.DATABASE_SERVER_URI + ":%s" % app_config.DATABASE_PORT
-    else:
-        address = config.DATABASE_SERVER_URI
-    app_config.database_connector = ""
-    app_config.DATABASE_URI = "postgresql://%s:%s@%s/%s" % (
-        config.DATABASE_USER,
-        config.DATABASE_PASSWORD,
-        address,
-        config.DATABASE_NAME,
-    )
-
+    from omero_search_engine.database.database_connector import DatabaseConnector
 
-def update_config_file(updated_configuration):
+    config.database_connectors={}
+    for source in config.DATA_SOURCES:
+        if source.get("DATABASE").get("DATABASE_PORT"):
+            address = source.get("DATABASE").get("DATABASE_SERVER_URI") + ":%s" % source.get("DATABASE").get("DATABASE_PORT")
+        else:
+            address = source.get("DATABASE").get("DATABASE_SERVER_URI")
+        DATABASE_URI = "postgresql://%s:%s@%s/%s" % (
+            source.get("DATABASE").get("DATABASE_USER"),
+            source.get("DATABASE").get("DATABASE_PASSWORD"),
+            address,
+            source.get("DATABASE").get("DATABASE_NAME")
+        )
+        database_connector = DatabaseConnector(
+            source.get("DATABASE").get("DATABASE_NAME"), DATABASE_URI
+        )
+        config.database_connectors[source.get("name")]= database_connector
+
+
+def update_config_file(updated_configuration, configure_database=False):
     is_changed = False
     with open(app_config.INSTANCE_CONFIG) as f:
         configuration = yaml.load(f)
-    found = []
-    for key, value in updated_configuration.items():
-        if key in configuration:
-            if configuration[key] != value:
-                configuration[key] = value
-                is_changed = True
-                print("%s is Updated, new value is %s " % (key, value))
-            else:
-                found.append(key)
-    if len(found) != len(updated_configuration):
+    if not configure_database:
+        found = []
         for key, value in updated_configuration.items():
-            if key not in found:
-                configuration[key] = value
-                print("%s value is added with value %s " % (key, value))
-                is_changed = True
+            if key in configuration:
+                if configuration[key] != value:
+                    configuration[key] = value
+                    is_changed = True
+                    print("%s is Updated, new value is %s " % (key, value))
+                else:
+                    found.append(key)
+        if len(found) != len(updated_configuration):
+            for key, value in updated_configuration.items():
+                if key not in found:
+                    configuration[key] = value
+                    print("%s value is added with value %s " % (key, value))
+                    is_changed = True
+    else:
+        is_changed = config_database(configuration, updated_configuration)
 
     if is_changed:
         with open(app_config.INSTANCE_CONFIG, "w") as f:
             yaml.dump(configuration, f)
 
+def config_database(configuration, updated_configuration):
+    for data_source in configuration.get("DATA_SOURCES"):
+        changed=False
+        Found = False
+        if data_source["name"].lower()==updated_configuration["name"].lower():
+            Found = True
+            for k, v in updated_configuration["DATABASE"].items():
+                if data_source["DATABASE"][k] !=v:
+                    data_source["DATABASE"][k]=v
+                    changed=True
+            break
+    if not Found:
+        configuration.get("DATA_SOURCES").append(updated_configuration)
+        changed = True
+    return changed
 
 class app_config(object):
     # the configuration can be loadd from yml file later

manage.py

@@ -127,12 +127,17 @@ def restore_postgresql_database():
     "--resource",
     help="resource name, creating all the indexes for all the resources is the default",  # noqa
 )
+@manager.option(
+    "-s",
+    "--source",
+    help="data source name, ndexeing all the data sources is the default",  # noqa
+)
 @manager.option(
     "-b",
     "--backup",
     help="if True, backup will be called ",  # noqa
 )
-def get_index_data_from_database(resource="all", backup="True"):
+def get_index_data_from_database(resource="all", source="all", backup="True"):
     """
     insert data in Elasticsearch index for each resource
     It gets the data from postgres database server
@@ -145,21 +150,27 @@ def get_index_data_from_database(resource="all", backup="True"):
         save_key_value_buckets,
     )
     import json
-
     backup = json.loads(backup.lower())
-    if resource != "all":
-        sql_st = sqls_resources.get(resource)
-        if not sql_st:
-            return
-        get_insert_data_to_index(sql_st, resource)
-    else:
+    for data_source in search_omero_app.config.database_connectors.keys():
+        if source.lower()!="all" and data_source.lower() != source.lower():
+            continue
+        #if resource != "all":
+        #    sql_st = sqls_resources.get(resource)
+        #    if not sql_st:
+        #        return
+       #     get_insert_data_to_index(sql_st, resource)
+       # else:
         for res, sql_st in sqls_resources.items():
-            get_insert_data_to_index(sql_st, res)
+            if resource.lower()!="all" and resource.lower() != res.lower():
+                continue
+            get_insert_data_to_index(sql_st, res, data_source)
+
+
         save_key_value_buckets(
-            resource_table_=None, re_create_index=True, only_values=False
+            resource_table_=None, data_source=data_source, re_create_index=True ,only_values=False
         )
         # validat ethe indexing
-        test_indexing_search_query(deep_check=False, check_studies=True)
+        test_indexing_search_query(source=data_source, deep_check=False, check_studies=True)
 
     # backup the index data
     if backup:
@@ -173,14 +184,20 @@ def get_index_data_from_database(resource="all", backup="True"):
 @manager.option("-d", "--database", help="database name")
 @manager.option("-n", "--name", help="database usernname")
 @manager.option("-p", "--password", help="database username password")
+@manager.option("-w", "--working_data_source", help="data source")
 def set_database_configuration(
-    url=None, server_port_number=None, database=None, name=None, password=None
+    working_data_source=None,url=None, server_port_number=None, database=None, name=None, password=None
 ):
+    if not  working_data_source:
+        print ("Data source is required to process")
     database_attrs = {}
-    if url:
-        database_attrs["DATABASE_SERVER_URI"] = url
+    databse_config={}
+    databse_config["name"]=working_data_source
+    databse_config["DATABASE"]=database_attrs
     if database:
         database_attrs["DATABASE_NAME"] = database
+    if url:
+        database_attrs["DATABASE_SERVER_URI"] = url
     if name:
         database_attrs["DATABASE_USER"] = name
     if password:
@@ -189,7 +206,7 @@ def set_database_configuration(
         database_attrs["DATABASE_PORT"] = server_port_number
 
     if len(database_attrs) > 0:
-        update_config_file(database_attrs)
+        update_config_file(databse_config, configure_database=True)
     else:
         search_omero_app.logger.info(
             "At least one database attribute\
@@ -319,8 +336,13 @@ def cache_key_value_index(resource=None, create_index=None, only_values=None):
     "--deep_check",
     help="compare all the images from both search engine and database server, default is False so it will compare the number of images and the first searchengine page",  # noqa
 )
+@manager.option(
+    "-s",
+    "--source",
+    help="data source name, ndexeing all the data sources is the default",  # noqa
+)
 def test_indexing_search_query(
-    json_file="app_data/test_index_data.json", deep_check=False, check_studies=False
+    json_file="app_data/test_index_data.json", source=None, deep_check=False, check_studies=False
 ):
     """
     test the indexing and the searchengine query functions
@@ -338,12 +360,15 @@ def test_indexing_search_query(
         get_omero_stats,
         get_no_images_sql_containers,
     )
+    if not source:
+        print ("Data source is required to process ")
+        return
 
-    validate_queries(json_file, deep_check)
+    validate_queries(json_file, source, deep_check)
     if check_studies:
         test_no_images()
     get_omero_stats()
-    get_no_images_sql_containers()
+    get_no_images_sql_containers(data_source=source)
 
 
 @manager.command

omero_search_engine/__init__.py

@@ -62,16 +62,16 @@ def create_app(config_name="development"):
     app_config = configLooader.get(config_name)
     load_configuration_variables_from_file(app_config)
     set_database_connection_variables(app_config)
-    database_connector = DatabaseConnector(
-        app_config.DATABASE_NAME, app_config.DATABASE_URI
-    )
+    print ("config.database_connectors::::::", app_config.database_connectors)
+    #atabase_connector = DatabaseConnector(
+    #   app_config.DATABASE_NAME, app_config.DATABASE_URI
+    #
     search_omero_app.config.from_object(app_config)
     search_omero_app.app_context()
     search_omero_app.app_context().push()
     search_omero_app.app_context()
     search_omero_app.app_context().push()
     ELASTIC_PASSWORD = app_config.ELASTIC_PASSWORD
-
     es_connector = Elasticsearch(
         app_config.ELASTICSEARCH_URL.split(","),
         verify_certs=app_config.verify_certs,
@@ -82,8 +82,8 @@ def create_app(config_name="development"):
         scheme="https",
         http_auth=("elastic", ELASTIC_PASSWORD),
     )
-
-    search_omero_app.config["database_connector"] = database_connector
+    search_omero_app.config.database_connectors= app_config.database_connectors
+    print(search_omero_app.config.database_connectors)
     search_omero_app.config["es_connector"] = es_connector
     log_folder = os.path.join(os.path.expanduser("~"), "logs")
     if not os.path.exists(log_folder):
@@ -117,7 +117,6 @@ def create_app(config_name="development"):
     resources_routers_blueprint_v1, url_prefix="/api/v1/resources"
 )
 
-
 # add it to account for CORS
 @search_omero_app.after_request
 def after_request(response):

omero_search_engine/api/v1/resources/query_handler.py

@@ -108,9 +108,9 @@ def adjust_resource(self):
                     ac_value = check_get_names(
                         self.value, self.resource, self.name, True
                     )
-                    if len(ac_value) == 1:
+                    if ac_value and len(ac_value) == 1:
                         self.value = ac_value[0]
-                    elif len(ac_value) == 0:
+                    elif not ac_value  or len(ac_value) == 0:
                         self.value = -1
                     else:
                         self.value = ac_value

omero_search_engine/api/v1/resources/resource_analyser.py

@@ -790,8 +790,13 @@ def get_the_results(resource, name, description, es_index="key_values_resource_c
                     and name.lower() in item.get("description").lower()
                 )
             ]
-        else:
+        elif "resourcename" in hits[0]["_source"]:
+            print("==================================")
+            print ("========>>>>",hits[0]["_source"])
+            print ("==================================")
             returned_results = [item for item in hits[0]["_source"]["resourcename"]]
+        else:
+            return returned_results
 
     # remove container description from the results,
     # should be added again later after cleaning up the description