diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 06eba1d..b9551f7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,7 @@ on: branches: [ main ] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 services: postgres: image: postgres:16 @@ -22,19 +22,6 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - postgres_2: - image: postgres:16 - env: - POSTGRES_USER: postgress - POSTGRES_PASSWORD: passwprd - POSTGRES_DB: omero_train - ports: - - 5433/tcp - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 elasticsearch: image: elasticsearch:8.8.1 @@ -58,21 +45,17 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt; # Configure database url - python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero1 -d omero -b omero.pgdump + #python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd + python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero1 -d omero -b omero.pgdump # configure elasticsearch python manage.py set_elasticsearch_configuration -e localhost:${{ job.services.elasticsearch.ports[9200] }} # download and extract the database backup file - wget https://downloads.openmicroscopy.org/images/omero_db_searchengine.zip -P app_data - unzip app_data/omero_db_searchengine.zip -d app_data/ - wget https://github.com/khaledk2/ice-archh-64/releases/download/new_re_db/omero_train.zip -P app_data - unzip app_data/omero_train.zip -d app_data/ + wget https://downloads.openmicroscopy.org/images/omero_db_searchengine.zip -P app_data + unzip app_data/omero_db_searchengine.zip -d app_data/ # run restore omero database - python manage.py restore_postgresql_database + python manage.py restore_postgresql_database # run indexing indexing - python manage.py get_index_data_from_database -b False - # set up the seond database - python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres_2.ports[5433] }} -n postgress -p passwprd -w omero_train -d omero_train -b omero_train.pgdump - #python manage.py restore_postgresql_database -s omero1 + python manage.py get_index_data_from_database -b False # run tests python -m unittest discover -s unit_tests upload: diff --git a/configurations/app_config.yml b/configurations/app_config.yml index c5a00db..a29d8af 100644 --- a/configurations/app_config.yml +++ b/configurations/app_config.yml @@ -4,12 +4,13 @@ ASYNCHRONOUS_SEARCH : True ELASTICSEARCH_URL : "https://localhost:9200" IDR_TEST_FILE_URL : "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv" PAGE_SIZE : 1000 -CACHE_ROWS : 10000 -MAX_RETUNED_ITEMS : 1700000 +CACHE_ROWS : 1000 +MAX_RETUNED_ITEMS : 1000 ELASTICSEARCH_BACKUP_FOLDER: "path/to/elasticsearch/backup/folder" verify_certs: False ELASTIC_PASSWORD: elasticsearch_user_password BASE_FOLDER: /etc/searchengine/ +#NO_PROCESSES: 1 DATA_SOURCES: - name: omero1 DATABASE: diff --git a/manage.py b/manage.py index ea84c18..4b03272 100644 --- a/manage.py +++ b/manage.py @@ -123,7 +123,6 @@ def restore_postgresql_database(source="all"): restore_database(source) - @manager.command @manager.option( "-r", @@ -176,24 +175,24 @@ def get_index_data_from_database(resource="all", source="all", backup="True"): if resource.lower() != "all" and resource.lower() != res.lower(): continue get_insert_data_to_index(sql_st, res, data_source, clean_index) - save_key_value_buckets( resource_table_=None, data_source=data_source, clean_index=clean_index, only_values=False, ) - + print("!Done for data_source: %s from %s" % (data_source, search_omero_app.config.database_connectors.keys())) if clean_index: clean_index = False + # validat ethe indexing #test_indexing_search_query( # source=data_source, deep_check=False, check_studies=True #) - # backup the index data - # if backup: - # backup_elasticsearch_data() + #backup the index data + if backup: + backup_elasticsearch_data() # set configurations @@ -334,6 +333,12 @@ def set_no_processes(no_processes=None): @manager.command +@manager.option( + "-d", + "--data_source", + help="data source name, the default is all", # noqa +) + @manager.option( "-r", "--resource", @@ -345,7 +350,7 @@ def set_no_processes(no_processes=None): help="creating the elastic search index if set to True", # noqa ) @manager.option("-o", "--only_values", help="creating cached values only ") -def cache_key_value_index(resource=None, create_index=None, only_values=None): +def cache_key_value_index(resource=None, data_source='all',create_index=None, only_values=None): """ Cache the value bucket for each value for each resource """ @@ -353,7 +358,7 @@ def cache_key_value_index(resource=None, create_index=None, only_values=None): save_key_value_buckets, ) - save_key_value_buckets(resource, create_index, only_values) + save_key_value_buckets(resource,data_source ,create_index, only_values) @manager.command diff --git a/omero_search_engine/__init__.py b/omero_search_engine/__init__.py index 40ebca6..b617199 100644 --- a/omero_search_engine/__init__.py +++ b/omero_search_engine/__init__.py @@ -82,6 +82,7 @@ def create_app(config_name="development"): scheme="https", http_auth=("elastic", ELASTIC_PASSWORD), ) + search_omero_app.config.database_connectors = app_config.database_connectors print(search_omero_app.config.database_connectors) search_omero_app.config["es_connector"] = es_connector @@ -104,6 +105,7 @@ def create_app(config_name="development"): search_omero_app.logger.setLevel(logging.INFO) search_omero_app.logger.info("app assistant startup") + return search_omero_app diff --git a/omero_search_engine/api/v1/resources/query_handler.py b/omero_search_engine/api/v1/resources/query_handler.py index 26a50f4..225f6c0 100644 --- a/omero_search_engine/api/v1/resources/query_handler.py +++ b/omero_search_engine/api/v1/resources/query_handler.py @@ -459,13 +459,13 @@ def search_query( search_omero_app.logger.info( "-------------------------------------------------" ) # noqa - search_omero_app.logger.info("1: %s"%query) - search_omero_app.logger.info("2: %s"%main_attributes) + search_omero_app.logger.info(query) + search_omero_app.logger.info(main_attributes) search_omero_app.logger.info(resource) search_omero_app.logger.info( "-------------------------------------------------" ) # noqa - search_omero_app.logger.info(("1... %s, 2....%s") % (resource, query)) + search_omero_app.logger.info(("%s, %s") % (resource, query)) if not main_attributes: q_data = {"query": {"query_details": query}} elif resource == "image": @@ -493,6 +493,9 @@ def search_query( ress = search_resource_annotation( resource, q_data.get("query"), return_containers=return_containers, data_source=data_source ) + if type (ress) is str: + return ress + ress["Error"] = "none" return ress except Exception as ex: diff --git a/omero_search_engine/api/v1/resources/resource_analyser.py b/omero_search_engine/api/v1/resources/resource_analyser.py index a94543c..3a681d0 100644 --- a/omero_search_engine/api/v1/resources/resource_analyser.py +++ b/omero_search_engine/api/v1/resources/resource_analyser.py @@ -41,7 +41,7 @@ }}}}}}}}""" ) key_number_search_template = Template( - """ + r""" { "size":0, "query":{ "bool" : {"must": { @@ -261,7 +261,11 @@ def get_number_of_buckets(key, data_source, res_index): def get_all_values_for_a_key(table_, data_source, key): res_index = resource_elasticsearchindex.get(table_) query = key_number_search_template.substitute(key=key, data_source=data_source) - res = search_index_for_value(res_index, query) + try: + res = search_index_for_value(res_index, query) + except Exception as ex: + print("Query: %s Error: %s"%(query,str(ex))) + raise ex number_of_buckets = ( res.get("aggregations") .get("value_search") @@ -333,7 +337,6 @@ def get_values_for_a_key(table_, key, data_source): start_time = time.time() res = search_index_for_value(res_index, query) query_time = "%.2f" % (time.time() - start_time) - print("TIME ...", query_time) returned_results = [] if res.get("aggregations"): for bucket in ( @@ -889,7 +892,7 @@ def get_resource_attribute_values( return returned_results -def get_resource_names(resource, name=None, description=False): +def get_resource_names(resource, name=None, description=False, data_source=None): """ return resources names attributes It works for projects and screens but can be extended. @@ -898,21 +901,22 @@ def get_resource_names(resource, name=None, description=False): return build_error_message( "This release does not support search by description." ) - if resource != "all": - returned_results = get_the_results(resource, name, description) + returned_results = get_the_results(resource, name, description, data_source) else: returned_results = {} ress = ["project", "screen"] for res in ress: - returned_results[res] = get_the_results(res, name, description) - + returned_results[res] = get_the_results(res, name, description, data_source) return returned_results -def get_the_results(resource, name, description, es_index="key_values_resource_cach"): +def get_the_results(resource, name, description, data_source, es_index="key_values_resource_cach"): returned_results = {} - query = key_values_buckets_template_2.substitute(resource=resource) + if data_source: + query = key_values_buckets_template_with_data_source.substitute(resource=resource, data_source=data_source) + else: + query = key_values_buckets_template_2.substitute(resource=resource) results_ = connect_elasticsearch( es_index, query ) # .search(index=es_index, body=query) diff --git a/omero_search_engine/api/v1/resources/swagger_docs/getresourcenames.yml b/omero_search_engine/api/v1/resources/swagger_docs/getresourcenames.yml index b23f9e6..07b6e10 100644 --- a/omero_search_engine/api/v1/resources/swagger_docs/getresourcenames.yml +++ b/omero_search_engine/api/v1/resources/swagger_docs/getresourcenames.yml @@ -12,6 +12,11 @@ parameters: in: query type: string required: false + - name: data_source + in: query + type: string + required: false + description: If it is provided, it will return the search results for a specific data source, otherwise it will return the results from all the data sources definitions: names: type: array diff --git a/omero_search_engine/api/v1/resources/urls.py b/omero_search_engine/api/v1/resources/urls.py index d2966d7..2219b8a 100644 --- a/omero_search_engine/api/v1/resources/urls.py +++ b/omero_search_engine/api/v1/resources/urls.py @@ -25,6 +25,7 @@ build_error_message, adjust_query_for_container, get_data_sources, + check_empty_string, ) from omero_search_engine.api.v1.resources.resource_analyser import ( search_value_for_resource, @@ -91,6 +92,8 @@ def search_resource_page(resource_table): pagination_dict = data.get("pagination") return_containers = data.get("return_containers") data_source = request.args.get("data_source") + if data_source: + data_source=data_source.strip() if return_containers: return_containers = json.loads(return_containers.lower()) @@ -172,6 +175,8 @@ def search_resource(resource_table): if validation_results == "OK": return_containers = request.args.get("return_containers") data_source = request.args.get("data_source") + if data_source: + data_source = data_source.strip() if return_containers: return_containers = json.loads(return_containers.lower()) @@ -190,6 +195,8 @@ def get_values_using_value(resource_table): """ value = request.args.get("value") data_source = request.args.get("data_source") + if data_source: + data_source = data_source.strip() if not value: return jsonify( build_error_message("Error: {error}".format(error="No value is provided ")) @@ -255,6 +262,8 @@ def search_values_for_a_key(resource_table): # if it sets to true, a CSV file content will be sent instead of dict csv = request.args.get("csv") data_source = request.args.get("data_source") + if data_source: + data_source = data_source.strip() if csv: try: csv = json.loads(csv.lower()) @@ -276,6 +285,8 @@ def get_resource_keys(resource_table): """ mode = request.args.get("mode") data_source = request.args.get("data_source") + if data_source: + data_source = data_source.strip() resource_keys = get_resource_attributes(resource_table, data_source=data_source, mode=mode) return jsonify(resource_keys) @@ -315,6 +326,11 @@ def get_resource_names_(resource_table): value = request.args.get("value") description = request.args.get("use_description") + data_source = request.args.get("data_source") + data_source=check_empty_string(data_source) + if data_source: + data_source=data_source.strip(",") + data_source=json.dumps(data_source) if description: if description.lower() in ["true", "false"]: description = json.loads(description.lower()) @@ -322,7 +338,7 @@ def get_resource_names_(resource_table): description = True else: description = False - return jsonify(get_resource_names(resource_table, value, description)) + return jsonify(get_resource_names(resource_table, value, description, data_source)) @resources.route("/submitquery/containers/", methods=["POST"]) @@ -339,6 +355,8 @@ def submit_query_return_containers(): adjust_query_for_container(query) return_columns = request.args.get("return_columns") data_source = request.args.get("data_source") + if data_source: + data_source = data_source.strip() if return_columns: try: return_columns = json.loads(return_columns.lower()) @@ -367,6 +385,8 @@ def submit_query(): adjust_query_for_container(query) return_columns = request.args.get("return_columns") data_source = request.args.get("data_source") + if data_source: + data_source = data_source.strip() if return_columns: try: return_columns = json.loads(return_columns.lower()) @@ -392,6 +412,7 @@ def search(resource_table): operator = request.args.get("operator") bookmark = request.args.get("bookmark") data_source = request.args.get("data_source") + data_source=check_empty_string(data_source) return_containers = request.args.get("return_containers") if return_containers: return_containers = json.loads(return_containers.lower()) diff --git a/omero_search_engine/api/v1/resources/utils.py b/omero_search_engine/api/v1/resources/utils.py index e834059..b11468f 100644 --- a/omero_search_engine/api/v1/resources/utils.py +++ b/omero_search_engine/api/v1/resources/utils.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import copy +import logging import os import sys import json @@ -1014,34 +1015,33 @@ def search_index_using_search_after( add_paination = False else: add_paination = True + if not data_source: + data_source = get_data_sources() es = search_omero_app.config.get("es_connector") if return_containers: ##### - if data_source: - if type(data_source) is str: - data_source = [itm.strip().lower() for itm in data_source.split(',')] - for data_s in data_source: - query2 = copy.deepcopy(query) - del query2["query"]["bool"]["must"][0] - main_dd = main_attribute_query_in_template.substitute( - attribute="data_source", - value=json.dumps([data_s]), - ) - #query["query"]["bool"]["must"][0] = json.loads(main_dd) - query2["query"]["bool"]["must"].append(json.loads(main_dd)) - res = es.search(index=e_index, body=query2) - if len(res["hits"]["hits"]) == 0: - search_omero_app.logger.info("No result is found") - continue - keys_counts = res["aggregations"]["key_count"]["buckets"] - idrs = [] - for ek in keys_counts: - idrs.append(ek["key"]) - res_res = get_studies_titles(ek["key"], ret_type, data_source) - res_res["image count"] = ek["doc_count"] - if data_source: - res_res["data_source"] =data_s - returned_results.append(res_res) + for data_s in data_source: + query2 = copy.deepcopy(query) + del query2["query"]["bool"]["must"][0] + main_dd = main_attribute_query_in_template.substitute( + attribute="data_source", + value=json.dumps([data_s]), + ) + #query["query"]["bool"]["must"][0] = json.loads(main_dd) + query2["query"]["bool"]["must"].append(json.loads(main_dd)) + res = es.search(index=e_index, body=query2) + if len(res["hits"]["hits"]) == 0: + search_omero_app.logger.info("No result is found") + continue + keys_counts = res["aggregations"]["key_count"]["buckets"] + idrs = [] + for ek in keys_counts: + idrs.append(ek["key"]) + res_res = get_studies_titles(ek["key"], ret_type, data_source) + res_res["image count"] = ek["doc_count"] + if data_source: + res_res["data_source"] =data_s + returned_results.append(res_res) return returned_results page_size = search_omero_app.config.get("PAGE_SIZE") @@ -1137,10 +1137,11 @@ def search_resource_annotation( or len(query_details) == 0 or isinstance(query_details, str) ): + logging.info("Error ") return build_error_message( "{query} is not a valid query".format(query=query) ) - if data_source and data_source.lower() != "all": + if data_source and data_source.lower() != "all": data_sources=get_data_sources() data_source = [itm.strip() for itm in data_source.split(',')] for data_s in data_source: @@ -1174,7 +1175,7 @@ def search_resource_annotation( if isinstance(query_string, dict): return query_string - search_omero_app.logger.info("Query %s" % query_string) + #search_omero_app.logger.info("Query %s" % query_string) query = json.loads(query_string, strict=False) raw_query_to_send_back = json.loads(query_string, strict=False) @@ -1316,3 +1317,8 @@ def get_data_sources(): for data_source in search_omero_app.config.database_connectors.keys(): data_sources.append(data_source) return data_sources + +def check_empty_string(string_to_check): + if string_to_check: + string_to_check=string_to_check.strip() + return string_to_check diff --git a/omero_search_engine/cache_functions/elasticsearch/transform_data.py b/omero_search_engine/cache_functions/elasticsearch/transform_data.py index 7b04fb6..58de101 100644 --- a/omero_search_engine/cache_functions/elasticsearch/transform_data.py +++ b/omero_search_engine/cache_functions/elasticsearch/transform_data.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import logging # Copyright (C) 2022 University of Dundee & Open Microscopy Environment. # All rights reserved. @@ -146,6 +147,7 @@ def delete_es_index(es_index): search_omero_app.logger.info("ERROR:" + error["root_cause"]) search_omero_app.logger.info("TYPE:" + error["type"]) return False + # print out the response: search_omero_app.logger.info("\nresponse:%s" % str(response)) else: @@ -153,6 +155,7 @@ def delete_es_index(es_index): return False return True + def delete_data_from_index(resource): if resource_elasticsearchindex.get(resource) and resource != "all": es_index = resource_elasticsearchindex[resource] @@ -529,7 +532,7 @@ def processor_work(lock, global_counter, val): lock.acquire() global_counter.value += 1 except Exception as ex: - print("Error is %s" % ex) + print("Error %s" % ex) raise ex finally: lock.release() @@ -585,6 +588,7 @@ def insert_resource_data_from_df(df, resource, data_source, lock=None): actions.append({"_index": es_index, "_source": record}) # , es = search_omero_app.config.get("es_connector") + #logging.getLogger("elasticsearch").setLevel(logging.ERROR) search_omero_app.logger.info("Pushing the data to the Elasticsearch") try: lock.acquire() @@ -728,6 +732,7 @@ def save_key_value_buckets( push_keys_cache_index( resource_keys, resource_table, data_source, es_index_2, name_results ) + logging.info(type(resource_keys), type(resource_table), es_index_2, type(name_results)) if only_values: continue search_omero_app.logger.info( @@ -756,7 +761,7 @@ def save_key_value_buckets( counter_val = manager.Value("i", 0) func = partial(save_key_value_buckets_process, lock, counter_val) res = pool.map(func, vals) - print(res) + search_omero_app.logger.info(res) finally: pool.close() @@ -803,8 +808,8 @@ def save_key_value_buckets_process(lock, global_counter, vals): search_omero_app.logger.info(helpers.bulk(es, actions)) except Exception as e: search_omero_app.logger.info("Error: %s" % str(e)) + #raise e raise e - # raise e finally: lock.release() except Exception as e: diff --git a/omero_search_engine/database/utils.py b/omero_search_engine/database/utils.py index 1d0f017..220a598 100644 --- a/omero_search_engine/database/utils.py +++ b/omero_search_engine/database/utils.py @@ -46,6 +46,29 @@ def restore_database(source): ): continue backup_filename = os.path.join(mm, "app_data/%s"%data_source.get("DATABASE").get("DATABASE_BACKUP_FILE")) + + create_database_comand = "psql --username %s --host %s --port %s -c 'create database %s'" % ( + data_source.get("DATABASE").get("DATABASE_USER"), + data_source.get("DATABASE").get("DATABASE_SERVER_URI"), + data_source.get("DATABASE").get("DATABASE_PORT"), + data_source.get("DATABASE").get("DATABASE_NAME") + ) + + print("create_database_comand: %s" % create_database_comand) + try: + proc = subprocess.Popen( + create_database_comand, + shell=True, + env={ + "PGPASSWORD": data_source.get("DATABASE").get("DATABASE_PASSWORD") + }, + ) + stdout, stderr =proc.communicate() + + + print ("Done for create %s, error %s"%(stdout,stderr)) + except Exception as e: + print("Error: exception happened during create database %s" % (e)) restore_command = "psql --username %s --host %s --port %s -d %s -f %s" % ( data_source.get("DATABASE").get("DATABASE_USER"), data_source.get("DATABASE").get("DATABASE_SERVER_URI"), @@ -55,13 +78,15 @@ def restore_database(source): ) print("Resore command: %s" % restore_command) try: - proc = subprocess.Popen( + proc2 = subprocess.Popen( restore_command, shell=True, env={ "PGPASSWORD": data_source.get("DATABASE").get("DATABASE_PASSWORD") }, ) - proc.wait() + stdout1, stderr1 = proc2.communicate() + print("Done for restore %s, error %s" % (stdout1, stderr1)) + except Exception as e: print("Error: exception happened during dump %s" % (e)) diff --git a/omero_search_engine/validation/results_validator.py b/omero_search_engine/validation/results_validator.py index c5d5966..d0513f9 100644 --- a/omero_search_engine/validation/results_validator.py +++ b/omero_search_engine/validation/results_validator.py @@ -299,7 +299,6 @@ def get_results_searchengine(self, operator=None): ) self.searchengine_results = json.loads(res.data) elif self.value: - self.searchengine_results = search_value_for_resource( "image", self.value, self.data_source ) @@ -798,8 +797,8 @@ def test_no_images(data_source): headers = lines[0] headers = headers.split("\t") - for i in range(len(headers) - 1): - print(i, headers[i]) + #for i in range(len(headers) - 1): + # print(i, headers[i]) names = {} for line in lines: if lines.index(line) == 0: @@ -925,7 +924,6 @@ def get_omero_stats(): with open(stats_file, "w") as f: f.write(report) - def check_number_images_sql_containers_using_ids(data_source): """ This method tests the number of images inside each container @@ -1017,9 +1015,12 @@ def check_number_images_sql_containers_using_ids(data_source): test_array.append(res.get("id")) for ress in results: if ress["id"] not in test_array: - print("================>>>>") print(ress["id"]) search_omero_app.logger.info("ERROR: Not equal results") + print (sql, query_data) + print ("searchengine_results:",searchengine_results) + print ("postgres_results: ",postgres_results) + print ("==============-=======") print( "Error checking %s name: %s, id: %s" % (resource, res_name, res_id) @@ -1032,7 +1033,6 @@ def check_number_images_sql_containers_using_ids(data_source): ) return dd - def get_no_images_sql_containers(data_source, write_report=True): """ This method tests the number of images inside each container @@ -1043,71 +1043,110 @@ def get_no_images_sql_containers(data_source, write_report=True): from omero_search_engine.api.v1.resources.urls import ( get_resource_names, ) + from omero_search_engine.api.v1.resources.utils import ( + search_resource_annotation, + ) from omero_search_engine.api.v1.resources.utils import adjust_query_for_container # conn = search_omero_app.config["database_connector"] - conn = search_omero_app.config.database_connectors[data_source] - all_names = get_resource_names("all") messages = [] - for resource in all_names: - messages.append( - "######################## Checking %s ########################\n" % resource - ) + for data_source_ in search_omero_app.config.database_connectors.keys(): + if data_source_.lower()!=data_source.lower(): + continue + conn = search_omero_app.config.database_connectors[data_source] - search_omero_app.logger.info( "######################## Checking %s ########################\n" % resource) - # this may be used for a specific data source - for data_source, res_name__ in all_names.get(resource).items(): + all_names = get_resource_names("all",data_source=json.dumps(data_source)) + #print (all_names) - for res_name_ in res_name__: # all_names.get(resource): - res_name = res_name_.get("name") - message1 = "Checking %s name: %s" % (resource, res_name) - messages.append(message1) - search_omero_app.logger.info("####>>>message: %s"%message1) - and_filters = [ - { - "name": "name", - "value": res_name, - "operator": "equals", - "resource": "container", + + for resource in all_names: + messages.append( + "######################## Checking %s ########################\n" % resource + ) + for ds, res_name__ in all_names.get(resource).items(): + for res_name_ in res_name__: + res_name = res_name_.get("name") + res_id = res_name_.get("id") + print (res_name) + message1 = "Checking %s name: %s" % (resource, res_name) + messages.append(message1) + search_omero_app.logger.info(message1) + + # and_filters = [ + # { + # "name": "name", + # "id": res_id, + # "operator": "equals", + # "resource": "container", + # } + # ] + # or_filters = [] + #query = {"and_filters": and_filters, "or_filters": or_filters} + #query_data = {"query_details": query} + #adjust_query_for_container(query_data) + and_filters = [] + main_attributes = { + "and_main_attributes": [ + { + "name": "%s_id" % resource, + "value": res_id, + "operator": "equals", + "resource": "image", + }, + { + "name": "data_source", + "value": data_source, + "operator": "equals", + "resource": "image", + }, + ] } - ] - or_filters = [] - query = {"and_filters": and_filters, "or_filters": or_filters} - query_data = {"query_details": query} - adjust_query_for_container(query_data) - returned_results = determine_search_results_(query_data, data_source=data_source) - if returned_results.get("results"): - if returned_results.get("results").get("size"): - seachengine_results = returned_results["results"]["size"] - else: - seachengine_results = 0 - message2 = ( - "No of images returned from searchengine: %s" % seachengine_results - ) - search_omero_app.logger.info(message2) - messages.append(message2) - sql = query_methods["%s_name" % resource].substitute( - name=res_name, operator="=" - ) - results = conn.execute_query(sql) - postgres_results = len(results) - message3 = ( - "Number of images returned from the database: %s" % postgres_results - ) - messages.append(message3) - search_omero_app.logger.info(message3) - if seachengine_results != postgres_results: - message4 = "ERROR: Not equal results" - messages.append(message4) - search_omero_app.logger.info(message4) - else: - message5 = "equal results" - messages.append(message5) - search_omero_app.logger.info(message5) - messages.append( - "\n-----------------------------------------------------------------------------\n" # noqa - ) + or_filters = [] + query = {"and_filters": and_filters, "or_filters": or_filters} + + query_data = { + "query_details": query, + "main_attributes": main_attributes, + } + + returned_results = search_resource_annotation("image", query_data) + #returned_results = determine_search_results_(query_data, data_source=data_source) + if returned_results.get("results"): + if returned_results.get("results").get("size"): + seachengine_results = returned_results["results"]["size"] + else: + seachengine_results = 0 + message2 = ( + "No of images returned from searchengine: %s" % seachengine_results + ) + search_omero_app.logger.info(message2) + messages.append(message2) + #sql = query_methods["%s_name" % resource].substitute( + # name=res_name, operator="=" + #) + if resource == "project": + sql = query_images_in_project_id.substitute(project_id=res_id) + elif resource == "screen": + sql = query_images_in_screen_id.substitute(screen_id=res_id) + results = conn.execute_query(sql) + postgres_results = len(results) + message3 = ( + "Number of images returned from the database: %s" % postgres_results + ) + messages.append(message3) + search_omero_app.logger.info(message3) + if seachengine_results != postgres_results: + message4 = "ERROR: Not equal results" + messages.append(message4) + search_omero_app.logger.info(message4) + else: + message5 = "equal results" + messages.append(message5) + search_omero_app.logger.info(message5) + messages.append( + "\n-----------------------------------------------------------------------------\n" # noqa + ) if write_report: base_folder = search_omero_app.config.get("BASE_FOLDER") if not os.path.isdir(base_folder): diff --git a/unit_tests/test_app.py b/unit_tests/test_app.py index 5a85faa..7924ea7 100644 --- a/unit_tests/test_app.py +++ b/unit_tests/test_app.py @@ -114,18 +114,17 @@ def test_query_database(self): data_source ].execute_query(sql) self.assertIsNotNone(res) - found_db_name = False + + found_db_name=False for source in search_omero_app.config.get("DATA_SOURCES"): - if source.get("DATABASE").get("DATABASE_NAME") == res[0]["current_database"]: - found_db_name = True + if source.get("DATABASE").get("DATABASE_NAME")==res[0]["current_database"]: + found_db_name=True break - #for data_source in search_omero_app.config.database_connectors.keys(): - # res = search_omero_app.config.database_connectors[ - # data_source - # ].execute_query(sql) - # self.assertIsNotNone(res) - # self.assertEqual(res[0]["current_database"], "omero") + self.assertTrue(found_db_name) + #self.assertEqual(res[0]["current_database"], search_omero_app.config.database_connectors[data_source]["DATABASE_NAME"]) + #self.assertEqual(res[0]["current_database"], search_omero_app.config.database_connectors[data_source]["DATABASE_NAME"]) + def validate_json_syntax(self, json_template): try: @@ -160,6 +159,7 @@ def test_add_submit_query_delete_es_index(self): """ table = "image1" es_index = "image_keyvalue_pair_metadata_1" + es_index = "image_keyvalue_pair_metadata_1" es_index_2 = "key_values_resource_cach" create_es_index_2 = True all_all_indices = get_all_indexes_from_elasticsearch()