diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bde6f3c..93e3036 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,14 +65,14 @@ jobs: # Configure database url sysctl -w vm.max_map_count=262144 python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero1 -d omero -b omero.pgdump - python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero_train -d omero_train -b omero_train.pgdump + # python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero_train -d omero_train -b omero_train.pgdump # configure elasticsearch python manage.py set_elasticsearch_configuration -e localhost:${{ job.services.elasticsearch.ports[9200] }} # download and extract the database backup file wget https://downloads.openmicroscopy.org/images/omero_db_searchengine.zip -P app_data unzip app_data/omero_db_searchengine.zip -d app_data/ - wget https://github.com/khaledk2/ice-archh-64/releases/download/new_re_db/omero_train.zip -P app_data - unzip app_data/omero_train.zip -d app_data/ + # wget https://github.com/khaledk2/ice-archh-64/releases/download/new_re_db/omero_train.zip -P app_data + # unzip app_data/omero_train.zip -d app_data/ # run restore omero database python manage.py restore_postgresql_database rm app_data/omero_db_searchengine.zip diff --git a/omero_search_engine/api/v1/resources/resource_analyser.py b/omero_search_engine/api/v1/resources/resource_analyser.py index e138792..fd81234 100644 --- a/omero_search_engine/api/v1/resources/resource_analyser.py +++ b/omero_search_engine/api/v1/resources/resource_analyser.py @@ -892,7 +892,7 @@ def get_resource_attribute_values( return returned_results -def get_resource_names(resource, name=None, description=False): +def get_resource_names(resource, name=None, description=False, data_source=None): """ return resources names attributes It works for projects and screens but can be extended. @@ -903,18 +903,21 @@ def get_resource_names(resource, name=None, description=False): ) if resource != "all": - returned_results = get_the_results(resource, name, description) + returned_results = get_the_results(resource, name, description, data_source) else: returned_results = {} ress = ["project", "screen"] for res in ress: - returned_results[res] = get_the_results(res, name, description) + returned_results[res] = get_the_results(res, name, description, data_source) return returned_results -def get_the_results(resource, name, description, es_index="key_values_resource_cach"): +def get_the_results(resource, name, description, data_source, es_index="key_values_resource_cach"): returned_results = {} - query = key_values_buckets_template_2.substitute(resource=resource) + if data_source: + query = key_values_buckets_template_with_data_source.substitute(resource=resource, data_source=data_source) + else: + query = key_values_buckets_template_2.substitute(resource=resource) results_ = connect_elasticsearch( es_index, query ) # .search(index=es_index, body=query) diff --git a/omero_search_engine/validation/results_validator.py b/omero_search_engine/validation/results_validator.py index f9b1095..e667863 100644 --- a/omero_search_engine/validation/results_validator.py +++ b/omero_search_engine/validation/results_validator.py @@ -1043,70 +1043,111 @@ def get_no_images_sql_containers(data_source, write_report=True): from omero_search_engine.api.v1.resources.urls import ( get_resource_names, ) + from omero_search_engine.api.v1.resources.utils import ( + search_resource_annotation, + ) from omero_search_engine.api.v1.resources.utils import adjust_query_for_container # conn = search_omero_app.config["database_connector"] - conn = search_omero_app.config.database_connectors[data_source] - all_names = get_resource_names("all") messages = [] - for resource in all_names: - messages.append( - "######################## Checking %s ########################\n" % resource - ) - # this may be used for a specific data source - for data_source, res_name__ in all_names.get(resource).items(): - pass - for res_name_ in res_name__: # all_names.get(resource): - res_name = res_name_.get("name") - message1 = "Checking %s name: %s" % (resource, res_name) - messages.append(message1) - search_omero_app.logger.info(message1) + for data_source_ in search_omero_app.config.database_connectors.keys(): + if data_source_.lower()!=data_source.lower(): + continue + print (data_source) + conn = search_omero_app.config.database_connectors[data_source] - and_filters = [ - { - "name": "name", - "value": res_name, - "operator": "equals", - "resource": "container", + all_names = get_resource_names("all",data_source=json.dumps(data_source)) + #print (all_names) + + + for resource in all_names: + messages.append( + "######################## Checking %s ########################\n" % resource + ) + for ds, res_name__ in all_names.get(resource).items(): + for res_name_ in res_name__: + res_name = res_name_.get("name") + res_id = res_name_.get("id") + print (res_name) + message1 = "Checking %s name: %s" % (resource, res_name) + messages.append(message1) + search_omero_app.logger.info(message1) + + # and_filters = [ + # { + # "name": "name", + # "id": res_id, + # "operator": "equals", + # "resource": "container", + # } + # ] + # or_filters = [] + #query = {"and_filters": and_filters, "or_filters": or_filters} + #query_data = {"query_details": query} + #adjust_query_for_container(query_data) + and_filters = [] + main_attributes = { + "and_main_attributes": [ + { + "name": "%s_id" % resource, + "value": res_id, + "operator": "equals", + "resource": "image", + }, + { + "name": "data_source", + "value": data_source, + "operator": "equals", + "resource": "image", + }, + ] } - ] - or_filters = [] - query = {"and_filters": and_filters, "or_filters": or_filters} - query_data = {"query_details": query} - adjust_query_for_container(query_data) - returned_results = determine_search_results_(query_data, data_source=data_source) - if returned_results.get("results"): - if returned_results.get("results").get("size"): - seachengine_results = returned_results["results"]["size"] - else: - seachengine_results = 0 - message2 = ( - "No of images returned from searchengine: %s" % seachengine_results - ) - search_omero_app.logger.info(message2) - messages.append(message2) - sql = query_methods["%s_name" % resource].substitute( - name=res_name, operator="=" - ) - results = conn.execute_query(sql) - postgres_results = len(results) - message3 = ( - "Number of images returned from the database: %s" % postgres_results - ) - messages.append(message3) - search_omero_app.logger.info(message3) - if seachengine_results != postgres_results: - message4 = "ERROR: Not equal results" - messages.append(message4) - search_omero_app.logger.info(message4) - else: - message5 = "equal results" - messages.append(message5) - search_omero_app.logger.info(message5) - messages.append( - "\n-----------------------------------------------------------------------------\n" # noqa - ) + or_filters = [] + query = {"and_filters": and_filters, "or_filters": or_filters} + + query_data = { + "query_details": query, + "main_attributes": main_attributes, + } + + returned_results = search_resource_annotation("image", query_data) + #returned_results = determine_search_results_(query_data, data_source=data_source) + if returned_results.get("results"): + if returned_results.get("results").get("size"): + seachengine_results = returned_results["results"]["size"] + else: + seachengine_results = 0 + message2 = ( + "No of images returned from searchengine: %s" % seachengine_results + ) + search_omero_app.logger.info(message2) + messages.append(message2) + #sql = query_methods["%s_name" % resource].substitute( + # name=res_name, operator="=" + #) + if resource == "project": + sql = query_images_in_project_id.substitute(project_id=res_id) + elif resource == "screen": + sql = query_images_in_screen_id.substitute(screen_id=res_id) + results = conn.execute_query(sql) + postgres_results = len(results) + message3 = ( + "Number of images returned from the database: %s" % postgres_results + ) + messages.append(message3) + search_omero_app.logger.info(message3) + if seachengine_results != postgres_results: + message4 = "ERROR: Not equal results" + messages.append(message4) + search_omero_app.logger.info(message4) + else: + message5 = "equal results" + messages.append(message5) + search_omero_app.logger.info(message5) + messages.append( + "\n-----------------------------------------------------------------------------\n" # noqa + ) if write_report: base_folder = search_omero_app.config.get("BASE_FOLDER") if not os.path.isdir(base_folder):