Skip to content

Commit

Permalink
fix getting number of image inside a container
Browse files Browse the repository at this point in the history
  • Loading branch information
@khaledk2
khaledk2 committed Sep 23, 2024
1 parent 5ff232b commit 5a899c5
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 65 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,14 @@ jobs:
# Configure database url
sysctl -w vm.max_map_count=262144
python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero1 -d omero -b omero.pgdump
python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero_train -d omero_train -b omero_train.pgdump
# python manage.py set_database_configuration -u localhost -s ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd -w omero_train -d omero_train -b omero_train.pgdump
# configure elasticsearch
python manage.py set_elasticsearch_configuration -e localhost:${{ job.services.elasticsearch.ports[9200] }}
# download and extract the database backup file
wget https://downloads.openmicroscopy.org/images/omero_db_searchengine.zip -P app_data
unzip app_data/omero_db_searchengine.zip -d app_data/
wget https://github.com/khaledk2/ice-archh-64/releases/download/new_re_db/omero_train.zip -P app_data
unzip app_data/omero_train.zip -d app_data/
# wget https://github.com/khaledk2/ice-archh-64/releases/download/new_re_db/omero_train.zip -P app_data
# unzip app_data/omero_train.zip -d app_data/
# run restore omero database
python manage.py restore_postgresql_database
rm app_data/omero_db_searchengine.zip
Expand Down
13 changes: 8 additions & 5 deletions omero_search_engine/api/v1/resources/resource_analyser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -892,7 +892,7 @@ def get_resource_attribute_values(
return returned_results


def get_resource_names(resource, name=None, description=False):
def get_resource_names(resource, name=None, description=False, data_source=None):
"""
return resources names attributes
It works for projects and screens but can be extended.
Expand All @@ -903,18 +903,21 @@ def get_resource_names(resource, name=None, description=False):
)

if resource != "all":
returned_results = get_the_results(resource, name, description)
returned_results = get_the_results(resource, name, description, data_source)
else:
returned_results = {}
ress = ["project", "screen"]
for res in ress:
returned_results[res] = get_the_results(res, name, description)
returned_results[res] = get_the_results(res, name, description, data_source)
return returned_results


def get_the_results(resource, name, description, es_index="key_values_resource_cach"):
def get_the_results(resource, name, description, data_source, es_index="key_values_resource_cach"):
returned_results = {}
query = key_values_buckets_template_2.substitute(resource=resource)
if data_source:
query = key_values_buckets_template_with_data_source.substitute(resource=resource, data_source=data_source)
else:
query = key_values_buckets_template_2.substitute(resource=resource)
results_ = connect_elasticsearch(
es_index, query
) # .search(index=es_index, body=query)
Expand Down
155 changes: 98 additions & 57 deletions omero_search_engine/validation/results_validator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1043,70 +1043,111 @@ def get_no_images_sql_containers(data_source, write_report=True):
from omero_search_engine.api.v1.resources.urls import (
get_resource_names,
)
from omero_search_engine.api.v1.resources.utils import (
search_resource_annotation,
)
from omero_search_engine.api.v1.resources.utils import adjust_query_for_container

# conn = search_omero_app.config["database_connector"]
conn = search_omero_app.config.database_connectors[data_source]

all_names = get_resource_names("all")
messages = []
for resource in all_names:
messages.append(
"######################## Checking %s ########################\n" % resource
)
# this may be used for a specific data source
for data_source, res_name__ in all_names.get(resource).items():
pass
for res_name_ in res_name__: # all_names.get(resource):
res_name = res_name_.get("name")
message1 = "Checking %s name: %s" % (resource, res_name)
messages.append(message1)
search_omero_app.logger.info(message1)
for data_source_ in search_omero_app.config.database_connectors.keys():
if data_source_.lower()!=data_source.lower():
continue
print (data_source)
conn = search_omero_app.config.database_connectors[data_source]

and_filters = [
{
"name": "name",
"value": res_name,
"operator": "equals",
"resource": "container",
all_names = get_resource_names("all",data_source=json.dumps(data_source))
#print (all_names)


for resource in all_names:
messages.append(
"######################## Checking %s ########################\n" % resource
)
for ds, res_name__ in all_names.get(resource).items():
for res_name_ in res_name__:
res_name = res_name_.get("name")
res_id = res_name_.get("id")
print (res_name)
message1 = "Checking %s name: %s" % (resource, res_name)
messages.append(message1)
search_omero_app.logger.info(message1)

# and_filters = [
# {
# "name": "name",
# "id": res_id,
# "operator": "equals",
# "resource": "container",
# }
# ]
# or_filters = []
#query = {"and_filters": and_filters, "or_filters": or_filters}
#query_data = {"query_details": query}
#adjust_query_for_container(query_data)
and_filters = []
main_attributes = {
"and_main_attributes": [
{
"name": "%s_id" % resource,
"value": res_id,
"operator": "equals",
"resource": "image",
},
{
"name": "data_source",
"value": data_source,
"operator": "equals",
"resource": "image",
},
]
}
]
or_filters = []
query = {"and_filters": and_filters, "or_filters": or_filters}
query_data = {"query_details": query}
adjust_query_for_container(query_data)
returned_results = determine_search_results_(query_data, data_source=data_source)
if returned_results.get("results"):
if returned_results.get("results").get("size"):
seachengine_results = returned_results["results"]["size"]
else:
seachengine_results = 0
message2 = (
"No of images returned from searchengine: %s" % seachengine_results
)
search_omero_app.logger.info(message2)
messages.append(message2)
sql = query_methods["%s_name" % resource].substitute(
name=res_name, operator="="
)
results = conn.execute_query(sql)
postgres_results = len(results)
message3 = (
"Number of images returned from the database: %s" % postgres_results
)
messages.append(message3)
search_omero_app.logger.info(message3)
if seachengine_results != postgres_results:
message4 = "ERROR: Not equal results"
messages.append(message4)
search_omero_app.logger.info(message4)
else:
message5 = "equal results"
messages.append(message5)
search_omero_app.logger.info(message5)
messages.append(
"\n-----------------------------------------------------------------------------\n" # noqa
)
or_filters = []
query = {"and_filters": and_filters, "or_filters": or_filters}

query_data = {
"query_details": query,
"main_attributes": main_attributes,
}

returned_results = search_resource_annotation("image", query_data)
#returned_results = determine_search_results_(query_data, data_source=data_source)
if returned_results.get("results"):
if returned_results.get("results").get("size"):
seachengine_results = returned_results["results"]["size"]
else:
seachengine_results = 0
message2 = (
"No of images returned from searchengine: %s" % seachengine_results
)
search_omero_app.logger.info(message2)
messages.append(message2)
#sql = query_methods["%s_name" % resource].substitute(
# name=res_name, operator="="
#)
if resource == "project":
sql = query_images_in_project_id.substitute(project_id=res_id)
elif resource == "screen":
sql = query_images_in_screen_id.substitute(screen_id=res_id)
results = conn.execute_query(sql)
postgres_results = len(results)
message3 = (
"Number of images returned from the database: %s" % postgres_results
)
messages.append(message3)
search_omero_app.logger.info(message3)
if seachengine_results != postgres_results:
message4 = "ERROR: Not equal results"
messages.append(message4)
search_omero_app.logger.info(message4)
else:
message5 = "equal results"
messages.append(message5)
search_omero_app.logger.info(message5)
messages.append(
"\n-----------------------------------------------------------------------------\n" # noqa
)
if write_report:
base_folder = search_omero_app.config.get("BASE_FOLDER")
if not os.path.isdir(base_folder):
Expand Down

0 comments on commit 5a899c5

Please sign in to comment.