diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac1c6ae2..48dd5956 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,10 +24,14 @@ jobs: --health-retries 5 elasticsearch: - image: elasticsearch:7.16.2 + image: elasticsearch:8.8.1 ports: - 9200/tcp - options: -e="discovery.type=single-node" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10 + env: + es_api_basic_auth_username: "elastic" + ELASTIC_PASSWORD: "elasticsearch_user_password" + + options: -e="discovery.type=single-node" --health-cmd="curl -k -u elastic:elasticsearch_user_password https://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - uses: actions/checkout@v2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9159f50b..a8622a84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black args: [--target-version=py35] diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..b38f4eed --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,23 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF and ePub +formats: all + +# Optionally set the version of Python and requirements required to build your docs +build: + os: ubuntu-22.04 + tools: + python: "3.11" diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eee6413..adf65a54 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +0.5.3 (September 2023): +----------------------- +- Secure the connection with elasticsearch [#92](https://github.com/ome/omero_search_engine/pull/92) + + +0.5.2 (June 2023): +------------------ +- Return all the available values for a key in a container [#77](https://github.com/ome/omero_search_engine/pull/77) +- Return the available attributes with a container [#77](https://github.com/ome/omero_search_engine/pull/77) + 0.5.1 (February 2023): ---------------------- - Fix the issue of filtering images by the container name [#81](https://github.com/ome/omero_search_engine/pull/81) diff --git a/README.rst b/README.rst index 9f11def3..c86caa36 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,10 @@ .. image:: https://github.com/ome/omero_search_engine/workflows/Build/badge.svg :target: https://github.com/ome/omero_search_engine/actions +.. image:: https://readthedocs.org/projects/omero-search-engine/badge/?version=latest + :target: https://omero-search-engine.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + OMERO Search Engine -------------------- diff --git a/configurations/app_config.yml b/configurations/app_config.yml index f888be60..4c1a2e21 100644 --- a/configurations/app_config.yml +++ b/configurations/app_config.yml @@ -6,9 +6,11 @@ DATABASE_NAME : "omero" CACHE_FOLDER : "path/to/folder/app_data" SECRET_KEY : "fsdasdh3424vvcsd467fgh" ASYNCHRONOUS_SEARCH : True -ELASTICSEARCH_URL : "http://localhost:9200" +ELASTICSEARCH_URL : "https://localhost:9200" IDR_TEST_FILE_URL : "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv" PAGE_SIZE : 1000 CACHE_ROWS : 10000 MAX_RETUNED_ITEMS : 1700000 ELASTICSEARCH_BACKUP_FOLDER: "path/to/elasticsearch/backup/folder" +verify_certs: False +ELASTIC_PASSWORD: elasticsearch_user_password diff --git a/configurations/configuration.py b/configurations/configuration.py index 9d2dc099..1742b249 100644 --- a/configurations/configuration.py +++ b/configurations/configuration.py @@ -20,6 +20,7 @@ import yaml from shutil import copyfile import os +import json def load_configuration_variables_from_file(config): @@ -29,6 +30,20 @@ def load_configuration_variables_from_file(config): cofg = yaml.load(f) for x, y in cofg.items(): setattr(config, x, y) + if hasattr(config, "verify_certs"): + try: + verify_certs = json.load(config.verify_certs) + except Exception as ex: + print(str(ex)) + verify_certs = False + else: + verify_certs = False + config.verify_certs = verify_certs + if not verify_certs: + import requests + from requests.packages.urllib3.exceptions import InsecureRequestWarning + + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) def set_database_connection_variables(config): diff --git a/docs/configuration/configuration_installation.rst b/docs/configuration/configuration_installation.rst index f680367f..1e4ba43b 100644 --- a/docs/configuration/configuration_installation.rst +++ b/docs/configuration/configuration_installation.rst @@ -14,10 +14,12 @@ The application should have the access attributes (e.g, URL, username, password, * ``DATABASE_NAME`` * ``ELASTICSEARCH__URL`` * ``PAGE_SIZE`` + * ``ELASTIC_PASSWORD`` * Although the user can edit this file to set the values, there are some methods inside :omero_search_engine:`manage.py ` which could help to set the configuration e.g. * ``set_database_configuration`` * ``set_elasticsearch_configuration`` + * ``set_elasticsearch_password`` * When the app runs for the first time, it will look for the application configuration file. diff --git a/docs/developer/developer.rst b/docs/developer/developer.rst index 10c0e8f9..b1348675 100644 --- a/docs/developer/developer.rst +++ b/docs/developer/developer.rst @@ -1,2 +1,20 @@ Developer's documents -===================== \ No newline at end of file +===================== + +The developer should clone the code from the project repo using the following command:: + + git clone https://github.com/ome/omero_search_engine.git + +Then they need to create a Python virtual environment variable using either venv or conda and install the packages inside requirements.txt + +The developer needs to set up the application configuration as it is explained in the System configuration part inside "docs\configuration\configuration_installation.rst" + +After that, they should run the indexer to index Omero's data using the following command:: + + python manage.py get_index_data_from_database + +The developer can run the application using the following command:: + + python manage.py runserver -p 5577 + +Running the scripts inside the examples folder can be a good starting point. diff --git a/examples/container_key_values.py b/examples/container_key_values.py new file mode 100644 index 00000000..d9eb836a --- /dev/null +++ b/examples/container_key_values.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (C) 2023 University of Dundee & Open Microscopy Environment. +# All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from utils import base_url +import requests +import json +import logging + +""" +Return the available keys in a containers +Also get the aviable values for a key """ + +resource = "image" +container_name = "idr0034" +key = "cell line" + +# the following url will return the existing key in this container +keys_url = ( + "{base_url}resources/image/container_keys/?container_name={container_name}".format( + base_url=base_url, container_name=container_name + ) +) + +resp = requests.get(url=keys_url) +keys_results = json.loads(resp.text) +for result in keys_results: + logging.info("%s: %s" % (result.get("type"), result.get("name"))) + for bucket in result.get("results"): + logging.info( + "Key: %s, no of images: %s " % (bucket.get("key"), bucket.get("no_image")) + ) + +""" It is possible to get all the available +values for a key +""" +values_key_url = ( + "{base_url}resources/image/" + "container_keyvalues/?container_name={container_name}&key={key}".format( + base_url=base_url, container_name=container_name, key=key + ) +) + +resp = requests.get(url=values_key_url) + +key_values_results = json.loads(resp.text) + +for result in key_values_results: + logging.info("%s: %s" % (result.get("type"), result.get("name"))) + for bucket in result.get("results"): + logging.info( + "Key: %s, value: %s, no of images: %s " + % (bucket.get("key"), bucket.get("value"), bucket.get("no_image")) + ) diff --git a/examples/search_with_bookmark_paging_using_submitquery.py b/examples/search_with_bookmark_paging_using_submitquery.py index 7cc432f5..95cb15bf 100644 --- a/examples/search_with_bookmark_paging_using_submitquery.py +++ b/examples/search_with_bookmark_paging_using_submitquery.py @@ -118,7 +118,6 @@ def call_omero_searchengine_return_results(url, data=None, method="post"): ) while len(received_results) < total_results: - page += 1 query_data_ = {"query_details": {"and_filters": and_filters}, "bookmark": bookmark} query_data_json_ = json.dumps(query_data_) diff --git a/manage.py b/manage.py index f91c1d80..81be9d0f 100644 --- a/manage.py +++ b/manage.py @@ -205,6 +205,24 @@ def set_elasticsearch_configuration(elasticsearch_url=None): search_omero_app.logger.info("No attribute is provided") +@manager.command +@manager.option("-e", "--elasticsearch_password", help="set elasticsearch password") +def set_elasticsearch_password(elasticsearch_password=None): + if elasticsearch_password: + update_config_file({"ELASTIC_PASSWORD": elasticsearch_password}) + else: + search_omero_app.logger.info("No attribute is provided") + + +@manager.command +@manager.option("-v", "--verify_certs", help="set elasticsearch password") +def set_verify_certs(verify_certs=None): + if verify_certs: + update_config_file({"verify_certs": verify_certs}) + else: + search_omero_app.logger.info("No attribute is provided") + + @manager.command @manager.option("-c", "--cache_folder", help="cache folder path") def set_cache_folder(cache_folder=None): @@ -386,6 +404,15 @@ def data_validator(screen_name=None, project_name=None): print(start2) print(end) +@manager.command +def test_container_key_value(): + from omero_search_engine.validation.results_validator import ( + check_container_keys_vakues, + ) + + check_container_keys_vakues() + + if __name__ == "__main__": manager.run() diff --git a/omero_search_engine/__init__.py b/omero_search_engine/__init__.py index 77012d72..adf38f99 100644 --- a/omero_search_engine/__init__.py +++ b/omero_search_engine/__init__.py @@ -22,7 +22,6 @@ import logging from elasticsearch import Elasticsearch from flasgger import Swagger, LazyString, LazyJSONEncoder - from omero_search_engine.database.database_connector import DatabaseConnector from configurations.configuration import ( configLooader, @@ -54,7 +53,6 @@ "version": "0.2.0", } - swagger = Swagger(search_omero_app, template=template) app_config = load_configuration_variables_from_file(config_) @@ -72,12 +70,17 @@ def create_app(config_name="development"): search_omero_app.app_context().push() search_omero_app.app_context() search_omero_app.app_context().push() + ELASTIC_PASSWORD = app_config.ELASTIC_PASSWORD + es_connector = Elasticsearch( app_config.ELASTICSEARCH_URL.split(","), + verify_certs=app_config.verify_certs, timeout=130, max_retries=20, retry_on_timeout=True, connections_per_node=10, + scheme="https", + http_auth=("elastic", ELASTIC_PASSWORD), ) search_omero_app.config["database_connector"] = database_connector @@ -127,7 +130,6 @@ def after_request(response): # added to let the user know the proper extension they should use @search_omero_app.errorhandler(404) def page_not_found(error): - search_omero_app.logger.info("Error: %s" % error) resp_message = ( "%s, You may use '/searchengine/api/v1/resources/' to test\ diff --git a/omero_search_engine/api/v1/resources/resource_analyser.py b/omero_search_engine/api/v1/resources/resource_analyser.py index 01474193..877a3bad 100644 --- a/omero_search_engine/api/v1/resources/resource_analyser.py +++ b/omero_search_engine/api/v1/resources/resource_analyser.py @@ -28,6 +28,7 @@ adjust_value, ) import math +from flask import jsonify, Response key_number_search_template = Template( """ @@ -375,8 +376,6 @@ def prepare_search_results_buckets(results_): def get_key_values_return_contents(name, resource, csv): - from flask import jsonify, Response - resource_keys = query_cashed_bucket(name, resource) # if a csv flag is true thenm iut will send a CSV file # which contains the results otherwise it will return a JSON file @@ -801,3 +800,136 @@ def get_the_results(resource, name, description, es_index="key_values_resource_c del item["description"] return returned_results + + +def get_container_values_for_key(table_, container_name, csv, key=None): + returned_results = [] + pr_names = get_resource_names("all") + for resourse, names in pr_names.items(): + act_name = [ + {"id": name["id"], "name": name["name"]} + for name in names + if name["name"] and container_name.lower() in name["name"].lower() + ] + if len(act_name) > 0: + for id in act_name: + if resourse != table_: + res = process_container_query( + table_, resourse + "_id", id["id"], key, table_ + ) + else: + res = process_container_query(table_, "id", id["id"], key, table_) + if len(res) > 0: + returned_results.append( + {"name": id["name"], "type": resourse, "results": res} + ) + if csv: + if key: + containers = [ + ",".join(["Container", "Type", "Key", "Value", "No of %s" % table_]) + ] + else: + containers = [",".join(["Container", "Type", "Key", "No of %s" % table_])] + for r_results in returned_results: + reso = r_results.get("name") + type = r_results.get("type") + for res in r_results.get("results"): + if key: + containers.append( + ",".join( + [ + reso, + type, + res.get("key"), + res.get("value"), + str(res.get("no_%s" % table_)), + ] + ) + ) + else: + containers.append( + ",".join( + [reso, type, res.get("key"), str(res.get("no_%s" % table_))] + ) + ) + if key: + file_name = "container_%s_%s_values.csv" % (container_name, key) + else: + file_name = "container_%s_keys.csv" % container_name + + return Response( + "\n".join(containers), + mimetype="text/csv", + headers={"Content-disposition": "attachment; filename=%s" % (file_name)}, + ) + return jsonify(returned_results) + + +def process_container_query(table_, attribute_name, container_id, key, resourse): + from omero_search_engine.api.v1.resources.utils import elasticsearch_query_builder + + res_index = resource_elasticsearchindex.get(table_) + main_attributes = { + "and_main_attributes": [ + {"name": attribute_name, "value": container_id, "operator": "equals"} + ] + } + query_ = elasticsearch_query_builder([], [], False, main_attributes=main_attributes) + query = json.loads(query_) + if key: + query["aggs"] = json.loads( + container_project_values_key_template.substitute(key=key.strip()) + ) + else: + query["aggs"] = container_project_keys_template + query["_source"] = {"includes": [""]} + res = search_index_for_value(res_index, query) + if key: + buckets = res["aggregations"]["key_values"]["key_filter"]["uniquesTerms"][ + "buckets" + ] + for bucket in buckets: + bucket["value"] = bucket["key"] + bucket["key"] = key + bucket["no_" + resourse] = bucket["doc_count"] + del bucket["doc_count"] + return buckets + + else: + buckets = res["aggregations"]["keys_search"]["uniquesTerms"]["buckets"] + for bucket in buckets: + bucket["no_" + resourse] = bucket["doc_count"] + del bucket["doc_count"] + return buckets + + +""" +get all the values buckets for a key""" +container_project_values_key_template = Template( + """{"key_values":{"nested":{"path":"key_values"},"aggs":{"key_filter":{ + "filter":{"terms":{"key_values.name.keynamenormalize":["$key"]} + },"aggs":{"required_values":{"cardinality": + {"field": "key_values.value.keyvalue", + "precision_threshold":4000}},"uniquesTerms": + {"terms": {"field": "key_values.value.keyvalue","size": 10000}}}}}}}""" +) + + +""" +Get all the keys bucket""" +container_project_keys_template = { + "keys_search": { + "nested": {"path": "key_values"}, + "aggs": { + "required_values": { + "cardinality": { + "field": "key_values.name.keynamenormalize", + "precision_threshold": 4000, + }, + }, + "uniquesTerms": { + "terms": {"field": "key_values.name.keynamenormalize", "size": 10000} + }, + }, + } +} diff --git a/omero_search_engine/api/v1/resources/swagger_docs/container_keys.yml b/omero_search_engine/api/v1/resources/swagger_docs/container_keys.yml new file mode 100644 index 00000000..db44eeb3 --- /dev/null +++ b/omero_search_engine/api/v1/resources/swagger_docs/container_keys.yml @@ -0,0 +1,25 @@ +A searchengine endpoint to return the available keys for a container +--- +tags: + - Container keys +parameters: + - name: resource_table + in: path + type: string + enum: ['image', 'project', 'screen', 'well', 'plate'] + required: true + - name: container_name + description: The container name + in: query + type: string + required: true + - name: csv + description: a flag to return a CSV file which is created on the fly instead of JSON + in: query + type: boolean + required: false +responses: + 200: + description: A JSON contains the search results + examples: + results: [] diff --git a/omero_search_engine/api/v1/resources/swagger_docs/container_keyvalues.yml b/omero_search_engine/api/v1/resources/swagger_docs/container_keyvalues.yml new file mode 100644 index 00000000..4b21c6d8 --- /dev/null +++ b/omero_search_engine/api/v1/resources/swagger_docs/container_keyvalues.yml @@ -0,0 +1,31 @@ +A searchengine endpoint to return the available values for a key in specific container +--- +tags: + - Available values for a key in specific container +parameters: + - name: resource_table + in: path + type: string + enum: ['image', 'project', 'screen', 'well', 'plate'] + required: true + - name: container_name + description: The container name + in: query + type: string + required: true + - name: key + description: the resource attribute + in: query + type: string + required: true + - name: csv + description: a flag to return a CSV file which is created on the fly instead of JSON + in: query + type: boolean + required: false + +responses: + 200: + description: A JSON contains the search results + examples: + results: [] diff --git a/omero_search_engine/api/v1/resources/swagger_docs/submitquery_returncontainers.yml b/omero_search_engine/api/v1/resources/swagger_docs/submitquery_returncontainers.yml index 5aab3e29..968ccfcf 100644 --- a/omero_search_engine/api/v1/resources/swagger_docs/submitquery_returncontainers.yml +++ b/omero_search_engine/api/v1/resources/swagger_docs/submitquery_returncontainers.yml @@ -50,7 +50,7 @@ Another example: --- tags: - - Mixed Complex query + - Return containers for a Mixed Complex query parameters: - name: return_columns diff --git a/omero_search_engine/api/v1/resources/urls.py b/omero_search_engine/api/v1/resources/urls.py index df8c9bba..77a3f037 100644 --- a/omero_search_engine/api/v1/resources/urls.py +++ b/omero_search_engine/api/v1/resources/urls.py @@ -387,3 +387,49 @@ def search(resource_table): return_containers, ) return jsonify(results) + + +@resources.route("//container_keyvalues/", methods=["GET"]) +def container_key_values_search(resource_table): + """ + file: swagger_docs/container_keyvalues.yml + """ + from omero_search_engine.api.v1.resources.resource_analyser import ( + get_container_values_for_key, + ) + + key = request.args.get("key") + container_name = request.args.get("container_name") + if not container_name or not key: + return build_error_message("Container name and key are required") + csv = request.args.get("csv") + if csv: + try: + csv = json.loads(csv.lower()) + except Exception: + csv = False + results = get_container_values_for_key(resource_table, container_name, csv, key) + return results + + +@resources.route("//container_keys/", methods=["GET"]) +def container_keys_search(resource_table): + """ + file: swagger_docs/container_keys.yml + """ + from omero_search_engine.api.v1.resources.resource_analyser import ( + get_container_values_for_key, + ) + + container_name = request.args.get("container_name") + if not container_name: + return build_error_message("Container name is required") + + csv = request.args.get("csv") + if csv: + try: + csv = json.loads(csv.lower()) + except Exception: + csv = False + results = get_container_values_for_key(resource_table, container_name, csv) + return results diff --git a/omero_search_engine/api/v1/resources/utils.py b/omero_search_engine/api/v1/resources/utils.py index 657a1307..6f2476f8 100644 --- a/omero_search_engine/api/v1/resources/utils.py +++ b/omero_search_engine/api/v1/resources/utils.py @@ -699,7 +699,6 @@ def elasticsearch_query_builder( should_part_list = should_part_list_ if len(should_part_list) > 0: - should_part_ = ",".join(should_part_list) should_part_ = should_term_template.substitute( should_term=should_part_, minimum_should_match=1 @@ -718,7 +717,6 @@ def elasticsearch_query_builder( all_terms = nested_must_part_ if len(nested_must_not_part) > 0: - nested_must_not_part_ = ",".join(nested_must_not_part) nested_must_not_part_ = must_not_term_template.substitute( must_not_term=nested_must_not_part_ diff --git a/omero_search_engine/cache_functions/elasticsearch/transform_data.py b/omero_search_engine/cache_functions/elasticsearch/transform_data.py index 7a3cff01..279a7d8e 100644 --- a/omero_search_engine/cache_functions/elasticsearch/transform_data.py +++ b/omero_search_engine/cache_functions/elasticsearch/transform_data.py @@ -338,7 +338,7 @@ def get_file_list(path_name): from os import walk f = [] - for (dirpath, dirnames, filenames) in walk(path_name): + for dirpath, dirnames, filenames in walk(path_name): f.extend(filenames) return f diff --git a/omero_search_engine/validation/psql_templates.py b/omero_search_engine/validation/psql_templates.py index 807076d1..26cc83ef 100644 --- a/omero_search_engine/validation/psql_templates.py +++ b/omero_search_engine/validation/psql_templates.py @@ -212,7 +212,6 @@ def substitute(self, **kwargs): and lower(annotation_mapvalue.value) =lower('$value')""" ) - head_space_query = Template( """ select image.id as image_id, screen.name as screen_name, project.name as project_name, @@ -274,5 +273,41 @@ def substitute(self, **kwargs): $condition group by project_name, screen_name,image.id, annotation_mapvalue.name, annotation_mapvalue.value HAVING COUNT(*)>1 + """ +) + + +project_key_values = Template( + """ +Select DISTINCT (annotation_mapvalue.value) from image +left join imageannotationlink on image.id =imageannotationlink.parent +left join annotation_mapvalue on +annotation_mapvalue.annotation_id=imageannotationlink.child +inner join datasetimagelink on datasetimagelink.child=image.id +inner join dataset on datasetimagelink.parent=dataset.id +inner join projectdatasetlink on dataset.id=projectdatasetlink.child +inner join project on project.id=projectdatasetlink.parent +where project.id= $id and lower (annotation_mapvalue.name) =lower('$name'); +""" +) + +screen_key_values = Template( + """ +Select DISTINCT (annotation_mapvalue.value) from image +left join imageannotationlink on image.id =imageannotationlink.parent +left join annotation_mapvalue on +annotation_mapvalue.annotation_id=imageannotationlink.child +inner join wellsample on wellsample.image=image.id +inner join well on wellsample.well= well.id +inner join plate on well.plate=plate.id +inner join screenplatelink on plate.id=screenplatelink.child +inner join screen on screen.id=screenplatelink.parent +where screen.id =$id and lower (annotation_mapvalue.name) =lower('$name'); + """ +) + +container_from_name = Template( + """ +select id from $container where name like '%$name%'; """ ) diff --git a/omero_search_engine/validation/results_validator.py b/omero_search_engine/validation/results_validator.py index 2eca8f50..1710eaa1 100644 --- a/omero_search_engine/validation/results_validator.py +++ b/omero_search_engine/validation/results_validator.py @@ -1032,13 +1032,10 @@ def get_no_images_sql_containers(write_report=True): if write_report: base_folder = "/etc/searchengine/" if not os.path.isdir(base_folder): - base_folder = os.path.expanduser("~") - - report_file = os.path.join(base_folder, "check_containers_report.txt") + base_folder = os.path.expanduser("~") report_file = os.path.join(base_folder, "check_containers_report.txt") report = "\n".join(messages) # noqa with open(report_file, "w") as f: - f.write(report) - + f.write(report) """ def set_owner_ship(resource , name, value, owener_id=None, group_id=None): @@ -1050,3 +1047,92 @@ def set_owner_ship(resource , name, value, owener_id=None, group_id=None): if group_id: sql = sql + " %s.%group_id=%s" % (resource, group_id) """ + +def check_container_keys_vakues(): + # This will be modified and the testing data will be adjusted and provided + # at run time + from omero_search_engine.validation.psql_templates import ( + container_from_name, + screen_key_values, + project_key_values, + ) + import json + from omero_search_engine.api.v1.resources.resource_analyser import ( + get_container_values_for_key, + ) + + csv = False + container_names = ["idr0034", "idr0114"] + keys = ["gene symbol", "cell line"] + for container_name in container_names: + for key in keys: + project_sql = container_from_name.substitute( + container="project", name=container_name + ) + screen_sql = container_from_name.substitute( + container="screen", name=container_name + ) + conn = search_omero_app.config["database_connector"] + project_ids_results = conn.execute_query(project_sql) + screen_ids_results = conn.execute_query(screen_sql) + + search_omero_app.logger.info("projects: %s" % project_ids_results) + search_omero_app.logger.info("screens: %s" % screen_ids_results) + + if len(screen_ids_results) > 0: + for id in screen_ids_results: + screen_sql = screen_key_values.substitute(id=id.get("id"), name=key) + screen_results = conn.execute_query(screen_sql) + scr_searchengine_results = get_container_values_for_key( + "image", container_name, csv, key + ) + if len(scr_searchengine_results.response) > 0: + scr_searchengine_results = json.loads( + scr_searchengine_results.response[0] + ) + else: + scr_searchengine_results = scr_searchengine_results.response + search_omero_app.logger.info( + "Results from PostgreSQL database: %s" % len(screen_results) + ) + if len(scr_searchengine_results) > 0 and scr_searchengine_results[ + 0 + ].get("results"): + search_omero_app.logger.info( + "Searchengine results: %s" + % len(scr_searchengine_results[0].get("results")) + ) + else: + search_omero_app.logger.info( + "No results returned from searchengine" + ) + if len(project_ids_results) > 0: + for id in project_ids_results: + project_sql = project_key_values.substitute( + id=id.get("id"), name=key + ) + project_results = conn.execute_query(project_sql) + pr_searchengine_results = get_container_values_for_key( + "image", container_name, csv, key + ) + if len(pr_searchengine_results.response) > 0: + pr_searchengine_results = json.loads( + pr_searchengine_results.response[0] + ) + else: + pr_searchengine_results = pr_searchengine_results.response + + search_omero_app.logger.info( + "Results from PostgreSQL database: %s" % len(project_results) + ) + if len(pr_searchengine_results) > 0 and pr_searchengine_results[ + 0 + ].get("results"): + search_omero_app.logger.info( + "Searchengine results: %s " + % len(pr_searchengine_results[0].get("results")) + ) + else: + search_omero_app.logger.info( + "No results returned from searchengine" + )