Merge branch 'main' into modify_unit_test_validation

.github/workflows/main.yml

@@ -24,10 +24,14 @@ jobs:
           --health-retries 5
 
       elasticsearch:
-        image: elasticsearch:7.16.2
+        image: elasticsearch:8.8.1
         ports:
           - 9200/tcp
-        options: -e="discovery.type=single-node" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
+        env:
+          es_api_basic_auth_username: "elastic"
+          ELASTIC_PASSWORD: "elasticsearch_user_password"
+
+        options: -e="discovery.type=single-node" --health-cmd="curl -k -u elastic:elasticsearch_user_password  https://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
     steps:
       - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.1.0
     hooks:
       - id: black
         args: [--target-version=py35]

.readthedocs.yml

@@ -0,0 +1,23 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build documentation with MkDocs
+#mkdocs:
+#  configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"

CHANGELOG.md

@@ -1,3 +1,13 @@
+0.5.3 (September 2023):
+-----------------------
+- Secure the connection with elasticsearch [#92](https://github.com/ome/omero_search_engine/pull/92)
+
+
+0.5.2 (June 2023):
+------------------
+- Return all the available values for a key in a container [#77](https://github.com/ome/omero_search_engine/pull/77)
+- Return the available attributes with a container [#77](https://github.com/ome/omero_search_engine/pull/77)
+
 0.5.1 (February 2023):
 ----------------------
 - Fix the issue of filtering images by the container name [#81](https://github.com/ome/omero_search_engine/pull/81)

README.rst

@@ -1,6 +1,10 @@
 .. image:: https://github.com/ome/omero_search_engine/workflows/Build/badge.svg
    :target: https://github.com/ome/omero_search_engine/actions
 
+.. image:: https://readthedocs.org/projects/omero-search-engine/badge/?version=latest
+    :target: https://omero-search-engine.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
 OMERO Search Engine
 --------------------
 

configurations/app_config.yml

@@ -6,9 +6,11 @@ DATABASE_NAME : "omero"
 CACHE_FOLDER : "path/to/folder/app_data"
 SECRET_KEY : "fsdasdh3424vvcsd467fgh"
 ASYNCHRONOUS_SEARCH : True
-ELASTICSEARCH_URL : "http://localhost:9200"
+ELASTICSEARCH_URL : "https://localhost:9200"
 IDR_TEST_FILE_URL : "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv"
 PAGE_SIZE : 1000
 CACHE_ROWS : 10000
 MAX_RETUNED_ITEMS : 1700000
 ELASTICSEARCH_BACKUP_FOLDER: "path/to/elasticsearch/backup/folder"
+verify_certs: False
+ELASTIC_PASSWORD: elasticsearch_user_password

configurations/configuration.py

@@ -20,6 +20,7 @@
 import yaml
 from shutil import copyfile
 import os
+import json
 
 
 def load_configuration_variables_from_file(config):
@@ -29,6 +30,20 @@ def load_configuration_variables_from_file(config):
         cofg = yaml.load(f)
     for x, y in cofg.items():
         setattr(config, x, y)
+    if hasattr(config, "verify_certs"):
+        try:
+            verify_certs = json.load(config.verify_certs)
+        except Exception as ex:
+            print(str(ex))
+            verify_certs = False
+    else:
+        verify_certs = False
+    config.verify_certs = verify_certs
+    if not verify_certs:
+        import requests
+        from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
 
 def set_database_connection_variables(config):

docs/configuration/configuration_installation.rst

@@ -14,10 +14,12 @@ The application should have the access attributes (e.g, URL, username, password,
       * ``DATABASE_NAME``
       * ``ELASTICSEARCH__URL``
       * ``PAGE_SIZE``
+      * ``ELASTIC_PASSWORD``
     * Although the user can edit this file to set the values, there are some methods inside :omero_search_engine:`manage.py <manage.py>` which could help to set the configuration e.g.
 
       * ``set_database_configuration``
       * ``set_elasticsearch_configuration``
+      * ``set_elasticsearch_password``
 
 * When the app runs for the first time, it will look for the application configuration file.
 

docs/developer/developer.rst

@@ -1,2 +1,20 @@
 Developer's documents
-=====================
+=====================
+
+The developer should clone the code from the project repo using the following command::
+
+    git clone https://github.com/ome/omero_search_engine.git
+
+Then they need to create a Python virtual environment variable using either venv or conda and install the packages inside requirements.txt
+
+The developer needs to set up the application configuration as it is explained in the System configuration part inside "docs\configuration\configuration_installation.rst"
+
+After that, they should run the indexer to index Omero's data using the following command::
+
+    python manage.py get_index_data_from_database
+
+The developer can run the application using the following command::
+
+    python manage.py runserver -p 5577
+
+Running the scripts inside the examples folder can be a good starting point.

examples/container_key_values.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2023 University of Dundee & Open Microscopy Environment.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from utils import base_url
+import requests
+import json
+import logging
+
+"""
+Return the available  keys in a containers
+Also get the aviable values for a key """
+
+resource = "image"
+container_name = "idr0034"
+key = "cell line"
+
+# the following url will return the existing key in this container
+keys_url = (
+    "{base_url}resources/image/container_keys/?container_name={container_name}".format(
+        base_url=base_url, container_name=container_name
+    )
+)
+
+resp = requests.get(url=keys_url)
+keys_results = json.loads(resp.text)
+for result in keys_results:
+    logging.info("%s: %s" % (result.get("type"), result.get("name")))
+    for bucket in result.get("results"):
+        logging.info(
+            "Key: %s, no of images: %s " % (bucket.get("key"), bucket.get("no_image"))
+        )
+
+""" It is possible to get all the available
+values for a key
+"""
+values_key_url = (
+    "{base_url}resources/image/"
+    "container_keyvalues/?container_name={container_name}&key={key}".format(
+        base_url=base_url, container_name=container_name, key=key
+    )
+)
+
+resp = requests.get(url=values_key_url)
+
+key_values_results = json.loads(resp.text)
+
+for result in key_values_results:
+    logging.info("%s: %s" % (result.get("type"), result.get("name")))
+    for bucket in result.get("results"):
+        logging.info(
+            "Key: %s, value: %s, no of images: %s "
+            % (bucket.get("key"), bucket.get("value"), bucket.get("no_image"))
+        )

examples/search_with_bookmark_paging_using_submitquery.py

@@ -118,7 +118,6 @@ def call_omero_searchengine_return_results(url, data=None, method="post"):
 )
 
 while len(received_results) < total_results:
-
     page += 1
     query_data_ = {"query_details": {"and_filters": and_filters}, "bookmark": bookmark}
     query_data_json_ = json.dumps(query_data_)

manage.py

@@ -205,6 +205,24 @@ def set_elasticsearch_configuration(elasticsearch_url=None):
         search_omero_app.logger.info("No attribute is provided")
 
 
+@manager.command
+@manager.option("-e", "--elasticsearch_password", help="set elasticsearch password")
+def set_elasticsearch_password(elasticsearch_password=None):
+    if elasticsearch_password:
+        update_config_file({"ELASTIC_PASSWORD": elasticsearch_password})
+    else:
+        search_omero_app.logger.info("No attribute is provided")
+
+
+@manager.command
+@manager.option("-v", "--verify_certs", help="set elasticsearch password")
+def set_verify_certs(verify_certs=None):
+    if verify_certs:
+        update_config_file({"verify_certs": verify_certs})
+    else:
+        search_omero_app.logger.info("No attribute is provided")
+
+
 @manager.command
 @manager.option("-c", "--cache_folder", help="cache folder path")
 def set_cache_folder(cache_folder=None):
@@ -386,6 +404,15 @@ def data_validator(screen_name=None, project_name=None):
     print(start2)
     print(end)
 
+@manager.command
+def test_container_key_value():
+    from omero_search_engine.validation.results_validator import (
+        check_container_keys_vakues,
+    )
+
+    check_container_keys_vakues()
+
+
 
 if __name__ == "__main__":
     manager.run()

omero_search_engine/__init__.py

@@ -22,7 +22,6 @@
 import logging
 from elasticsearch import Elasticsearch
 from flasgger import Swagger, LazyString, LazyJSONEncoder
-
 from omero_search_engine.database.database_connector import DatabaseConnector
 from configurations.configuration import (
     configLooader,
@@ -54,7 +53,6 @@
     "version": "0.2.0",
 }
 
-
 swagger = Swagger(search_omero_app, template=template)
 
 app_config = load_configuration_variables_from_file(config_)
@@ -72,12 +70,17 @@ def create_app(config_name="development"):
     search_omero_app.app_context().push()
     search_omero_app.app_context()
     search_omero_app.app_context().push()
+    ELASTIC_PASSWORD = app_config.ELASTIC_PASSWORD
+
     es_connector = Elasticsearch(
         app_config.ELASTICSEARCH_URL.split(","),
+        verify_certs=app_config.verify_certs,
         timeout=130,
         max_retries=20,
         retry_on_timeout=True,
         connections_per_node=10,
+        scheme="https",
+        http_auth=("elastic", ELASTIC_PASSWORD),
     )
 
     search_omero_app.config["database_connector"] = database_connector
@@ -127,7 +130,6 @@ def after_request(response):
 # added to let the user know the proper extension they should use
 @search_omero_app.errorhandler(404)
 def page_not_found(error):
-
     search_omero_app.logger.info("Error: %s" % error)
     resp_message = (
         "%s, You may use '/searchengine/api/v1/resources/' to test\