Merge branch 'main' into search_by_value_only

.github/workflows/main.yml

@@ -24,10 +24,14 @@ jobs:
           --health-retries 5
 
       elasticsearch:
-        image: elasticsearch:7.16.2
+        image: elasticsearch:8.8.1
         ports:
           - 9200/tcp
-        options: -e="discovery.type=single-node" --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
+        env:
+          es_api_basic_auth_username: "elastic"
+          ELASTIC_PASSWORD: "elasticsearch_user_password"
+
+        options: -e="discovery.type=single-node" --health-cmd="curl -k -u elastic:elasticsearch_user_password  https://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
 
     steps:
       - uses: actions/checkout@v2
@@ -44,6 +48,13 @@ jobs:
           python manage.py set_database_configuration -u localhost -s  ${{ job.services.postgres.ports[5432] }} -n postgress -p passwprd
           # configure elasticsearch
           python manage.py set_elasticsearch_configuration -e localhost:${{ job.services.elasticsearch.ports[9200] }}
+          # download and extract the database backup file           
+          wget https://downloads.openmicroscopy.org/images/omero_db_searchengine.zip -P app_data
+          unzip app_data/omero_db_searchengine.zip -d app_data/          
+          # run  restore omero database
+          python manage.py restore_postgresql_database
+          # run indexing indexing
+          python manage.py get_index_data_from_database -b False
           # run tests
           python -m unittest discover -s unit_tests
   upload:
@@ -81,8 +92,6 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          file: deployment/docker/centos/Dockerfile
+          file: deployment/docker/rockylinux/Dockerfile
           push: true
           tags: ${{ join(fromJson(steps.gettags.outputs.tags)) }}
-
-

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 ---
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 24.4.2
     hooks:
       - id: black
         args: [--target-version=py35]
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 7.1.0
     hooks:
       - id: flake8
         args: [

.readthedocs.yml

@@ -0,0 +1,23 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# Build documentation with MkDocs
+#mkdocs:
+#  configuration: mkdocs.yml
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats: all
+
+# Optionally set the version of Python and requirements required to build your docs
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"

CHANGELOG.md

@@ -1,3 +1,16 @@
+0.5.4 (March 2024):
+--------------------
+- Support Rocky Linux 9  [#93](https://github.com/ome/omero_search_engine/pull/93)
+
+0.5.3 (September 2023):
+-----------------------
+- Secure the connection with elasticsearch [#92](https://github.com/ome/omero_search_engine/pull/92)
+
+0.5.2 (June 2023):
+------------------
+- Return all the available values for a key in a container [#77](https://github.com/ome/omero_search_engine/pull/77)
+- Return the available attributes with a container [#77](https://github.com/ome/omero_search_engine/pull/77)
+
 0.5.1 (February 2023):
 ----------------------
 - Fix the issue of filtering images by the container name [#81](https://github.com/ome/omero_search_engine/pull/81)

README.rst

@@ -1,6 +1,10 @@
 .. image:: https://github.com/ome/omero_search_engine/workflows/Build/badge.svg
    :target: https://github.com/ome/omero_search_engine/actions
 
+.. image:: https://readthedocs.org/projects/omero-search-engine/badge/?version=latest
+    :target: https://omero-search-engine.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
 OMERO Search Engine
 --------------------
 

app_data/test_index_data.json

@@ -108,5 +108,27 @@
             "validation screen"
          ]
       ]
-   }
+   },
+    "query_in": {
+       "image": [
+         [
+          "Gene Symbol",
+          [
+             "Duoxa2",
+             "Bach2",
+             "Cxcr2",
+             "Mysm1"
+          ]
+          ],
+          [
+          "Organism",
+          [
+             "homo sapiens",
+             "mus musculus",
+             "mus musculus x mus spretus",
+             "human adenovirus 2"
+          ]
+       ]
+          ]
+    }
 }

configurations/app_config.yml

@@ -6,9 +6,12 @@ DATABASE_NAME : "omero"
 CACHE_FOLDER : "path/to/folder/app_data"
 SECRET_KEY : "fsdasdh3424vvcsd467fgh"
 ASYNCHRONOUS_SEARCH : True
-ELASTICSEARCH_URL : "http://localhost:9200"
+ELASTICSEARCH_URL : "https://localhost:9200"
 IDR_TEST_FILE_URL : "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv"
 PAGE_SIZE : 1000
 CACHE_ROWS : 10000
 MAX_RETUNED_ITEMS : 1700000
 ELASTICSEARCH_BACKUP_FOLDER: "path/to/elasticsearch/backup/folder"
+verify_certs: False
+ELASTIC_PASSWORD: elasticsearch_user_password
+BASE_FOLDER: /etc/searchengine/

configurations/configuration.py

@@ -20,6 +20,7 @@
 import yaml
 from shutil import copyfile
 import os
+import json
 
 
 def load_configuration_variables_from_file(config):
@@ -29,6 +30,20 @@ def load_configuration_variables_from_file(config):
         cofg = yaml.load(f)
     for x, y in cofg.items():
         setattr(config, x, y)
+    if hasattr(config, "verify_certs"):
+        try:
+            verify_certs = json.load(config.verify_certs)
+        except Exception as ex:
+            print(str(ex))
+            verify_certs = False
+    else:
+        verify_certs = False
+    config.verify_certs = verify_certs
+    if not verify_certs:
+        import requests
+        from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+        requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
 
 def set_database_connection_variables(config):

deployment/docker/rockylinux/.dockerignore

@@ -0,0 +1,3 @@
+Dockerfile
+.git
+.gitignore

deployment/docker/rockylinux/Dockerfile

@@ -0,0 +1,23 @@
+#docker build . -t searchengine
+# docker build . -f deployment/docker/rockylinux/Dockerfile -t searchengine
+FROM rockylinux/rockylinux:9.0
+USER root
+RUN dnf update -y
+RUN dnf groupinstall "Development Tools" -y
+RUN  dnf install libpq-devel -y
+RUN dnf install python3-pip -y
+RUN dnf install -y   python3-devel.x86_64
+RUN dnf clean all && rm -rf /var/cache/yum
+RUN mkdir /searchengine
+ADD  deployment/docker/rockylinux/start_gunicorn_serch_engine.sh /searchengine
+ADD  deployment/docker/rockylinux/run_app.sh /searchengine
+ADD .  /searchengine
+RUN cd /searchengine
+RUN mkdir /etc/searchengine
+RUN mkdir /etc/searchengine/chachedata
+RUN mkdir /etc/searchengine/logs
+WORKDIR /searchengine
+RUN pip3 install -r  requirements.txt
+RUN pip3 install gunicorn
+EXPOSE 5577
+ENTRYPOINT ["bash", "run_app.sh"]

deployment/docker/rockylinux/run_app.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+echo "$@"
+
+#test if the configuration file exists, if not it will copy it from the app configuration folder
+test -f /etc/searchengine/.app_config.yml || cp /searchengine/configurations/app_config.yml /etc/searchengine/.app_config.yml
+
+#Check the script input
+if [[ $@ == run_app* ]] ; then
+  url_perfix=${@/run_app/}
+  echo using prefix: $url_perfix
+  bash start_gunicorn_serch_engine.sh $url_perfix
+elif [ -z  "$@" ] || [ "$@" = "run_app" ]; then
+  echo "Starting the app"
+  bash start_gunicorn_serch_engine.sh
+else
+  echo "$@"
+  python3 manage.py "$@"
+fi

deployment/docker/rockylinux/start_gunicorn_serch_engine.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+NAME="omero_search_engine"
+USER root
+APPPATH=/searchengine
+SOCKFILE=/etc/searchengine/sock3 #change this to project_dir/sock (new file will be created)
+echo "Starting $NAME as `whoami`"
+export PATH="$APPPATH:$PATH"
+echo "staring the app"
+# Create the run directory if it doesn't exist
+RUNDIR=$(dirname $SOCKFILE)
+echo "$RUNDIR"
+test -d $RUNDIR || mkdir -p $RUNDIR
+LOGS=/etc/searchengine/logs
+LOGSDIR=$(dirname $LOGS)
+test -d $LOGSDIR || mkdir -p $LOGSDIR
+user=$USER
+echo "Start Gunicorn ...."
+echo "$HOME"
+echo pwd
+cd $APPPATH
+if [ -z  "$@" ]; then
+  exec gunicorn "omero_search_engine:create_app('production')" -b 0.0.0.0:5577 --timeout 0 --name "$NAME"   --bind=unix:$SOCKFILE  --log-file=$LOGSDIR/logs/engine_gunilog.log --access-logfile=$LOGSDIR/logs/engine_access.log -error-logfile=$LOGSDIR/logs/engine_logs/engine_error.log  --workers 4
+else
+  echo Run with SCRIPT_NAME=$@
+  SCRIPT_NAME=/"$@"/ exec gunicorn "omero_search_engine:create_app('production')" -b 0.0.0.0:5577 --timeout 0 --name "$NAME"   --bind=unix:$SOCKFILE  --log-file=$LOGSDIR/logs/engine_gunilog.log --access-logfile=$LOGSDIR/logs/engine_access.log -error-logfile=$LOGSDIR/logs/engine_logs/engine_error.log  --workers 4
+fi

docs/configuration/configuration_installation.rst

@@ -14,10 +14,12 @@ The application should have the access attributes (e.g, URL, username, password,
       * ``DATABASE_NAME``
       * ``ELASTICSEARCH__URL``
       * ``PAGE_SIZE``
+      * ``ELASTIC_PASSWORD``
     * Although the user can edit this file to set the values, there are some methods inside :omero_search_engine:`manage.py <manage.py>` which could help to set the configuration e.g.
 
       * ``set_database_configuration``
       * ``set_elasticsearch_configuration``
+      * ``set_elasticsearch_password``
 
 * When the app runs for the first time, it will look for the application configuration file.
 

docs/developer/developer.rst

@@ -1,2 +1,20 @@
 Developer's documents
-=====================
+=====================
+
+The developer should clone the code from the project repo using the following command::
+
+    git clone https://github.com/ome/omero_search_engine.git
+
+Then they need to create a Python virtual environment variable using either venv or conda and install the packages inside requirements.txt
+
+The developer needs to set up the application configuration as it is explained in the System configuration part inside "docs\configuration\configuration_installation.rst"
+
+After that, they should run the indexer to index Omero's data using the following command::
+
+    python manage.py get_index_data_from_database
+
+The developer can run the application using the following command::
+
+    python manage.py runserver -p 5577
+
+Running the scripts inside the examples folder can be a good starting point.

examples/container_key_values.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2023 University of Dundee & Open Microscopy Environment.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from utils import base_url
+import requests
+import json
+import logging
+
+"""
+Return the available  keys in a containers
+Also get the aviable values for a key """
+
+resource = "image"
+container_name = "idr0034"
+key = "cell line"
+
+# the following url will return the existing key in this container
+keys_url = (
+    "{base_url}resources/image/container_keys/?container_name={container_name}".format(
+        base_url=base_url, container_name=container_name
+    )
+)
+
+resp = requests.get(url=keys_url)
+keys_results = json.loads(resp.text)
+for result in keys_results:
+    logging.info("%s: %s" % (result.get("type"), result.get("name")))
+    for bucket in result.get("results"):
+        logging.info(
+            "Key: %s, no of images: %s " % (bucket.get("key"), bucket.get("no_image"))
+        )
+
+""" It is possible to get all the available
+values for a key
+"""
+values_key_url = (
+    "{base_url}resources/image/"
+    "container_keyvalues/?container_name={container_name}&key={key}".format(
+        base_url=base_url, container_name=container_name, key=key
+    )
+)
+
+resp = requests.get(url=values_key_url)
+
+key_values_results = json.loads(resp.text)
+
+for result in key_values_results:
+    logging.info("%s: %s" % (result.get("type"), result.get("name")))
+    for bucket in result.get("results"):
+        logging.info(
+            "Key: %s, value: %s, no of images: %s "
+            % (bucket.get("key"), bucket.get("value"), bucket.get("no_image"))
+        )

examples/search.py

@@ -88,5 +88,9 @@ def call_omero_return_results(url, data=None, method="post"):
         % (len(received_results), total_results, page, total_pages, bookmark)
     )
 
-# 2000 /11686633, page: 1/11687, bookmark: 109600
-# 2000 /12225067, page: 1/12226, bookmark:  109600
+# another example using in operators and send items inside value as a string,
+# The List items are separated by ','
+logging.info("Using in operator")
+url = "%s%s?key=Gene Symbol&value=Pdgfc,Rnase10&operator=in" % (base_url, image_search)
+bookmark, total_results, total_pages = call_omero_return_results(url, method="get")
+logging.info("%s,%s" % (total_results, total_pages))

examples/search_with_bookmark_paging_using_submitquery.py

@@ -118,7 +118,6 @@ def call_omero_searchengine_return_results(url, data=None, method="post"):
 )
 
 while len(received_results) < total_results:
-
     page += 1
     query_data_ = {"query_details": {"and_filters": and_filters}, "bookmark": bookmark}
     query_data_json_ = json.dumps(query_data_)