diff --git a/deployment/ansible/check_indexing_service.yml b/deployment/ansible/check_indexing_service.yml deleted file mode 100644 index b14beb87..00000000 --- a/deployment/ansible/check_indexing_service.yml +++ /dev/null @@ -1,22 +0,0 @@ -#Issue setup ip address inside the hpa config file for postgres to accept the connection from it -- name: Check indexing - connection: local - hosts: local - vars_files: - searchengine_vars.yml - tasks: - - name: Get infos on container - docker_container_info: - name: "{{ searchengine_index }}" - register: result - - - name: does Indexing exisit? - debug: - msg: "The indexing container {{ 'exists' if result.exists else 'does not exist' }}" - - - name: did Indexing finish? - debug: - msg: "The indexing container status is {{ result.container['State']['Status'] }}" - when: result.exists - - diff --git a/deployment/ansible/deploy_elasticsearch_cluster.yml b/deployment/ansible/deploy_elasticsearch_cluster.yml deleted file mode 100644 index 9f3ab3ee..00000000 --- a/deployment/ansible/deploy_elasticsearch_cluster.yml +++ /dev/null @@ -1,273 +0,0 @@ -#Deploy Elasticsearch cluster -tasks: -- name: Create app top level directory - become: yes - file: - path: "{{ apps_folder }}/searchengine" - recurse: yes - state: directory - owner: root - group: root - mode: 0755 - -- name: Create elasticsearch directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - mode: 0755 - -- name: Create elasticsearch certs folder - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch/certs" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - mode: 0755 - - -- name: Create elasticsearch main nodes directories - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - mode: 0755 - with_sequence: start=1 count={{ elasticsearch_no_nodes }} - -- name: Create elasticsearch logs directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/logs" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - mode: 0755 - with_sequence: start=1 count={{ elasticsearch_no_nodes }} - -- name: Create elasticsearch data directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/data" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - mode: 0755 - with_sequence: start=1 count={{ elasticsearch_no_nodes }} - -- name: Create ELasticsearch backup folder - become: yes - file: - path: "{{ elasticsearch_backup_folder }}" - recurse: yes - state: directory - owner: 1000 - group: root - mode: 0755 - - - -- name: Add elastic nodes to instances_nodes - set_fact: - instances_nodes: "{{instances_nodes + [( {'name' : 'searchengine_elasticsearch_node'+item, 'dns': ['searchengine_elasticsearch_node'+item,'localhost'],'ip': '127.0.0.1'})] }}" - - with_sequence: start=1 count={{ elasticsearch_no_nodes }} - - -- name: Add elastic nodes to instances - set_fact: - instances: "{{instances | combine ( {'instances' : instances_nodes}) }}" - - -#Add all elasticsearch nodes -- name: Add elastic nodes to elasticsearch_nodes - set_fact: - elasticsearch_nodes: '{{ elasticsearch_nodes + ["searchengine_elasticsearch_node"+item] }}' - with_sequence: start=1 count={{ elasticsearch_no_nodes }} - -- name: Create docker network - become: yes - docker_network: - name: searchengine-net - ipam_config: - - subnet=10.12.0.0/16 - -# I got some memory exceptions when start using elasticsearch cluster and -# increasing the mmap counts limits fix this issue -#https://www.elastic.co/guide/en/elasticsearch/reference/7.16 /vm-max-map-count.html -- name: set vm.max_map_count to 262144 in sysctl - become: yes - sysctl: name={{ item.varname }} value={{ item.varvalue }} - with_items: - - { varname: "vm.max_map_count", varvalue: "262144" } - -- name: create CA - become: yes - docker_container: - image: "{{ search_engineelasticsearch_docker_image }}" - name: "creat_ca_for_elasticsearch_cluster" - cleanup: True - command: "bash -c '/usr/share/elasticsearch/bin/elasticsearch-certutil ca -s -out /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --pass {{ ca_password }} - '" - #fi; - state: started - volumes: - - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs" - - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates" - -- name: Wait for CA file - ansible.builtin.wait_for: - path: "{{ apps_folder }}/searchengine/elasticsearch/certs/elastic-ca.p12" - state: present - register: check_ca_file_result - -- name: Show wait CA file result - debug: - var: check_ca_file_result - -- copy: - dest: /tmp/instances.yaml - content: | - {{ instances |to_nice_yaml }} - - - -- name: Create nodes' cert - become: yes - docker_container: - image: "{{ search_engineelasticsearch_docker_image }}" - name: "creat_ca_for_elasticsearch_cluster" - cleanup: True - command: "bash -c 'bin/elasticsearch-certutil cert --ca /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --ca-pass {{ ca_password }} --pass {{ keystore_password }} --in /tmp/instances.yaml -out /usr/share/elasticsearch/config/certificates/bundle.zip; - echo 'done'; - unzip /usr/share/elasticsearch/config/certificates/bundle.zip -d /usr/share/elasticsearch/config/certificates/; - '" - - state: started - volumes: - - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs" - - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates" - - /tmp/instances.yaml:/tmp/instances.yaml - - - -- name: Run first docker elasticsearch main node - become: yes - docker_container: - image: "{{ search_engineelasticsearch_docker_image }}" - name: "searchengine_elasticsearch_node1" - cleanup: True - ulimits: - - 'memlock:-1:-1' - env: - path.data: "/var/lib/elasticsearch" - path.logs: "/var/log/elasticsearch" - path.repo: "{{ elasticsearch_backup_folder }}" - node.name: searchengine_elasticsearch_node1 - bootstrap.memory_lock: "true" - network.host: 0.0.0.0 - cluster.name: "searchengine-cluster" - cluster.initial_master_nodes: "{{ elasticsearch_nodes | join(',') }}" - http.host: 0.0.0.0 - #http.port: 9200 - ES_JAVA_OPTS: "-Xms2g -Xmx2g" - ingest.geoip.downloader.enabled: "false" - ######################################## - es_api_basic_auth_username: "elastic" - ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}" - es_validate_certs: "no" - es_enable_http_ssl: "true" - xpack.security.http.ssl.enabled: "true" - xpack.security.enabled: "true" - xpack.security.authc.realms.file.file1.order: "0" - xpack.security.authc.realms.native.native1.order: "1" - xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12" - xpack.security.http.ssl.truststore.password: "{{ ca_password }}" - xpack.security.http.ssl.keystore.password: "{{ ca_password }}" - xpack.security.transport.ssl.enabled: "true" - xpack.security.transport.ssl.verification_mode: "certificate" - xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12" - xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12" - xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}" - xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}" - ######################################################################### - networks: - - name: "searchengine-net" - ipv4_address: 10.12.0.2 - published_ports: - - "9201:9200" - - "9301:9300" - state: started - restart_policy: always - volumes: - - "{{ apps_folder }}/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch" - - "{{ apps_folder }}/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch" - - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates" - - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}" - - -- name: Run docker elasticsearch for the remaining nodes - become: yes - docker_container: - image: "{{ search_engineelasticsearch_docker_image }}" - name: "searchengine_elasticsearch_node{{ item }}" - ulimits: - - 'memlock:-1:-1' - cleanup: True - env: - path.data: "/var/lib/elasticsearch" - path.logs: "/var/log/elasticsearch" - path.repo: "{{ elasticsearch_backup_folder }}" - node.name: "searchengine_elasticsearch_node{{ item }}" - bootstrap.memory_lock: "true" - network.host: 0.0.0.0 - cluster.name: "searchengine-cluster" - discovery.seed_hosts: "searchengine_elasticsearch_node1" - cluster.initial_master_nodes: "{{ elasticsearch_nodes | join(',') }}" - http.host: 0.0.0.0 - #http.port: 9200 - ES_JAVA_OPTS: "-Xms1g -Xmx1g" - ingest.geoip.downloader.enabled: "false" - #################################################################### - es_api_basic_auth_username: "elastic" - ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}" - es_validate_certs: "no" - es_enable_http_ssl: "true" - xpack.security.http.ssl.enabled: "true" - xpack.security.enabled: "true" - xpack.security.authc.realms.file.file1.order: "0" - xpack.security.authc.realms.native.native1.order: "1" - xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12" - xpack.security.http.ssl.truststore.password: "{{ ca_password }}" - xpack.security.http.ssl.keystore.password: "{{ ca_password }}" - xpack.security.transport.ssl.enabled: "true" - xpack.security.transport.ssl.verification_mode: "certificate" - xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12" - xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12" - xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}" - xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}" - #################################################################### - - networks: - - name: "searchengine-net" - ipv4_address: 10.12.0.{{ item | int + 1 }} - published_ports: - - "920{{ item }}:9200" - - "930{{ item }}:9300" - state: started - restart_policy: always - volumes: - - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/data:/var/lib/elasticsearch" - - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/logs:/var/log/elasticsearch" - - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}" - - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates" - with_sequence: start=2 count={{ elasticsearch_no_nodes | int -1 }} diff --git a/deployment/ansible/deploy_searchengine.yml b/deployment/ansible/deploy_searchengine.yml deleted file mode 100644 index 28c32512..00000000 --- a/deployment/ansible/deploy_searchengine.yml +++ /dev/null @@ -1,159 +0,0 @@ -#Deploy Search Engine - -tasks: -- name: Get database host - set_fact: - database_server_url: - -- name: Create searchengine folder directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/searchengine" - recurse: yes - state: directory - owner: root - group: root - mode: 0755 - -- name: Create searchengine logs directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/searchengine/logs" - state: directory - mode: 0755 - -- name: Create searchengine cached directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/searchengine/cacheddata" - state: directory - mode: 0755 - -#Add all elasticsearch nodes -- name: Add elastic nodes to elasticsearch_nodes_urls - set_fact: - elasticsearch_nodes_urls: '{{ elasticsearch_nodes_urls + ["https://10.12.0."+item+":9200"] }}' - with_sequence: start=2 count={{ elasticsearch_no_nodes }} - -- name: configure elasticsearch backup folder for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_elasticsearch - cleanup: True - command: "set_elasticsearch_backup_folder -b {{ elasticsearch_backup_folder }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - -- name: configure elasticsearch for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_elasticsearch - cleanup: True - command: "set_elasticsearch_configuration -e {{ elasticsearch_nodes_urls | join(',') }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - -- name: configure database for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_database - cleanup: True - #auto_remove: yes - command: > - set_database_configuration -u {{ database_server_url }} - -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }} - #networks: - #- name: searchengine-net - #published_ports: - #- "5577:5577" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - -- name: configure IDR_TEST_FILE_URL item - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_IDR_TEST_FILE - cleanup: True - command: "set_idr_test_file -i {{ IDR_TEST_FILE_URL }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - -- name: configure cache folder for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_cache - cleanup: True - #auto_remove: yes - command: "set_cache_folder -c /etc/searchengine/cachedata" - #networks: - #- name: searchengine-net - #published_ports: - #- "5577:5577" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - -- name: configure number of cache rows for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_cache - cleanup: True - #auto_remove: yes - command: "set_cache_rows_number -n {{ cache_rows }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - -- name: configure secret key for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_cache - cleanup: True - #auto_remove: yes - command: "set_searchengine_secret_key -s {{ searchengine_secret_key }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - -- name: configure elastics search password - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: elastics_password - cleanup: True - #auto_remove: yes - command: "set_elasticsearch_password -e {{ ELASTIC_PASSWORD }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - -- name: Run docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine - cleanup: True - command: "run_app {{ searchengineurlprefix }}" - networks: - - name: searchengine-net - published_ports: - - "5577:5577" - #restart: "{{ searchengine_conf_status | changed }}" - state: started - restart_policy: always - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata" diff --git a/deployment/ansible/management-searchengine.yml b/deployment/ansible/management-searchengine.yml deleted file mode 100644 index 43cfff3c..00000000 --- a/deployment/ansible/management-searchengine.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Deploy Elasticsearch Cluster - -- name: Deploying search engine - connection: local - hosts: local - vars_files: - searchengine_vars.yml - vars: - elasticsearch_nodes: [ ] - instances_nodes: [ ] - instances: { } - elasticsearch_nodes_urls: [ ] - - tasks: - - name: Deploy elastic search cluster - ansible.builtin.include_tasks: - file: deploy_elasticsearch_cluster.yml - - name: Deploy serach engine - ansible.builtin.include_tasks: - file: deploy_searchengine.yml diff --git a/deployment/ansible/run_searchengine_index_services.yml b/deployment/ansible/run_searchengine_index_services.yml deleted file mode 100644 index c92304e5..00000000 --- a/deployment/ansible/run_searchengine_index_services.yml +++ /dev/null @@ -1,24 +0,0 @@ -#Issue setup ip address inside the hpa config file for postgres to accept the connection from it -- name: Deploying search engine cache and indexing - connection: local - hosts: local - vars_files: - searchengine_vars.yml - tasks: - - - name: Get data from postgres database and insert them to Elasticsearch index using docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_index - cleanup: True - auto_remove: yes - command: "get_index_data_from_database" - networks: - - name: searchengine-net - ipv4_address: 10.11.0.11 - published_ports: - - "5577:5577" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" diff --git a/deployment/ansible/searchengine_vars.yml b/deployment/ansible/searchengine_vars.yml deleted file mode 100644 index a82c5dcb..00000000 --- a/deployment/ansible/searchengine_vars.yml +++ /dev/null @@ -1,18 +0,0 @@ -apps_folder: /path_to_app_base_folder -database_port: 5432 -database_name: omero -database_username: dtabase_user -database_user_password: database_user_password -database_server_url: postgres_database_url -search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1 -searchengine_docker_image: openmicroscopy/omero-searchengine:latest -searchengine_index: searchengine_index -cache_rows: 100000 -searchengine_secret_key: omero_searchengine_secret_key -searchengineurlprefix: "searchengine" -IDR_TEST_FILE_URL: "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv" -elasticsearch_no_nodes: 3 -elasticsearch_backup_folder: "/searchengine_backup" -ca_password: searchengine_secret_elastic_ca_password -keystore_password: searchengine_secret_elastic_keystore_password -ELASTIC_PASSWORD: searchengine_secret_elastic_password