elastic support https

create_elastic_certs

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+if [ ! -f ./certs/elastic_ca/ca.crt ] && [ ! -f ./certs/elastic_ca/ca.key ] && [ ! -f ./certs/elastic_instance/instance.crt ] && [ ! -f ./certs/elastic_instance/instance.key ]; then
+    # start container
+    docker pull docker.elastic.co/elasticsearch/elasticsearch:8.15.0 &&
+    docker run -d --name elasticsearch_cert -v ./elasticsearch_instances.yml:/usr/share/elasticsearch/elasticsearch_instances.yml -it docker.elastic.co/elasticsearch/elasticsearch:8.15.0 &&
+    # generate ca
+    docker exec -ti elasticsearch_cert ./bin/elasticsearch-certutil ca --pem --out ca.zip &&
+    docker exec -ti elasticsearch_cert unzip ca.zip &&
+    # generate cert signed with the ca previously generate
+    docker exec -ti elasticsearch_cert ./bin/elasticsearch-certutil cert --in /usr/share/elasticsearch/elasticsearch_instances.yml --pem --ca-cert ./ca/ca.crt --ca-key ./ca/ca.key --silent --out cert.zip &&
+    docker exec -ti elasticsearch_cert unzip cert.zip &&
+    # extract files from the container
+    docker cp elasticsearch_cert:/usr/share/elasticsearch/ca ./certs/elastic_ca &&
+    docker cp elasticsearch_cert:/usr/share/elasticsearch/elasticsearch ./certs/elastic_instance &&
+    # down container
+    docker kill elasticsearch_cert &&
+    docker rm elasticsearch_cert
+else
+    echo "files already exists"
+fi

docker/elasticsearch.override.yml

@@ -21,7 +21,10 @@ services:
       - ../certs:/usr/share/elasticsearch/config/certificates
     environment:
       - discovery.type=single-node
-      - xpack.security.http.ssl.enabled=false
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certificates/elastic_instance/elasticsearch.key
+      - xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certificates/elastic_ca/ca.crt
+      - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certificates/elastic_instance/elasticsearch.crt
 
 volumes:
   elastic_data:

elasticsearch_instances.yml

@@ -0,0 +1,2 @@
+instances:
+  - name: elasticsearch

intel_owl/settings/elasticsearch.py

@@ -35,6 +35,13 @@
     ELASTIC_PASSWORD = secrets.get_secret("ELASTIC_PASSWORD")
     if ELASTIC_PASSWORD:
         elastic_client_settings["basic_auth"] = ("elastic", ELASTIC_PASSWORD)
+    ca_path = "/opt/deploy/intel_owl/certs/elastic_ca/ca.crt"
+    cert_path = "/opt/deploy/intel_owl/certs/elastic_instance/elasticsearch.crt"
+    if "elasticsearch:9200" in ELASTIC_HOST:
+        # in case we use Elastic as container we need the generated
+        # in case we use Elastic as external service it should have a valid cert
+        elastic_client_settings["verify_certs"] = cert_path
+        elastic_client_settings["ca_certs"] = ca_path
     ELASTICSEARCH_DSL = {"default": elastic_client_settings}
 
     ELASTICSEARCH_DSL_INDEX_SETTINGS = {

intel_owl/tasks.py

@@ -444,6 +444,7 @@ def _convert_report_to_elastic_document(_class: AbstractReport) -> List[Dict]:
         + _convert_report_to_elastic_document(PivotReport)
         + _convert_report_to_elastic_document(VisualizerReport)
     )
+    logger.info(f"documents to add to elastic: {len(document_list)}")
     try:
         bulk(connections.get_connection(), document_list)
     except ApiError as error:

start

@@ -201,6 +201,7 @@ while [[ $# -gt 0 ]]; do
   ;;
   --elastic)
     params["elastic"]=true
+    ./create_elastic_certs
     shift 1
   ;;
   --pycti-version)