Security: updated cert generation for root-ca/NiFi.

CogStack · Aug 14, 2023 · 1f50ed8 · 1f50ed8
1 parent 73417cb
commit 1f50ed8
Show file tree

Hide file tree

Showing 9 changed files with 232 additions and 29 deletions.
diff --git a/.gitignore b/.gitignore
@@ -19,7 +19,6 @@ security/nifi_certificates/*
 security/nifi_toolkit
 
 # other
-nifi/nifi-app.log
 scripts/data_processing/anonymistion_hw_clean/anonymised
 
 # docs

diff --git a/deploy/export_env_vars.sh b/deploy/export_env_vars.sh
@@ -27,6 +27,9 @@ for env_file in ${env_files[@]}; do
   source $env_file
 done
 
+# for nginx vars
+export DOLLAR="$"
+
 set +a
 
 set +o allexport
diff --git a/deploy/services.yml b/deploy/services.yml
@@ -1100,6 +1100,7 @@ services:
       - "${NIFI_EXTERNAL_PORT_NGINX:-8443}:${NIFI_INTERNAL_PORT_NGINX:-8443}"
     networks:
       - cognet
+    command: /bin/bash -c "envsubst < /etc/nginx/config/nginx.conf.template > /etc/nginx/config/nginx.conf && nginx -g 'daemon off;'"
 
 #---------------------------------------------------------------------------#
 # Jupyter Hub                                                               #

diff --git a/security/certificates_nifi.env b/security/certificates_nifi.env
@@ -1,5 +1,5 @@
 # NiFi certificate vars,
 NIFI_TOOLKIT_VERSION="1.22.0"
 NIFI_CERTIFICATE_TIME_VAILIDITY_IN_DAYS=730
-NIFI_SUBJ_LINE_CERTIFICATE_CN="C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack"
+NIFI_SUBJ_LINE_CERTIFICATE_CN="CN=cogstack,OU=NIFI,C=UK,ST=UK,L=UK,O=cogstack"
 NIFI_KEY_PASSWORD=cogstackNifi
diff --git a/security/create_root_ca_cert.sh b/security/create_root_ca_cert.sh
@@ -22,8 +22,8 @@ else
 fi
 
 if [[ -z "${ROOT_CERTIFICATE_SUBJ_LINE}" ]]; then
-    ROOT_CERTIFICATE_SUBJ_LINE="/C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack"
-    echo "ROOT_CERTIFICATE_SUBJ_LINE not set, defaulting to ROOT_CERTIFICATE_SUBJ_LINE=/C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack"
+    ROOT_CERTIFICATE_SUBJ_LINE="/CN=cogstack/OU=NIFI/C=UK/ST=UK/L=UK/O=cogstack"
+    echo "ROOT_CERTIFICATE_SUBJ_LINE not set, defaulting to ROOT_CERTIFICATE_SUBJ_LINE=CN=cogstack/OU=NIFI/C=UK/ST=UK/L=UK/O=cogstack"
 else
     ROOT_CERTIFICATE_SUBJ_LINE=${ROOT_CERTIFICATE_SUBJ_LINE}
 fi

diff --git a/security/nifi_toolkit_security.sh b/security/nifi_toolkit_security.sh
@@ -45,8 +45,8 @@ OUTPUT_DIRECTORY="./nifi_certificates"
 # -C,--clientCertDn <arg> Generate client certificate suitable for use in browser with specified DN (Can be specified multiple times)
 # this should respect whatever is used to generate the other certificate with regards CN=nifi, this needs to match the HOSTNAME of the nifi container(s)
 if [[ -z "${NIFI_SUBJ_LINE_CERTIFICATE_CN}" ]]; then
-    NIFI_SUBJ_LINE_CERTIFICATE_CN="C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack"
-    echo "NIFI_SUBJ_LINE_CERTIFICATE_CN not set, defaulting to NIFI_SUBJ_LINE_CERTIFICATE_CN=C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack"
+    NIFI_SUBJ_LINE_CERTIFICATE_CN="CN=cogstack, OU=cogstack, C=UK, ST=UK, L=UK, O=cogstack"
+    echo "NIFI_SUBJ_LINE_CERTIFICATE_CN not set, defaulting to NIFI_SUBJ_LINE_CERTIFICATE_CN=CN=cogstack, OU=cogstack, C=UK, ST=UK, L=UK, O=cogstack"
 else
     NIFI_SUBJ_LINE_CERTIFICATE_CN=${NIFI_SUBJ_LINE_CERTIFICATE_CN}
 fi

diff --git a/security/nifi_toolkit_security_client_cert.sh b/security/nifi_toolkit_security_client_cert.sh
diff --git a/services/nginx/config/nginx.conf b/services/nginx/config/nginx.conf
@@ -27,7 +27,7 @@ http {
     # Specify MIME types for files.
     include /etc/nginx/mime.types;
 	default_type application/octet-stream;
-    
+
     #upstream elasticsearch {
     #    server elasticsearch-1:9200;
     #}
@@ -57,11 +57,20 @@ http {
     #server {
     #    listen 5601;
     #    server_name kibana.local;
+    #    ssl_certificate /etc/nginx/es_certificates/elastic-stack-ca.crt.pem;
+    #    ssl_certificate_key /etc/nginx/es_certificates/elastic-stack-ca.key.pem;
+
+    #
+    #    proxy_ssl_certificate /etc/nginx/es_certificates/elasticsearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt;
+    #    proxy_ssl_certificate_key /etc/nginx/es_certificates/elasticsearch/elasticsearch/elasticsearch-1/elasticsearch-1.pem;
+
+    #    proxy_ssl_trusted_certificate /etc/nginx/es_certificates/elasticsearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt;
+    #    proxy_ssl_server_name on;
 
     #    location / {
     #        proxy_pass http://kibana;
     #        
-    #        proxy_set_header X-ProxyScheme http;
+    #        proxy_set_header X-ProxyScheme https;
     #        proxy_set_header X-ProxyHost nginx.docker.localhost;
     #        proxy_set_header X-ProxyPort 5601;
     #        proxy_set_header X-ProxyContextPath /;
@@ -72,7 +81,7 @@ http {
     #    listen 80;
     #    listen [::]:80;
     #    server_name nginx.local; 
-    #    return 301 https://$host$request_uri;
+    #    return 301 https://hostrequest_uri;
     #}
 
 
@@ -91,10 +100,10 @@ http {
         location / {
             proxy_set_header Host nifi;
             proxy_set_header X-Real-IP nifi;
-            proxy_set_header X-ProxyHost $host;
+            proxy_set_header X-ProxyHost host;
             proxy_set_header X-ProxyPort 8443;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-ProxyScheme $scheme;
+            proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme scheme;
             proxy_set_header X-ProxyContextPath /;
             proxy_set_header X-Forwarded-Context /;
             proxy_pass https://nifi;
@@ -103,44 +112,44 @@ http {
         location ^~ /nifi {
             proxy_set_header Host nifi;
             proxy_set_header X-Real-IP nifi;
-            proxy_set_header X-ProxyHost $host;
+            proxy_set_header X-ProxyHost host;
             proxy_set_header X-ProxyPort 8443;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-ProxyScheme $scheme;
+            proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme scheme;
             proxy_set_header X-Forwarded-Context /nifi;
             proxy_set_header X-ProxyContextPath /nifi;
             proxy_pass https://nifi;
         }
 
         location ^~ /nifi-api/ {
-            proxy_set_header Host nifi;
-            proxy_set_header X-Real-IP nifi;
-            proxy_set_header X-ProxyHost $host;
+            proxy_set_header Host NIFI;
+            proxy_set_header X-Real-IP NIFI;
+            proxy_set_header X-ProxyHost NIFI;
             proxy_set_header X-ProxyPort 8443;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-ProxyScheme $scheme;
+            proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme scheme;
             proxy_set_header X-ProxyContextPath "";
             proxy_pass https://nifi/nifi-api/;
         }
 
         location ^~ /nifi-content-viewer/ {
-            proxy_set_header Host nifi;
-            proxy_set_header X-Real-IP nifi;
-            proxy_set_header X-ProxyHost $host;
+            proxy_set_header Host NIFI;
+            proxy_set_header X-Real-IP NIFI;
+            proxy_set_header X-ProxyHost host;
             proxy_set_header X-ProxyPort 8443;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-ProxyScheme $scheme;
+            proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme scheme;
             proxy_set_header X-ProxyContextPath "";
             proxy_pass https://nifi/nifi-content-viewer/;
         }
 
         location ^~ /nifi-registry-api/ {
-            proxy_set_header Host nifi;
+            proxy_set_header Host NIFI;
             proxy_set_header X-Real-IP nifi;
-            proxy_set_header X-ProxyHost $host;
+            proxy_set_header X-ProxyHost host;
             proxy_set_header X-ProxyPort 8443;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-ProxyScheme $scheme;
+            proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme scheme;
             proxy_set_header X-ProxyContextPath "";
             proxy_pass https://nifi/nifi-registry-api/;
         }

diff --git a/services/nginx/config/nginx.conf.template b/services/nginx/config/nginx.conf.template
@@ -0,0 +1,191 @@
+user nginx;
+pid /var/run/nginx.pid;
+
+worker_processes 1;
+
+events {
+        worker_connections 1024;
+    }
+
+http {
+
+    ##
+	# Basic Settings
+	##
+
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 8192;
+
+    # Hide nginx version information.
+    server_tokens off;
+
+    server_name_in_redirect off;
+
+    # Specify MIME types for files.
+    include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+    #upstream elasticsearch {
+    #    server elasticsearch-1:9200;
+    #}
+
+    #upstream kibana {
+    #    server kibana:5601;
+    #}
+
+    upstream NIFI {
+        server NIFI:8443;
+    }
+
+    #server {
+    #    listen 9200;
+    #    server_name elasticsearch.local;
+
+    #    location / {
+    #        proxy_pass http://elasticsearch;
+    #
+    #        proxy_set_header X-ProxyScheme http;
+    #        proxy_set_header X-ProxyHost nginx.docker.localhost;
+    #        proxy_set_header X-ProxyPort 9200;
+    #        proxy_set_header X-ProxyContextPath /;
+    #    }
+    #}
+
+    #server {
+    #    listen ${KIBANA_SERVER_OUTPUT_PORT};
+    #    server_name kibana.local;
+
+    #    ssl_certificate /etc/nginx/es_certificates/elastic-stack-ca.crt.pem;
+    #    ssl_certificate_key /etc/nginx/es_certificates/elastic-stack-ca.key.pem;
+
+    #
+    #    proxy_ssl_certificate /etc/nginx/es_certificates/${ELASTICSEARCH_VERSION}/elasticsearch/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}.crt;
+    #    proxy_ssl_certificate_key /etc/nginx/es_certificates/${ELASTICSEARCH_VERSION}/elasticsearch/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}.pem;
+
+    #    proxy_ssl_trusted_certificate /etc/nginx/es_certificates/${ELASTICSEARCH_VERSION}/elasticsearch/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}.crt;
+    #    proxy_ssl_server_name on;
+
+    #    location / {
+    #        proxy_pass http://${NGINX_KIBANA_HOST};
+    #
+    #        proxy_set_header X-ProxyScheme https;
+    #        proxy_set_header X-ProxyHost nginx.docker.localhost;
+    #        proxy_set_header X-ProxyPort ${NGINX_KIBANA_PROXY_PORT};
+    #        proxy_set_header X-ProxyContextPath /;
+    #    }
+    #}
+
+    #server {
+    #    listen 80;
+    #    listen [::]:80;
+    #    server_name nginx.local;
+    #    return 301 https://${DOLLAR}host${DOLLAR}request_uri;
+    #}
+
+
+    server {
+        listen 8443 ssl;
+        server_name nginx.local cogstack-nifi NIFI;
+        ssl_certificate /etc/nginx/root-ca.pem;
+        ssl_certificate_key /etc/nginx/root-ca.key;
+
+        proxy_ssl_certificate /etc/nginx/nifi-cert.pem;
+        proxy_ssl_certificate_key /etc/nginx/nifi-key.key;
+
+        proxy_ssl_trusted_certificate /etc/nginx/nifi-cert.pem;
+        proxy_ssl_server_name on;
+
+        location / {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-ProxyContextPath /;
+            proxy_set_header X-Forwarded-Context /;
+            proxy_pass https://nifi;
+        }
+
+        location ^~ /nifi {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-Forwarded-Context /nifi;
+            proxy_set_header X-ProxyContextPath /nifi;
+            proxy_pass https://nifi;
+        }
+
+        location ^~ /nifi-api/ {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-ProxyContextPath "";
+            proxy_pass https://nifi/nifi-api/;
+        }
+
+        location ^~ /nifi-content-viewer/ {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-ProxyContextPath "";
+            proxy_pass https://nifi/nifi-content-viewer/;
+        }
+
+        location ^~ /nifi-registry-api/ {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-ProxyContextPath "";
+            proxy_pass https://nifi/nifi-registry-api/;
+        }
+
+        location ^~ /tenants {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-ProxyContextPath "";
+            proxy_pass https://nifi/tenants/;
+        }
+
+        location ^~ /controller {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-ProxyContextPath "";
+            proxy_pass https://nifi/controller/;
+        }
+
+        location ^~ /policies {
+            proxy_set_header Host ${DOLLAR}host;
+            proxy_set_header X-Real-IP ${DOLLER}remote_addr;
+            proxy_set_header X-ProxyHost ${DOLLAR}proxy_host;
+            proxy_set_header X-ProxyPort 8443;
+            proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for;
+            proxy_set_header X-ProxyScheme ${DOLLAR}scheme;
+            proxy_set_header X-ProxyContextPath "";
+            proxy_pass https://nifi/policies/;
+        }
+    }
+}