diff --git a/.gitignore b/.gitignore index c20be61f..a5035cb4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ security/nifi_certificates/* security/nifi_toolkit # other -nifi/nifi-app.log scripts/data_processing/anonymistion_hw_clean/anonymised # docs diff --git a/deploy/export_env_vars.sh b/deploy/export_env_vars.sh index 4ff801f0..4f096078 100755 --- a/deploy/export_env_vars.sh +++ b/deploy/export_env_vars.sh @@ -27,6 +27,9 @@ for env_file in ${env_files[@]}; do source $env_file done +# for nginx vars +export DOLLAR="$" + set +a set +o allexport \ No newline at end of file diff --git a/deploy/services.yml b/deploy/services.yml index 2b9a9eb1..a58eea9b 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -1100,6 +1100,7 @@ services: - "${NIFI_EXTERNAL_PORT_NGINX:-8443}:${NIFI_INTERNAL_PORT_NGINX:-8443}" networks: - cognet + command: /bin/bash -c "envsubst < /etc/nginx/config/nginx.conf.template > /etc/nginx/config/nginx.conf && nginx -g 'daemon off;'" #---------------------------------------------------------------------------# # Jupyter Hub # diff --git a/security/certificates_nifi.env b/security/certificates_nifi.env index 7f876abf..afe91ac3 100644 --- a/security/certificates_nifi.env +++ b/security/certificates_nifi.env @@ -1,5 +1,5 @@ # NiFi certificate vars, NIFI_TOOLKIT_VERSION="1.22.0" NIFI_CERTIFICATE_TIME_VAILIDITY_IN_DAYS=730 -NIFI_SUBJ_LINE_CERTIFICATE_CN="C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack" +NIFI_SUBJ_LINE_CERTIFICATE_CN="CN=cogstack,OU=NIFI,C=UK,ST=UK,L=UK,O=cogstack" NIFI_KEY_PASSWORD=cogstackNifi \ No newline at end of file diff --git a/security/create_root_ca_cert.sh b/security/create_root_ca_cert.sh index 8dc9e08a..f741c79c 100644 --- a/security/create_root_ca_cert.sh +++ b/security/create_root_ca_cert.sh @@ -22,8 +22,8 @@ else fi if [[ -z "${ROOT_CERTIFICATE_SUBJ_LINE}" ]]; then - ROOT_CERTIFICATE_SUBJ_LINE="/C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack" - echo "ROOT_CERTIFICATE_SUBJ_LINE not set, defaulting to ROOT_CERTIFICATE_SUBJ_LINE=/C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack" + ROOT_CERTIFICATE_SUBJ_LINE="/CN=cogstack/OU=NIFI/C=UK/ST=UK/L=UK/O=cogstack" + echo "ROOT_CERTIFICATE_SUBJ_LINE not set, defaulting to ROOT_CERTIFICATE_SUBJ_LINE=CN=cogstack/OU=NIFI/C=UK/ST=UK/L=UK/O=cogstack" else ROOT_CERTIFICATE_SUBJ_LINE=${ROOT_CERTIFICATE_SUBJ_LINE} fi diff --git a/security/nifi_toolkit_security.sh b/security/nifi_toolkit_security.sh index ae1528c1..5df7e636 100644 --- a/security/nifi_toolkit_security.sh +++ b/security/nifi_toolkit_security.sh @@ -45,8 +45,8 @@ OUTPUT_DIRECTORY="./nifi_certificates" # -C,--clientCertDn Generate client certificate suitable for use in browser with specified DN (Can be specified multiple times) # this should respect whatever is used to generate the other certificate with regards CN=nifi, this needs to match the HOSTNAME of the nifi container(s) if [[ -z "${NIFI_SUBJ_LINE_CERTIFICATE_CN}" ]]; then - NIFI_SUBJ_LINE_CERTIFICATE_CN="C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack" - echo "NIFI_SUBJ_LINE_CERTIFICATE_CN not set, defaulting to NIFI_SUBJ_LINE_CERTIFICATE_CN=C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack" + NIFI_SUBJ_LINE_CERTIFICATE_CN="CN=cogstack, OU=cogstack, C=UK, ST=UK, L=UK, O=cogstack" + echo "NIFI_SUBJ_LINE_CERTIFICATE_CN not set, defaulting to NIFI_SUBJ_LINE_CERTIFICATE_CN=CN=cogstack, OU=cogstack, C=UK, ST=UK, L=UK, O=cogstack" else NIFI_SUBJ_LINE_CERTIFICATE_CN=${NIFI_SUBJ_LINE_CERTIFICATE_CN} fi diff --git a/security/nifi_toolkit_security_client_cert.sh b/security/nifi_toolkit_security_client_cert.sh new file mode 100644 index 00000000..e69de29b diff --git a/services/nginx/config/nginx.conf b/services/nginx/config/nginx.conf index 0e11a628..c0b71f25 100644 --- a/services/nginx/config/nginx.conf +++ b/services/nginx/config/nginx.conf @@ -27,7 +27,7 @@ http { # Specify MIME types for files. include /etc/nginx/mime.types; default_type application/octet-stream; - + #upstream elasticsearch { # server elasticsearch-1:9200; #} @@ -57,11 +57,20 @@ http { #server { # listen 5601; # server_name kibana.local; + # ssl_certificate /etc/nginx/es_certificates/elastic-stack-ca.crt.pem; + # ssl_certificate_key /etc/nginx/es_certificates/elastic-stack-ca.key.pem; + + # + # proxy_ssl_certificate /etc/nginx/es_certificates/elasticsearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt; + # proxy_ssl_certificate_key /etc/nginx/es_certificates/elasticsearch/elasticsearch/elasticsearch-1/elasticsearch-1.pem; + + # proxy_ssl_trusted_certificate /etc/nginx/es_certificates/elasticsearch/elasticsearch/elasticsearch-1/elasticsearch-1.crt; + # proxy_ssl_server_name on; # location / { # proxy_pass http://kibana; # - # proxy_set_header X-ProxyScheme http; + # proxy_set_header X-ProxyScheme https; # proxy_set_header X-ProxyHost nginx.docker.localhost; # proxy_set_header X-ProxyPort 5601; # proxy_set_header X-ProxyContextPath /; @@ -72,7 +81,7 @@ http { # listen 80; # listen [::]:80; # server_name nginx.local; - # return 301 https://$host$request_uri; + # return 301 https://hostrequest_uri; #} @@ -91,10 +100,10 @@ http { location / { proxy_set_header Host nifi; proxy_set_header X-Real-IP nifi; - proxy_set_header X-ProxyHost $host; + proxy_set_header X-ProxyHost host; proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; + proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme scheme; proxy_set_header X-ProxyContextPath /; proxy_set_header X-Forwarded-Context /; proxy_pass https://nifi; @@ -103,44 +112,44 @@ http { location ^~ /nifi { proxy_set_header Host nifi; proxy_set_header X-Real-IP nifi; - proxy_set_header X-ProxyHost $host; + proxy_set_header X-ProxyHost host; proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; + proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme scheme; proxy_set_header X-Forwarded-Context /nifi; proxy_set_header X-ProxyContextPath /nifi; proxy_pass https://nifi; } location ^~ /nifi-api/ { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP nifi; - proxy_set_header X-ProxyHost $host; + proxy_set_header Host NIFI; + proxy_set_header X-Real-IP NIFI; + proxy_set_header X-ProxyHost NIFI; proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; + proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme scheme; proxy_set_header X-ProxyContextPath ""; proxy_pass https://nifi/nifi-api/; } location ^~ /nifi-content-viewer/ { - proxy_set_header Host nifi; - proxy_set_header X-Real-IP nifi; - proxy_set_header X-ProxyHost $host; + proxy_set_header Host NIFI; + proxy_set_header X-Real-IP NIFI; + proxy_set_header X-ProxyHost host; proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; + proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme scheme; proxy_set_header X-ProxyContextPath ""; proxy_pass https://nifi/nifi-content-viewer/; } location ^~ /nifi-registry-api/ { - proxy_set_header Host nifi; + proxy_set_header Host NIFI; proxy_set_header X-Real-IP nifi; - proxy_set_header X-ProxyHost $host; + proxy_set_header X-ProxyHost host; proxy_set_header X-ProxyPort 8443; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-ProxyScheme $scheme; + proxy_set_header X-Forwarded-For proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme scheme; proxy_set_header X-ProxyContextPath ""; proxy_pass https://nifi/nifi-registry-api/; } diff --git a/services/nginx/config/nginx.conf.template b/services/nginx/config/nginx.conf.template new file mode 100644 index 00000000..49346d0d --- /dev/null +++ b/services/nginx/config/nginx.conf.template @@ -0,0 +1,191 @@ +user nginx; +pid /var/run/nginx.pid; + +worker_processes 1; + +events { + worker_connections 1024; + } + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 8192; + + # Hide nginx version information. + server_tokens off; + + server_name_in_redirect off; + + # Specify MIME types for files. + include /etc/nginx/mime.types; + default_type application/octet-stream; + + #upstream elasticsearch { + # server elasticsearch-1:9200; + #} + + #upstream kibana { + # server kibana:5601; + #} + + upstream NIFI { + server NIFI:8443; + } + + #server { + # listen 9200; + # server_name elasticsearch.local; + + # location / { + # proxy_pass http://elasticsearch; + # + # proxy_set_header X-ProxyScheme http; + # proxy_set_header X-ProxyHost nginx.docker.localhost; + # proxy_set_header X-ProxyPort 9200; + # proxy_set_header X-ProxyContextPath /; + # } + #} + + #server { + # listen ${KIBANA_SERVER_OUTPUT_PORT}; + # server_name kibana.local; + + # ssl_certificate /etc/nginx/es_certificates/elastic-stack-ca.crt.pem; + # ssl_certificate_key /etc/nginx/es_certificates/elastic-stack-ca.key.pem; + + # + # proxy_ssl_certificate /etc/nginx/es_certificates/${ELASTICSEARCH_VERSION}/elasticsearch/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}.crt; + # proxy_ssl_certificate_key /etc/nginx/es_certificates/${ELASTICSEARCH_VERSION}/elasticsearch/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}.pem; + + # proxy_ssl_trusted_certificate /etc/nginx/es_certificates/${ELASTICSEARCH_VERSION}/elasticsearch/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}/${NGINX_ES_NODE_SOURCE_INSTANCE_NAME}.crt; + # proxy_ssl_server_name on; + + # location / { + # proxy_pass http://${NGINX_KIBANA_HOST}; + # + # proxy_set_header X-ProxyScheme https; + # proxy_set_header X-ProxyHost nginx.docker.localhost; + # proxy_set_header X-ProxyPort ${NGINX_KIBANA_PROXY_PORT}; + # proxy_set_header X-ProxyContextPath /; + # } + #} + + #server { + # listen 80; + # listen [::]:80; + # server_name nginx.local; + # return 301 https://${DOLLAR}host${DOLLAR}request_uri; + #} + + + server { + listen 8443 ssl; + server_name nginx.local cogstack-nifi NIFI; + ssl_certificate /etc/nginx/root-ca.pem; + ssl_certificate_key /etc/nginx/root-ca.key; + + proxy_ssl_certificate /etc/nginx/nifi-cert.pem; + proxy_ssl_certificate_key /etc/nginx/nifi-key.key; + + proxy_ssl_trusted_certificate /etc/nginx/nifi-cert.pem; + proxy_ssl_server_name on; + + location / { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-ProxyContextPath /; + proxy_set_header X-Forwarded-Context /; + proxy_pass https://nifi; + } + + location ^~ /nifi { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-Forwarded-Context /nifi; + proxy_set_header X-ProxyContextPath /nifi; + proxy_pass https://nifi; + } + + location ^~ /nifi-api/ { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-ProxyContextPath ""; + proxy_pass https://nifi/nifi-api/; + } + + location ^~ /nifi-content-viewer/ { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-ProxyContextPath ""; + proxy_pass https://nifi/nifi-content-viewer/; + } + + location ^~ /nifi-registry-api/ { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-ProxyContextPath ""; + proxy_pass https://nifi/nifi-registry-api/; + } + + location ^~ /tenants { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-ProxyContextPath ""; + proxy_pass https://nifi/tenants/; + } + + location ^~ /controller { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-ProxyContextPath ""; + proxy_pass https://nifi/controller/; + } + + location ^~ /policies { + proxy_set_header Host ${DOLLAR}host; + proxy_set_header X-Real-IP ${DOLLER}remote_addr; + proxy_set_header X-ProxyHost ${DOLLAR}proxy_host; + proxy_set_header X-ProxyPort 8443; + proxy_set_header X-Forwarded-For ${DOLLAR}proxy_add_x_forwarded_for; + proxy_set_header X-ProxyScheme ${DOLLAR}scheme; + proxy_set_header X-ProxyContextPath ""; + proxy_pass https://nifi/policies/; + } + } +}