Merge branch 'master' of https://github.com/CogStack/CogStack-NiFi

.gitmodules

@@ -1,5 +1,5 @@
 [submodule "working_with_cogstack"]
-	path = services/jupyter-hub/notebooks/working_with_cogstack
+	path = services/jupyter-hub/notebooks/demo_working_with_cogstack
 	url = https://github.com/CogStack/working_with_cogstack.git
 	branch = main
 

nifi/conf/nifi.properties

@@ -168,7 +168,7 @@ nifi.web.max.header.size=32 KB
 # IMPORTANT : be careful with this setting, it is buggy and single '/' don't work
 # After testing it seems that when running nifi in Docker you should set it via the NIFI_WEB_PROXY_CONTEXT_PATH ENV var, otherwise it wont work (needs a bug report)
 nifi.web.proxy.context.path="/,/nifi,/nifi-api,/nifi-api/,/nifi/"
-#nifi.web.proxy.host="0.0.0.0:8443,cogstack:8443,nifi:8443"
+nifi.web.proxy.host="0.0.0.0:8443,cogstack:8443,nifi:8443"
 nifi.web.max.content.size=
 nifi.web.max.requests.per.second=30000
 nifi.web.max.access.token.requests.per.second=25

scripts/installation_utils/install_docker_and_utils.sh

@@ -92,12 +92,10 @@ fi;
 echo "Installing require python packages.."
 
 sudo -H pip3 install --upgrade pip
-sudo -H pip3 install wheel docker-compose
-sudo -H pip3 install html2text jsoncsv detect
+sudo -H pip3 install html2text jsoncsv detect --break-system-packages
 
 echo "Finished installing docker and utils.."
 
-
 sudo sysctl -w vm.max_map_count=262144
 
 sudo sh -c "echo 'vm.max_map_count=262144' >> /etc/sysctl.conf"

security/certificates_general.env

@@ -3,6 +3,7 @@
 ROOT_CERTIFICATE_NAME=root-ca
 ROOT_CERTIFICATE_KEY_PASSWORD=cogstackNifi
 ROOT_CERTIFICATE_SUBJ_LINE="/C=UK/ST=UK/L=UK/O=cogstack/OU=cogstack/CN=cogstack"
+ROOT_CERTIFICATE_SUBJ_ALT_NAMES="subjectAltName=DNS:cogstack-net.test"
 ROOT_CERTIFICATE_ALIAS_NAME=root-ca
 ROOT_CERTIFICATE_TIME_VAILIDITY_IN_DAYS=730
 ROOT_CERTIFICATE_KEY_SIZE=4096

security/certificates_nifi.env

@@ -2,4 +2,5 @@
 NIFI_TOOLKIT_VERSION="1.24.0"
 NIFI_CERTIFICATE_TIME_VAILIDITY_IN_DAYS=730
 NIFI_SUBJ_LINE_CERTIFICATE_CN="CN=cogstack,OU=NIFI,C=UK,ST=UK,L=UK,O=cogstack"
-NIFI_KEY_PASSWORD=cogstackNifi
+NIFI_KEY_PASSWORD=cogstackNifi
+NIFI_SUBJ_ALT_NAMES="test[1-6].cogstack.net"

security/create_root_ca_cert.sh

@@ -28,6 +28,13 @@ else
     ROOT_CERTIFICATE_SUBJ_LINE=${ROOT_CERTIFICATE_SUBJ_LINE}
 fi
 
+if [[ -z "${ROOT_CERTIFICATE_SUBJ_ALT_NAMES}" ]]; then
+    ROOT_CERTIFICATE_SUBJ_ALT_NAMES="subjectAltName=DNS:cogstack-net.test"
+    echo "ROOT_CERTIFICATE_SUBJ_ALT_NAMES not set, defaulting to ROOT_CERTIFICATE_SUBJ_ALT_NAMES=subjectAltName=DNS:cogstack-net.test"
+else
+    ROOT_CERTIFICATE_SUBJ_ALT_NAMES=${ROOT_CERTIFICATE_SUBJ_ALT_NAMES}
+fi
+
 if [[ -z "${ROOT_CERTIFICATE_ALIAS_NAME}" ]]; then
     ROOT_CERTIFICATE_ALIAS_NAME=$ROOT_CERTIFICATE_NAME
     echo "ROOT_CERTIFICATE_ALIAS_NAME not set, defaulting to ROOT_CERTIFICATE_ALIAS_NAME=$ROOT_CERTIFICATE_NAME"
@@ -56,7 +63,7 @@ echo "Generating root CA key"
 openssl genrsa -out $CA_ROOT_KEY $ROOT_CERTIFICATE_KEY_SIZE
 
 echo "Generating root CA cert"
-openssl req -x509 -new -key $CA_ROOT_KEY -sha256 -out $CA_ROOT_CERT -days $ROOT_CERTIFICATE_TIME_VAILIDITY_IN_DAYS -subj $ROOT_CERTIFICATE_SUBJ_LINE
+openssl req -x509 -new -key $CA_ROOT_KEY -sha256 -out $CA_ROOT_CERT -days $ROOT_CERTIFICATE_TIME_VAILIDITY_IN_DAYS -subj $ROOT_CERTIFICATE_SUBJ_LINE -addext $ROOT_CERTIFICATE_SUBJ_ALT_NAMES 
 
 # create p12 version manually
 echo "Generation pkcs12 keystore"

security/nifi_toolkit_security.sh

@@ -52,6 +52,14 @@ else
 fi
 
 
+if [[ -z "${NIFI_SUBJ_ALT_NAMES}" ]]; then
+    NIFI_SUBJ_ALT_NAMES="test[1-6].cogstack.net"
+    echo "NIFI_SUBJ_ALT_NAMES not set, defaulting to NIFI_SUBJ_ALT_NAMES=test[1-6].cogstack.net"
+else
+    NIFI_SUBJ_ALT_NAMES=${NIFI_SUBJ_ALT_NAMES}
+fi
+
+
 # IMPRTANT: this is used in StandardSSLContextService controllers on the NiFi side, trusted keystore password field.
 if [[ -z "${NIFI_KEY_PASSWORD}" ]]; then
     NIFI_KEY_PASSWORD="cogstackNifi"
@@ -72,7 +80,7 @@ export JAVA_OPTS="-Xmx2048m -Xms2048m"
 
 for win_os in ${windows_unames[@]}; do
     if [[ $win_os == *"$os_name"* ]]; then
-        ./nifi_toolkit/bin/tls-toolkit.bat standalone -k $KEY_SIZE -n $HOSTNAMES -o $OUTPUT_DIRECTORY -O -f $PATH_TO_NIFI_PROPERTIES_FILE -d $NIFI_CERTIFICATE_TIME_VAILIDITY_IN_DAYS -C $NIFI_SUBJ_LINE_CERTIFICATE_CN -K $NIFI_KEY_PASSWORD
+        ./nifi_toolkit/bin/tls-toolkit.bat standalone -k $KEY_SIZE -n $HOSTNAMES -o $OUTPUT_DIRECTORY -O -f $PATH_TO_NIFI_PROPERTIES_FILE -d $NIFI_CERTIFICATE_TIME_VAILIDITY_IN_DAYS -C $NIFI_SUBJ_LINE_CERTIFICATE_CN -K $NIFI_KEY_PASSWORD --subjectAlternativeNames $NIFI_SUBJ_ALT_NAMES
         is_os_windows=1
     fi
 done

services/jupyter-hub/Dockerfile_singleuser

@@ -146,9 +146,10 @@ RUN pip3 install --no-cache-dir docker dockerspawner jupyterhub-firstuseauthenti
 RUN pip3 install --no-cache-dir jupyterhub-nanowireauthenticator jupyterhub-ldapauthenticator jupyterhub-kubespawner jupyterhub-nativeauthenticator
 
 # extra packages for DB connections & utilities (charts, data formats, and other useful tools such as neo4j)
-RUN pip3 install --no-cache-dir pytesseract ipyparallel py7zr cython isort html2text jsoncsv simplejson detect wheel nltk keras bokeh seaborn matplotlib graphviz plotly tqdm 
+RUN pip3 install --no-cache-dir pytesseract ipyparallel py7zr cython isort html2text jsoncsv simplejson detect wheel nltk keras bokeh seaborn matplotlib graphviz plotly tqdm
 RUN pip3 install --no-cache-dir pymssql mysql-connector-python cx-Oracle dataclasses numpy matplotlib pandas dill jsonpickle jsonext psycopg2 psycopg2-binary pyodbc openpyxl
 RUN pip3 install --no-cache-dir dvc flask GitPython elasticsearch opensearch-py neo4j eland --ignore-installed PyYAML 
+RUN pip3 install --no-cache-dir opencv-python torchvision
 
 # XNAT
 RUN pip3 install --no-cache-dir xnat
@@ -161,7 +162,7 @@ RUN pip3 install --no-cache-dir -U spacy click torch thinc
 #RUN for spacy_model in ${SPACY_MODELS}; do python3 -m spacy download $spacy_model; done
 
 # install requirements for working with cogstack scripts
-# RUN pip3 isntall --no-cache-dir -r notebooks/working_with_cogstack/requirements.txt
+# RUN pip3 isntall --no-cache-dir -r notebooks/demo_working_with_cogstack/requirements.txt
 
 RUN pip3 install --no-cache-dir medcat==1.12.0 eland plotly
 

services/jupyter-hub/Dockerfile_singleuser_gpu

@@ -166,9 +166,10 @@ RUN pip3 install --no-cache-dir docker dockerspawner jupyterhub-firstuseauthenti
 RUN pip3 install --no-cache-dir jupyterhub-nanowireauthenticator jupyterhub-ldapauthenticator jupyterhub-kubespawner jupyterhub-nativeauthenticator
 
 # extra packages for DB connections & utilities (charts, data formats, and other useful tools such as neo4j)
-RUN pip3 install --no-cache-dir pytesseract ipyparallel py7zr cython isort html2text jsoncsv simplejson detect wheel nltk keras bokeh seaborn matplotlib graphviz plotly tqdm 
+RUN pip3 install --no-cache-dir pytesseract ipyparallel py7zr cython isort html2text jsoncsv simplejson detect wheel nltk keras bokeh seaborn matplotlib graphviz plotly tqdm
 RUN pip3 install --no-cache-dir pymssql mysql-connector-python cx-Oracle dataclasses numpy matplotlib pandas dill jsonpickle jsonext psycopg2 psycopg2-binary pyodbc openpyxl
-RUN pip3 install --no-cache-dir dvc flask GitPython elasticsearch opensearch-py neo4j eland --ignore-installed PyYAML 
+RUN pip3 install --no-cache-dir dvc flask GitPython elasticsearch opensearch-py neo4j eland --ignore-installed PyYAML
+RUN pip3 install --no-cache-dir opencv-python torchvision
 
 # XNAT
 RUN pip3 install --no-cache-dir xnat
@@ -181,7 +182,7 @@ RUN pip3 install --no-cache-dir -U spacy click torch thinc
 #RUN for spacy_model in ${SPACY_MODELS}; do python3 -m spacy download $spacy_model; done
 
 # install requirements for working with cogstack scripts
-# RUN pip3 isntall --no-cache-dir -r notebooks/working_with_cogstack/requirements.txt
+# RUN pip3 isntall --no-cache-dir -r notebooks/demo_working_with_cogstack/requirements.txt
 
 RUN pip3 install --no-cache-dir medcat==1.12.0 eland plotly