Merge branch 'master' of https://github.com/CogStack/CogStack-NiFi

CogStack · Mar 11, 2024 · 89d4ef3 · 89d4ef3
2 parents 37b44af + 3242394
commit 89d4ef3
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 7 deletions.
diff --git a/nifi/conf/state-management.xml b/nifi/conf/state-management.xml
@@ -35,9 +35,9 @@
         <id>local-provider</id>
         <class>org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider</class>
         <property name="Directory">./state/local</property>
-        <property name="Always Sync">false</property>
+        <property name="Always Sync">true</property>
         <property name="Partitions">16</property>
-        <property name="Checkpoint Interval">2 mins</property>
+        <property name="Checkpoint Interval">10 mins</property>
     </local-provider>
 
     <!--

diff --git a/services/nlp-services/applications/medcat/config/env_app b/services/nlp-services/applications/medcat/config/env_app
@@ -2,7 +2,7 @@
 
 # IMPORTANT : log level set 
 # CRITICAL - 50, ERROR - 40, WARNING - 30, INFO - 20, DEBUG - 10, NOTSET - 0
-APP_LOG_LEVEL=ERROR
+APP_LOG_LEVEL=INFO
 
 # can include only one model for NER
 APP_NAME=MedCAT
@@ -23,16 +23,16 @@ APP_MEDCAT_MODEL_PACK=
 # APP_MODEL_CUI_FILTER_PATH=/cat/models/cui_filter.txt
 
 # NLP processing
-APP_BULK_NPROC=16
+APP_BULK_NPROC=8
 APP_TRAINING_MODE=False
 
 # Flask server config
 SERVER_HOST=0.0.0.0
 SERVER_PORT=5000
 SERVER_WORKERS=1
 SERVER_WORKER_TIMEOUT=300
-SERVER_THREADS=16
+SERVER_THREADS=1
 
 # GPU SETTING
 # CAUTION, use only if you are using the GPU docker image.
-APP_TORCH_THREADS=0
+APP_TORCH_THREADS=-1
diff --git a/services/nlp-services/applications/medcat/config/env_medcat b/services/nlp-services/applications/medcat/config/env_medcat
@@ -2,7 +2,7 @@ TYPE=NOT_UMLS
 
 # IMPORTANT : log level set 
 # CRITICAL - 50, ERROR - 40, WARNING - 30, INFO - 20, DEBUG - 10, NOTSET - 0
-LOG_LEVEL=20
+LOG_LEVEL=40
 
 NESTED_ENTITIES=False
 
@@ -31,3 +31,9 @@ SPACY_MODEL=
 #    (the NiFi groovy scripts and annotation ingester are two such services that process the output, and so they might require further customisation)
 # POSSIBLE VALUES: [list, dict], if left empty then "dict" is the default.
 ANNOTATIONS_ENTITY_OUTPUT_MODE=dict
+
+# you will need a De-ID model for this, otherwise there might be unexpected behaviour
+DEID_MODE=False
+
+# replaces the type of the de-id entity e.g  patient name: [Name]  with patient name: [****] 
+DEID_REDACT=True