CU-8692nnx2z: resubmit all validated docs on startup, default is true.

envs/env

@@ -24,4 +24,7 @@ CONCEPT_SEARCH_SERVICE_PORT=8983
 ### DB backup dir ###
 DB_DIR=/home/api/db
 DB_PATH=${DB_DIR}/db.sqlite3   # currently only supports sqlite3 dbs
-DB_BACKUP_DIR=/home/api/db-backup
+DB_BACKUP_DIR=/home/api/db-backup
+
+# Resubmit all on startup
+RESUBMIT_ALL_ON_STARTUP=1

envs/env-prod

@@ -26,3 +26,6 @@ CONCEPT_SEARCH_SERVICE_PORT=8983
 DB_DIR=/home/api/db
 DB_PATH=${DB_DIR}/db.sqlite3    # currently only supports sqlite3 dbs
 DB_BACKUP_DIR=                  # should be set to 'good' backup location
+
+# Resubmit all on startup
+RESUBMIT_ALL_ON_STARTUP=1

webapp/api/api/__init__.py

@@ -0,0 +1,2 @@
+
+default_app_config = 'api.apps.ApiConfig'

webapp/api/api/apps.py

@@ -1,5 +1,30 @@
+import logging
+import os
+
 from django.apps import AppConfig
 
+logger = logging.getLogger(__name__)
+
+
+class ApiConfig(AppConfig):
+    name = 'api'
 
-class AnnoappConfig(AppConfig):
-    name = 'annoapp'
+    def ready(self):
+        from api.views import _submit_document
+        from api.models import ProjectAnnotateEntities
+        resubmit_all = os.environ.get('RESUBMIT_ALL_ON_STARTUP', False)
+        if resubmit_all:
+            logger.info('Found env var RESUBMIT_ALL_ON_STARTUP is True. '
+                        'Attempting to resubmit all currently submitted state documents')
+            projects = ProjectAnnotateEntities.objects.all()
+            for project in projects:
+                validated_docs = project.validated_documents.all()
+                if len(validated_docs):
+                    for doc in validated_docs:
+                        try:
+                            _submit_document(project, doc)
+                            logger.info("Submitted doc: %s", doc.name)
+                        except Exception as e:
+                            logger.error("Failed to re-submit doc on startup with exception %s", e)
+                logger.info("Finished resubmitting Project %s", project.name)
+        logger.info("MedCATTrainer App API ready...")

webapp/api/api/views.py

@@ -382,29 +382,19 @@ def import_cdb_concepts(request):
     return Response({'message': 'submitted cdb import job.'})
 
 
-@api_view(http_method_names=['POST'])
-def submit_document(request):
-    # Get project id
-    p_id = request.data['project_id']
-    d_id = request.data['document_id']
-
-    # Get project and the right version of cat
-    project = ProjectAnnotateEntities.objects.get(id=p_id)
-    document = Document.objects.get(id=d_id)
-
-    cat = get_medcat(CDB_MAP=CDB_MAP, VOCAB_MAP=VOCAB_MAP,
-                     CAT_MAP=CAT_MAP, project=project)
-
+def _submit_document(project: ProjectAnnotateEntities, document: Document):
     if project.train_model_on_submit:
         try:
+            cat = get_medcat(CDB_MAP=CDB_MAP, VOCAB_MAP=VOCAB_MAP,
+                             CAT_MAP=CAT_MAP, project=project)
             train_medcat(cat, project, document)
         except Exception as e:
             if project.vocab.id:
                 if len(VOCAB_MAP[project.vocab.id].unigram_table) == 0:
-                    return HttpResponseServerError('Vocab is missing the unigram table. On the vocab instance '
+                    return Exception('Vocab is missing the unigram table. On the vocab instance '
                                                    'use vocab.make_unigram_table() to build')
             else:
-                return HttpResponseServerError(e.message)
+                raise e
 
     # Add cuis to filter if they did not exist
     cuis = []
@@ -423,6 +413,22 @@ def submit_document(request):
             project.cuis += ',' + ','.join(extra_doc_cuis)
             project.save()
 
+
+@api_view(http_method_names=['POST'])
+def submit_document(request):
+    # Get project id
+    p_id = request.data['project_id']
+    d_id = request.data['document_id']
+
+    # Get project and the right version of cat
+    project = ProjectAnnotateEntities.objects.get(id=p_id)
+    document = Document.objects.get(id=d_id)
+
+    try:
+        _submit_document(project, document)
+    except Exception as e:
+        HttpResponseServerError(e.message)
+
     return Response({'message': 'Document submited successfully'})
 
 

webapp/scripts/run.sh

@@ -24,3 +24,5 @@ if [ $LOAD_EXAMPLES ]; then
 fi
 
 uwsgi --http-timeout 360s --http :8000 --master --chdir /home/api/  --module core.wsgi
+
+