Merge pull request #152 from CogStack/remove-project-cui-counter

CU-8692t41m8: remove project cui count references. Not used.

webapp/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update -y && \
 RUN apt-get install -y vim
 
 # Get node and npm
-RUN apt install -y nodejs && apt-get install -y npm && npm install -g npm@latest
+RUN apt install -y nodejs && apt install -y npm
 
 # Install Rust - for tokenziers dep in medcat.
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y

webapp/api/api/admin.py

@@ -35,9 +35,6 @@ def reset_project(modeladmin, request, queryset):
         # Remove all annotations and cascade to meta anns
         AnnotatedEntity.objects.filter(project=project).delete()
 
-        # Remove cui_counts
-        ProjectCuiCounter.objects.filter(project=project).delete()
-
         # Set all validated documents to none
         project.validated_documents.clear()
 
@@ -454,13 +451,6 @@ class ConceptDBAdmin(admin.ModelAdmin):
 admin.site.register(ConceptDB, ConceptDBAdmin)
 
 
-class ProjectCuiCounterAdmin(admin.ModelAdmin):
-    model = ProjectCuiCounter
-    list_filter = ('project',)
-    list_display = ['entity', 'count', 'project']
-admin.site.register(ProjectCuiCounter, ProjectCuiCounterAdmin)
-
-
 def remove_all_documents(modeladmin, request, queryset):
     Document.objects.all().delete()
 

webapp/api/api/migrations/0072_delete_projectcuicounter.py

@@ -0,0 +1,16 @@
+# Generated by Django 2.2.28 on 2023-09-21 12:09
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0071_auto_20230711_1654'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='ProjectCuiCounter',
+        ),
+    ]

webapp/api/api/models.py

@@ -168,15 +168,6 @@ def __str__(self):
         return str(self.label)
 
 
-class ProjectCuiCounter(models.Model):
-    project = models.ForeignKey('Project', on_delete=models.CASCADE)
-    entity = models.ForeignKey('Entity', on_delete=models.CASCADE)
-    count = models.IntegerField()
-
-    def __str__(self):
-        return str(self.entity) + " - " + str(self.count)
-
-
 class EntityRelation(models.Model):
     user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
     project = models.ForeignKey('Project', on_delete=models.CASCADE)

webapp/api/api/utils.py

@@ -13,7 +13,7 @@
 from medcat.utils.helpers import tkns_from_doc
 from medcat.vocab import Vocab
 
-from .models import Entity, AnnotatedEntity, ICDCode, OPCSCode, ProjectAnnotateEntities, ProjectCuiCounter, \
+from .models import Entity, AnnotatedEntity, ICDCode, OPCSCode, ProjectAnnotateEntities, \
     ConceptDB
 
 log = logging.getLogger('trainer')
@@ -68,35 +68,27 @@ def check_ents(ent):
         else:
             entity = Entity.objects.get(label=label)
 
-        cui_count_limit = cat.config.general.get("cui_count_limit", -1)
-        pcc = ProjectCuiCounter.objects.filter(project=project, entity=entity).first()
-        if pcc is not None:
-            cui_count = pcc.count
-        else:
-            cui_count = 1
-
-        if cui_count_limit < 0 or cui_count <= cui_count_limit:
-            if AnnotatedEntity.objects.filter(project=project,
-                                              document=document,
-                                              start_ind=ent.start_char,
-                                              end_ind=ent.end_char).count() == 0:
-                # If this entity doesn't exist already
-                ann_ent = AnnotatedEntity()
-                ann_ent.user = user
-                ann_ent.project = project
-                ann_ent.document = document
-                ann_ent.entity = entity
-                ann_ent.value = ent.text
-                ann_ent.start_ind = ent.start_char
-                ann_ent.end_ind = ent.end_char
-                ann_ent.acc = ent._.context_similarity
-
-                MIN_ACC = cat.config.linking.get('similarity_threshold_trainer', 0.2)
-                if ent._.context_similarity < MIN_ACC:
-                    ann_ent.deleted = True
-                    ann_ent.validated = True
-
-                ann_ent.save()
+        if AnnotatedEntity.objects.filter(project=project,
+                                          document=document,
+                                          start_ind=ent.start_char,
+                                          end_ind=ent.end_char).count() == 0:
+            # If this entity doesn't exist already
+            ann_ent = AnnotatedEntity()
+            ann_ent.user = user
+            ann_ent.project = project
+            ann_ent.document = document
+            ann_ent.entity = entity
+            ann_ent.value = ent.text
+            ann_ent.start_ind = ent.start_char
+            ann_ent.end_ind = ent.end_char
+            ann_ent.acc = ent._.context_similarity
+
+            MIN_ACC = cat.config.linking.get('similarity_threshold_trainer', 0.2)
+            if ent._.context_similarity < MIN_ACC:
+                ann_ent.deleted = True
+                ann_ent.validated = True
+
+            ann_ent.save()
 
 
 def _create_linked_codes(mod: Union[Type[ICDCode], Type[OPCSCode]], linked_codes: List[Dict], cdb_model: ConceptDB):
@@ -235,18 +227,6 @@ def train_medcat(cat, project, document):
                           negative=ann.deleted,
                           devalue_others=manually_created)
 
-            # Add entity to cui_counter
-            pcc = ProjectCuiCounter.objects.filter(project=project, entity=ann.entity).first()
-            if pcc is not None:
-                pcc.count = pcc.count + 1
-                pcc.save()
-            else:
-                pcc = ProjectCuiCounter()
-                pcc.project = project
-                pcc.entity = ann.entity
-                pcc.count = 1
-                pcc.save()
-
     # Completely remove concept names that the user killed
     killed_anns = AnnotatedEntity.objects.filter(project=project, document=document, killed=True)
     for ann in killed_anns: