Feature/sckan 275 (#250)

* SCKAN-275 feat: Add layer and region models

* SCKAN-275 feat: Update neurondm ingestion

* SCKAN-275 feat: WIP - Update anatomical entity model

* SCKAN-275 feat: WIP - Remove old anatomical entity model

* SCKAN-275 feat: Connect new anatomical entity model

* SCKAN-275 feat: Add basic admin panel controls

* SCKAN-275 feat: Update models

* SCKAN-275 feat: Add basic serializers

* SCKAN-275 refactor: Split ingestion script in multiple modules

* SCKAN-275 feat: Update changes detector

* SCKAN-275 feat: Update validators

* SCKAN-275 feat: Update ingest anatomical entity script

* SCKAN-275 fix: Fix typo

* SCKAN-275 feat: Update frontend to be compliant with model changes (basic functionalities only)

* SCKAN-275 feat: Update Anatomical Entity Admin page

* SCKAN-275 feat: Add automatic creation of anatomical entities

* SCKAN-275 feat: Update admin page names

* SCKAN-275 feat: Add anatomical entity constraint

* chore(): added feature to use a local PostgreSQL database

* SCKAN-275 feat: Set name as string representation of AnatomicalEntity

* Feature/csckan-277 (#252)

* #277 add ontology_uri property to AnatomicalEntity

* #277 - simplify the @Property for name and ontology_uri in AE

---------

Co-authored-by: Zoran Sinnema <[email protected]>
Co-authored-by: D. Gopal Krishna <[email protected]>

README.md

@@ -48,17 +48,50 @@ python3 manage.py migrate
 python3 manage.py runsslserver
 ```
 
-### Running on docker with docker-compose
+### Running the Backend on docker with docker-compose
 the command below will start a docker container that maps/uses the backend folder
 into the container. It will also start the Django development server with DEBUG=True
 
 ```bash
 BUILDKIT_PROGRESS=plain docker-compose -f docker-compose-dev.yaml up --build
 ```
 
-to stop:
+### Running the PostgreSQL database with docker-compose
+the command below will start a docker container that runs the PostgreSQL database.
+To use it within your development Django server you need to set the following env vars
+in your launch(file)
+
+```
+USE_PG=True
+DB_HOST=localhost
+DB_PORT=5432
+DB_NAME=composer
+DB_USER=composer
+DB_PASSWORD=composer
+```
+
+To start the database server run this command:
+```
+docker-compose --file docker-compose-db.yaml up --build
+```
+
+to stop the database run this command:
 ```bash
-docker-compose -f docker-compose-dev.yaml down
+docker-compose -f docker-compose-db.yaml down
+```
+
+Example to run the backend using the Docker PostgreSQL database
+```
+cd backend
+
+export USE_PG=True
+export DB_HOST=localhost
+export DB_PORT=5432
+export DB_NAME=composer
+export DB_USER=composer
+export DB_PASSWORD=composer
+
+python ./manage.py runsslserver
 ```
 
 ### Ingest sample NLP data

backend/backend/settings.py

@@ -24,6 +24,7 @@
 
 # SECURITY WARNING: don't run with debug turned on in production!
 PRODUCTION = os.environ.get("PRODUCTION", "False").lower() in ("true", "1")
+USE_PG = os.environ.get("USE_PG", "False").lower() in ("true", "1")
 DEBUG = os.environ.get("DEBUG", str(not PRODUCTION)).lower() in ("true", "1")
 
 ALLOWED_HOSTS = [
@@ -101,7 +102,7 @@
 
 # Database
 # https://docs.djangoproject.com/en/4.1/ref/settings/#databases
-if PRODUCTION:
+if PRODUCTION or USE_PG:
     DATABASES = {
         "default": {
             "ENGINE": "django.db.backends.postgresql",
@@ -148,7 +149,7 @@
 
 USE_I18N = True
 
-USE_TZ = False
+USE_TZ = True
 
 
 # Static files (CSS, JavaScript, Images)

backend/composer/admin.py

@@ -6,7 +6,6 @@
 from fsm_admin.mixins import FSMTransitionMixin
 
 from composer.models import (
-    AnatomicalEntity,
     Phenotype,
     Sex,
     ConnectivityStatement,
@@ -19,7 +18,9 @@
     Tag,
     Via,
     FunctionalCircuitRole,
-    ProjectionPhenotype, Destination, Synonym
+    ProjectionPhenotype, Destination, Synonym, AnatomicalEntityMeta, Layer, Region,
+    AnatomicalEntityIntersection,
+    AnatomicalEntity
 )
 
 
@@ -95,10 +96,36 @@ class SynonymInline(admin.TabularInline):
 
 
 class AnatomicalEntityAdmin(admin.ModelAdmin):
+    search_fields = ('simple_entity__name', 'region_layer__layer__name', 'region_layer__region__name')
+
+    def get_model_perms(self, request):
+        """
+        Return empty dict to hide the model from admin index.
+        """
+        return {}
+
+
+class AnatomicalEntityMetaAdmin(admin.ModelAdmin):
     list_display = ("name", "ontology_uri")
     list_display_links = ("name", "ontology_uri")
-    search_fields = ("name",)  # or ("^name",) for search to start with
-    inlines = [SynonymInline]
+    search_fields = ("name",)
+
+
+class LayerAdmin(admin.ModelAdmin):
+    list_display = ('name', 'ontology_uri',)
+    search_fields = ('name',)
+
+
+class RegionAdmin(admin.ModelAdmin):
+    list_display = ('name', 'ontology_uri',)
+    search_fields = ('name',)
+    filter_horizontal = ('layers',)
+
+
+class AnatomicalEntityIntersectionAdmin(admin.ModelAdmin):
+    list_display = ('layer', 'region',)
+    list_filter = ('layer', 'region',)
+    raw_id_fields = ('layer', 'region',)
 
 
 class ViaInline(SortableStackedInline):
@@ -143,8 +170,6 @@ class ConnectivityStatementAdmin(
         "sentence__pmid",
         "sentence__pmcid",
         "knowledge_statement",
-        "origins__name",
-        "destinations__anatomical_entities__name",
     )
 
     fieldsets = ()
@@ -205,6 +230,10 @@ def get_form(self, request, obj=None, change=False, **kwargs):
 admin.site.register(User, UserAdmin)
 
 #
+admin.site.register(AnatomicalEntityMeta, AnatomicalEntityMetaAdmin)
+admin.site.register(Layer, LayerAdmin)
+admin.site.register(Region, RegionAdmin)
+admin.site.register(AnatomicalEntityIntersection, AnatomicalEntityIntersectionAdmin)
 admin.site.register(AnatomicalEntity, AnatomicalEntityAdmin)
 admin.site.register(Phenotype)
 admin.site.register(Sex)

backend/composer/api/serializers.py

@@ -19,7 +19,7 @@
     Sentence,
     Specie,
     Tag,
-    Via, Destination,
+    Via, Destination, AnatomicalEntityIntersection, Region, Layer, AnatomicalEntityMeta,
 )
 from ..services.connections_service import get_complete_from_entities_for_destination, \
     get_complete_from_entities_for_via
@@ -80,17 +80,63 @@ class Meta:
         dept = 2
 
 
-class AnatomicalEntitySerializer(UniqueFieldsMixin, serializers.ModelSerializer):
-    """Anatomical Entity"""
+class LayerSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Layer
+        fields = (
+            "id",
+            "name",
+            "ontology_uri",
+        )
+
+
+class RegionSerializer(serializers.ModelSerializer):
+    layers = LayerSerializer(many=True, read_only=True)
 
     class Meta:
-        model = AnatomicalEntity
+        model = Region
+        fields = (
+            "id",
+            "name",
+            "ontology_uri",
+            "layers",
+        )
+
+
+class AnatomicalEntityMetaSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = AnatomicalEntityMeta
         fields = (
             "id",
             "name",
             "ontology_uri",
         )
-        read_only_fields = ("ontology_uri",)
+
+
+class AnatomicalEntityIntersectionSerializer(serializers.ModelSerializer):
+    layer = LayerSerializer(read_only=True)
+    region = RegionSerializer(read_only=True)
+
+    class Meta:
+        model = AnatomicalEntityIntersection
+        fields = (
+            "id",
+            "layer",
+            "region",
+        )
+
+
+class AnatomicalEntitySerializer(serializers.ModelSerializer):
+    simple_entity = AnatomicalEntityMetaSerializer(read_only=True)
+    region_layer = AnatomicalEntityIntersectionSerializer(read_only=True)
+
+    class Meta:
+        model = AnatomicalEntity
+        fields = (
+            "id",
+            "simple_entity",
+            "region_layer",
+        )
 
 
 class NoteSerializer(serializers.ModelSerializer):
@@ -486,7 +532,6 @@ def get_statement_preview(self, instance):
             self.context['journey'] = instance.get_journey()
         return self.create_statement_preview(instance, self.context['journey'])
 
-
     def create_statement_preview(self, instance, journey):
         sex = instance.sex.sex_str if instance.sex else None
 
@@ -514,7 +559,7 @@ def create_statement_preview(self, instance, journey):
             statement = f"In {sex or ''} {species}, the {phenotype.lower()} connection goes {journey_sentence}.\n"
         else:
             statement = f"A {phenotype.lower()} connection goes {journey_sentence}.\n"
-        
+
         statement += f"This "
         if projection:
             statement += f"{projection.lower()} "

backend/composer/management/commands/ingest_anatomical_entities.py

@@ -3,7 +3,7 @@
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 from django.db.utils import IntegrityError
-from composer.models import AnatomicalEntity, Synonym
+from composer.models import AnatomicalEntity, Synonym, AnatomicalEntityMeta
 
 URI = "o"
 NAME = "o_label"
@@ -24,18 +24,23 @@ def _process_anatomical_entity(self, name, ontology_uri, synonym, show_complete_
         try:
             is_first_occurrence = ontology_uri not in processed_uris
 
-            anatomical_entity, created = AnatomicalEntity.objects.get_or_create(
+            entity_meta, meta_created = AnatomicalEntityMeta.objects.get_or_create(
                 ontology_uri=ontology_uri,
                 defaults={"name": name},
             )
-            if not created and is_first_occurrence:
-                if anatomical_entity.name != name:
-                    anatomical_entity.name = name
-                    anatomical_entity.save()
-                    if show_complete_logs:
-                        self.stdout.write(
-                            self.style.SUCCESS(f"Updated {anatomical_entity.ontology_uri} name to {name}.")
-                        )
+
+            # Update the name if it has changed and this is the first occurrence of the ontology URI
+            if not meta_created and is_first_occurrence and entity_meta.name != name:
+                entity_meta.name = name
+                entity_meta.save()
+                if show_complete_logs:
+                    self.stdout.write(
+                        self.style.SUCCESS(f"Updated {entity_meta.ontology_uri} name to {name}.")
+                    )
+
+            anatomical_entity, created = AnatomicalEntity.objects.get_or_create(
+                simple_entity=entity_meta
+            )
 
             processed_uris.add(ontology_uri)
 
@@ -45,7 +50,9 @@ def _process_anatomical_entity(self, name, ontology_uri, synonym, show_complete_
                     unique_synonyms[synonym_key] = Synonym(anatomical_entity=anatomical_entity, name=synonym)
                     if show_complete_logs:
                         self.stdout.write(
-                            self.style.SUCCESS(f"Synonym '{synonym}' added for {anatomical_entity.ontology_uri}."))
+                            self.style.SUCCESS(
+                                f"Synonym '{synonym}' added for {anatomical_entity.simple_entity.ontology_uri}.")
+                        )
         except IntegrityError as e:
             self.stdout.write(self.style.ERROR(f"Error processing {ontology_uri}: {e}"))
 

backend/composer/management/commands/ingest_statements.py

@@ -1,6 +1,6 @@
 import time
 
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 from composer.services.cs_ingestion.cs_ingestion_services import ingest_statements
 
 
@@ -13,13 +13,19 @@ def add_arguments(self, parser):
             action='store_true',
             help='Set this flag to update upstream statements.',
         )
+        parser.add_argument(
+            '--update_anatomical_entities',
+            action='store_true',
+            help='Set this flag to try move anatomical entities to specific layer, region.',
+        )
 
     def handle(self, *args, **options):
         update_upstream = options['update_upstream']
+        update_anatomical_entities = options['update_anatomical_entities']
 
         start_time = time.time()
 
-        ingest_statements(update_upstream)
+        ingest_statements(update_upstream, update_anatomical_entities)
 
         end_time = time.time()