diff --git a/README.md b/README.md index 02ff5380..49af870a 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ python3 manage.py migrate python3 manage.py runsslserver ``` -### Running on docker with docker-compose +### Running the Backend on docker with docker-compose the command below will start a docker container that maps/uses the backend folder into the container. It will also start the Django development server with DEBUG=True @@ -56,9 +56,42 @@ into the container. It will also start the Django development server with DEBUG= BUILDKIT_PROGRESS=plain docker-compose -f docker-compose-dev.yaml up --build ``` -to stop: +### Running the PostgreSQL database with docker-compose +the command below will start a docker container that runs the PostgreSQL database. +To use it within your development Django server you need to set the following env vars +in your launch(file) + +``` +USE_PG=True +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=composer +DB_USER=composer +DB_PASSWORD=composer +``` + +To start the database server run this command: +``` +docker-compose --file docker-compose-db.yaml up --build +``` + +to stop the database run this command: ```bash -docker-compose -f docker-compose-dev.yaml down +docker-compose -f docker-compose-db.yaml down +``` + +Example to run the backend using the Docker PostgreSQL database +``` +cd backend + +export USE_PG=True +export DB_HOST=localhost +export DB_PORT=5432 +export DB_NAME=composer +export DB_USER=composer +export DB_PASSWORD=composer + +python ./manage.py runsslserver ``` ### Ingest sample NLP data diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 7bcb7242..cb9aa67e 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -24,6 +24,7 @@ # SECURITY WARNING: don't run with debug turned on in production! PRODUCTION = os.environ.get("PRODUCTION", "False").lower() in ("true", "1") +USE_PG = os.environ.get("USE_PG", "False").lower() in ("true", "1") DEBUG = os.environ.get("DEBUG", str(not PRODUCTION)).lower() in ("true", "1") ALLOWED_HOSTS = [ @@ -101,7 +102,7 @@ # Database # https://docs.djangoproject.com/en/4.1/ref/settings/#databases -if PRODUCTION: +if PRODUCTION or USE_PG: DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", @@ -148,7 +149,7 @@ USE_I18N = True -USE_TZ = False +USE_TZ = True # Static files (CSS, JavaScript, Images) diff --git a/backend/composer/admin.py b/backend/composer/admin.py index c3b76394..e6c09687 100644 --- a/backend/composer/admin.py +++ b/backend/composer/admin.py @@ -6,7 +6,6 @@ from fsm_admin.mixins import FSMTransitionMixin from composer.models import ( - AnatomicalEntity, Phenotype, Sex, ConnectivityStatement, @@ -19,7 +18,9 @@ Tag, Via, FunctionalCircuitRole, - ProjectionPhenotype, Destination, Synonym + ProjectionPhenotype, Destination, Synonym, AnatomicalEntityMeta, Layer, Region, + AnatomicalEntityIntersection, + AnatomicalEntity ) @@ -95,10 +96,36 @@ class SynonymInline(admin.TabularInline): class AnatomicalEntityAdmin(admin.ModelAdmin): + search_fields = ('simple_entity__name', 'region_layer__layer__name', 'region_layer__region__name') + + def get_model_perms(self, request): + """ + Return empty dict to hide the model from admin index. + """ + return {} + + +class AnatomicalEntityMetaAdmin(admin.ModelAdmin): list_display = ("name", "ontology_uri") list_display_links = ("name", "ontology_uri") - search_fields = ("name",) # or ("^name",) for search to start with - inlines = [SynonymInline] + search_fields = ("name",) + + +class LayerAdmin(admin.ModelAdmin): + list_display = ('name', 'ontology_uri',) + search_fields = ('name',) + + +class RegionAdmin(admin.ModelAdmin): + list_display = ('name', 'ontology_uri',) + search_fields = ('name',) + filter_horizontal = ('layers',) + + +class AnatomicalEntityIntersectionAdmin(admin.ModelAdmin): + list_display = ('layer', 'region',) + list_filter = ('layer', 'region',) + raw_id_fields = ('layer', 'region',) class ViaInline(SortableStackedInline): @@ -143,8 +170,6 @@ class ConnectivityStatementAdmin( "sentence__pmid", "sentence__pmcid", "knowledge_statement", - "origins__name", - "destinations__anatomical_entities__name", ) fieldsets = () @@ -205,6 +230,10 @@ def get_form(self, request, obj=None, change=False, **kwargs): admin.site.register(User, UserAdmin) # +admin.site.register(AnatomicalEntityMeta, AnatomicalEntityMetaAdmin) +admin.site.register(Layer, LayerAdmin) +admin.site.register(Region, RegionAdmin) +admin.site.register(AnatomicalEntityIntersection, AnatomicalEntityIntersectionAdmin) admin.site.register(AnatomicalEntity, AnatomicalEntityAdmin) admin.site.register(Phenotype) admin.site.register(Sex) diff --git a/backend/composer/api/serializers.py b/backend/composer/api/serializers.py index 123b67c3..d4a6cab1 100644 --- a/backend/composer/api/serializers.py +++ b/backend/composer/api/serializers.py @@ -19,7 +19,7 @@ Sentence, Specie, Tag, - Via, Destination, + Via, Destination, AnatomicalEntityIntersection, Region, Layer, AnatomicalEntityMeta, ) from ..services.connections_service import get_complete_from_entities_for_destination, \ get_complete_from_entities_for_via @@ -80,17 +80,63 @@ class Meta: dept = 2 -class AnatomicalEntitySerializer(UniqueFieldsMixin, serializers.ModelSerializer): - """Anatomical Entity""" +class LayerSerializer(serializers.ModelSerializer): + class Meta: + model = Layer + fields = ( + "id", + "name", + "ontology_uri", + ) + + +class RegionSerializer(serializers.ModelSerializer): + layers = LayerSerializer(many=True, read_only=True) class Meta: - model = AnatomicalEntity + model = Region + fields = ( + "id", + "name", + "ontology_uri", + "layers", + ) + + +class AnatomicalEntityMetaSerializer(serializers.ModelSerializer): + class Meta: + model = AnatomicalEntityMeta fields = ( "id", "name", "ontology_uri", ) - read_only_fields = ("ontology_uri",) + + +class AnatomicalEntityIntersectionSerializer(serializers.ModelSerializer): + layer = LayerSerializer(read_only=True) + region = RegionSerializer(read_only=True) + + class Meta: + model = AnatomicalEntityIntersection + fields = ( + "id", + "layer", + "region", + ) + + +class AnatomicalEntitySerializer(serializers.ModelSerializer): + simple_entity = AnatomicalEntityMetaSerializer(read_only=True) + region_layer = AnatomicalEntityIntersectionSerializer(read_only=True) + + class Meta: + model = AnatomicalEntity + fields = ( + "id", + "simple_entity", + "region_layer", + ) class NoteSerializer(serializers.ModelSerializer): @@ -486,7 +532,6 @@ def get_statement_preview(self, instance): self.context['journey'] = instance.get_journey() return self.create_statement_preview(instance, self.context['journey']) - def create_statement_preview(self, instance, journey): sex = instance.sex.sex_str if instance.sex else None @@ -514,7 +559,7 @@ def create_statement_preview(self, instance, journey): statement = f"In {sex or ''} {species}, the {phenotype.lower()} connection goes {journey_sentence}.\n" else: statement = f"A {phenotype.lower()} connection goes {journey_sentence}.\n" - + statement += f"This " if projection: statement += f"{projection.lower()} " diff --git a/backend/composer/management/commands/ingest_anatomical_entities.py b/backend/composer/management/commands/ingest_anatomical_entities.py index 3b275251..5fa0b91e 100644 --- a/backend/composer/management/commands/ingest_anatomical_entities.py +++ b/backend/composer/management/commands/ingest_anatomical_entities.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand, CommandError from django.db import transaction from django.db.utils import IntegrityError -from composer.models import AnatomicalEntity, Synonym +from composer.models import AnatomicalEntity, Synonym, AnatomicalEntityMeta URI = "o" NAME = "o_label" @@ -24,18 +24,23 @@ def _process_anatomical_entity(self, name, ontology_uri, synonym, show_complete_ try: is_first_occurrence = ontology_uri not in processed_uris - anatomical_entity, created = AnatomicalEntity.objects.get_or_create( + entity_meta, meta_created = AnatomicalEntityMeta.objects.get_or_create( ontology_uri=ontology_uri, defaults={"name": name}, ) - if not created and is_first_occurrence: - if anatomical_entity.name != name: - anatomical_entity.name = name - anatomical_entity.save() - if show_complete_logs: - self.stdout.write( - self.style.SUCCESS(f"Updated {anatomical_entity.ontology_uri} name to {name}.") - ) + + # Update the name if it has changed and this is the first occurrence of the ontology URI + if not meta_created and is_first_occurrence and entity_meta.name != name: + entity_meta.name = name + entity_meta.save() + if show_complete_logs: + self.stdout.write( + self.style.SUCCESS(f"Updated {entity_meta.ontology_uri} name to {name}.") + ) + + anatomical_entity, created = AnatomicalEntity.objects.get_or_create( + simple_entity=entity_meta + ) processed_uris.add(ontology_uri) @@ -45,7 +50,9 @@ def _process_anatomical_entity(self, name, ontology_uri, synonym, show_complete_ unique_synonyms[synonym_key] = Synonym(anatomical_entity=anatomical_entity, name=synonym) if show_complete_logs: self.stdout.write( - self.style.SUCCESS(f"Synonym '{synonym}' added for {anatomical_entity.ontology_uri}.")) + self.style.SUCCESS( + f"Synonym '{synonym}' added for {anatomical_entity.simple_entity.ontology_uri}.") + ) except IntegrityError as e: self.stdout.write(self.style.ERROR(f"Error processing {ontology_uri}: {e}")) diff --git a/backend/composer/management/commands/ingest_statements.py b/backend/composer/management/commands/ingest_statements.py index 09ec24ea..736e6c70 100644 --- a/backend/composer/management/commands/ingest_statements.py +++ b/backend/composer/management/commands/ingest_statements.py @@ -1,6 +1,6 @@ import time -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from composer.services.cs_ingestion.cs_ingestion_services import ingest_statements @@ -13,13 +13,19 @@ def add_arguments(self, parser): action='store_true', help='Set this flag to update upstream statements.', ) + parser.add_argument( + '--update_anatomical_entities', + action='store_true', + help='Set this flag to try move anatomical entities to specific layer, region.', + ) def handle(self, *args, **options): update_upstream = options['update_upstream'] + update_anatomical_entities = options['update_anatomical_entities'] start_time = time.time() - ingest_statements(update_upstream) + ingest_statements(update_upstream, update_anatomical_entities) end_time = time.time() diff --git a/backend/composer/migrations/0046_anatomicalentityintersection_anatomicalentitymeta_and_more.py b/backend/composer/migrations/0046_anatomicalentityintersection_anatomicalentitymeta_and_more.py new file mode 100644 index 00000000..45fd0a32 --- /dev/null +++ b/backend/composer/migrations/0046_anatomicalentityintersection_anatomicalentitymeta_and_more.py @@ -0,0 +1,188 @@ +# Generated by Django 4.1.4 on 2024-03-20 13:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0045_alter_synonym_unique_together"), + ] + + operations = [ + migrations.CreateModel( + name="AnatomicalEntityIntersection", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ], + ), + migrations.CreateModel( + name="AnatomicalEntityMeta", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(db_index=True, max_length=200)), + ("ontology_uri", models.URLField(unique=True)), + ], + options={ + "verbose_name_plural": "Anatomical Entities Meta", + "ordering": ["name"], + }, + ), + migrations.AlterField( + model_name="synonym", + name="anatomical_entity", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="synonyms", + to="composer.anatomicalentity", + ), + ), + migrations.CreateModel( + name="Layer", + fields=[ + ( + "anatomicalentitymeta_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="composer.anatomicalentitymeta", + ), + ), + ], + bases=("composer.anatomicalentitymeta",), + ), + migrations.CreateModel( + name="AnatomicalEntityNew", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "region_layer", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="composer.anatomicalentityintersection", + ), + ), + ( + "simple_entity", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="composer.anatomicalentitymeta", + ), + ), + ], + ), + migrations.AddField( + model_name="connectivitystatement", + name="origins_new", + field=models.ManyToManyField( + related_name="origins_relations_new", to="composer.anatomicalentitynew" + ), + ), + migrations.AddField( + model_name="destination", + name="anatomical_entities_new", + field=models.ManyToManyField( + blank=True, + related_name="destination_connection_layers_new", + to="composer.anatomicalentitynew", + ), + ), + migrations.AddField( + model_name="destination", + name="from_entities_new", + field=models.ManyToManyField(blank=True, to="composer.anatomicalentitynew"), + ), + migrations.AddField( + model_name="synonym", + name="anatomical_entity_new", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="synonyms_new", + to="composer.anatomicalentitynew", + ), + ), + migrations.AddField( + model_name="via", + name="anatomical_entities_new", + field=models.ManyToManyField( + blank=True, + related_name="via_connection_layers", + to="composer.anatomicalentitynew", + ), + ), + migrations.AddField( + model_name="via", + name="from_entities_new", + field=models.ManyToManyField(blank=True, to="composer.anatomicalentitynew"), + ), + migrations.CreateModel( + name="Region", + fields=[ + ( + "anatomicalentitymeta_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="composer.anatomicalentitymeta", + ), + ), + ( + "layers", + models.ManyToManyField( + through="composer.AnatomicalEntityIntersection", + to="composer.layer", + ), + ), + ], + bases=("composer.anatomicalentitymeta",), + ), + migrations.AddField( + model_name="anatomicalentityintersection", + name="layer", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="composer.layer" + ), + ), + migrations.AddField( + model_name="anatomicalentityintersection", + name="region", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="composer.region" + ), + ), + ] diff --git a/backend/composer/migrations/0047_auto_20240320_1340.py b/backend/composer/migrations/0047_auto_20240320_1340.py new file mode 100644 index 00000000..30b78c3f --- /dev/null +++ b/backend/composer/migrations/0047_auto_20240320_1340.py @@ -0,0 +1,52 @@ +# Generated by Django 4.1.4 on 2024-03-20 13:40 + +from django.db import migrations + + +def copy_anatomical_entities_to_new(apps, schema_editor): + AnatomicalEntity = apps.get_model('composer', 'AnatomicalEntity') + AnatomicalEntityNew = apps.get_model('composer', 'AnatomicalEntityNew') + AnatomicalEntityMeta = apps.get_model('composer', 'AnatomicalEntityMeta') + Destination = apps.get_model('composer', 'Destination') + Via = apps.get_model('composer', 'Via') + ConnectivityStatement = apps.get_model('composer', 'ConnectivityStatement') + Synonym = apps.get_model('composer', 'Synonym') + + # Copy AnatomicalEntity instances to AnatomicalEntityNew + for entity in AnatomicalEntity.objects.all(): + new_entity_meta = AnatomicalEntityMeta.objects.create( + name=entity.name, + ontology_uri=entity.ontology_uri, + ) + + new_entity = AnatomicalEntityNew.objects.create(simple_entity=new_entity_meta) + + # Update ManyToMany relationships for Destination + for destination in Destination.objects.filter(anatomical_entities=entity): + destination.anatomical_entities_new.add(new_entity) + + for destination in Destination.objects.filter(from_entities=entity): + destination.from_entities_new.add(new_entity) + + # Update ManyToMany relationships for Via + for via in Via.objects.filter(anatomical_entities=entity): + via.anatomical_entities_new.add(new_entity) + + for via in Via.objects.filter(from_entities=entity): + via.from_entities_new.add(new_entity) + + # Update ManyToMany relationships for ConnectivityStatement origins + for cs in ConnectivityStatement.objects.filter(origins=entity): + cs.origins_new.add(new_entity) + + Synonym.objects.filter(anatomical_entity=entity).update( + anatomical_entity_new=new_entity + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0046_anatomicalentityintersection_anatomicalentitymeta_and_more"), + ] + + operations = [migrations.RunPython(copy_anatomical_entities_to_new), ] diff --git a/backend/composer/migrations/0048_remove_connectivitystatement_origins_and_more.py b/backend/composer/migrations/0048_remove_connectivitystatement_origins_and_more.py new file mode 100644 index 00000000..b5721be0 --- /dev/null +++ b/backend/composer/migrations/0048_remove_connectivitystatement_origins_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.1.4 on 2024-03-20 14:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0047_auto_20240320_1340"), + ] + + operations = [ + migrations.RemoveField( + model_name="connectivitystatement", + name="origins", + ), + migrations.RemoveField( + model_name="destination", + name="anatomical_entities", + ), + migrations.RemoveField( + model_name="destination", + name="from_entities", + ), + migrations.RemoveField( + model_name="via", + name="anatomical_entities", + ), + migrations.RemoveField( + model_name="via", + name="from_entities", + ), + migrations.AlterUniqueTogether( + name="synonym", + unique_together={("anatomical_entity_new", "name")}, + ), + migrations.RemoveField( + model_name="synonym", + name="anatomical_entity", + ), + migrations.DeleteModel( + name="AnatomicalEntity", + ), + ] diff --git a/backend/composer/migrations/0049_rename_anatomicalentitynew_anatomicalentity_and_more.py b/backend/composer/migrations/0049_rename_anatomicalentitynew_anatomicalentity_and_more.py new file mode 100644 index 00000000..3df00901 --- /dev/null +++ b/backend/composer/migrations/0049_rename_anatomicalentitynew_anatomicalentity_and_more.py @@ -0,0 +1,55 @@ +# Generated by Django 4.1.4 on 2024-03-20 14:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0048_remove_connectivitystatement_origins_and_more"), + ] + + operations = [ + migrations.RenameModel( + old_name="AnatomicalEntityNew", + new_name="AnatomicalEntity", + ), + migrations.RenameField( + model_name="destination", + old_name="from_entities_new", + new_name="from_entities", + ), + migrations.RenameField( + model_name="via", + old_name="anatomical_entities_new", + new_name="anatomical_entities", + ), + migrations.RenameField( + model_name="via", + old_name="from_entities_new", + new_name="from_entities", + ), + migrations.RenameField( + model_name="connectivitystatement", + old_name="origins_new", + new_name="origins", + ), + migrations.RenameField( + model_name="destination", + old_name="anatomical_entities_new", + new_name="anatomical_entities", + ), + migrations.RenameField( + model_name="synonym", + old_name="anatomical_entity_new", + new_name="anatomical_entity", + ), + migrations.AlterUniqueTogether( + name="synonym", + unique_together=set(), + ), + migrations.AlterUniqueTogether( + name="synonym", + unique_together={("anatomical_entity", "name")}, + ), + ] diff --git a/backend/composer/migrations/0050_alter_anatomicalentityintersection_options_and_more.py b/backend/composer/migrations/0050_alter_anatomicalentityintersection_options_and_more.py new file mode 100644 index 00000000..465e2b14 --- /dev/null +++ b/backend/composer/migrations/0050_alter_anatomicalentityintersection_options_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 4.1.4 on 2024-03-21 16:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("composer", "0049_rename_anatomicalentitynew_anatomicalentity_and_more"), + ] + + operations = [ + migrations.AlterModelOptions( + name="anatomicalentityintersection", + options={ + "verbose_name": "Region/Layer Combination", + "verbose_name_plural": "Region/Layer Combinations", + }, + ), + migrations.AlterModelOptions( + name="anatomicalentitymeta", + options={ + "ordering": ["name"], + "verbose_name": "Anatomical Entity", + "verbose_name_plural": "Anatomical Entities", + }, + ), + migrations.AlterField( + model_name="anatomicalentity", + name="region_layer", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="composer.anatomicalentityintersection", + ), + ), + migrations.AlterField( + model_name="connectivitystatement", + name="origins", + field=models.ManyToManyField( + related_name="origins_relations", to="composer.anatomicalentity" + ), + ), + migrations.AlterField( + model_name="destination", + name="anatomical_entities", + field=models.ManyToManyField( + blank=True, + related_name="destination_connection_layers", + to="composer.anatomicalentity", + ), + ), + migrations.AlterField( + model_name="synonym", + name="anatomical_entity", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="synonyms", + to="composer.anatomicalentity", + ), + ), + migrations.AddConstraint( + model_name="anatomicalentity", + constraint=models.CheckConstraint( + check=models.Q( + models.Q( + ("region_layer__isnull", True), ("simple_entity__isnull", False) + ), + models.Q( + ("region_layer__isnull", False), ("simple_entity__isnull", True) + ), + _connector="OR", + ), + name="check_anatomical_entity_exclusivity", + ), + ), + ] diff --git a/backend/composer/models.py b/backend/composer/models.py index db0557e6..f6f08520 100644 --- a/backend/composer/models.py +++ b/backend/composer/models.py @@ -1,8 +1,7 @@ from django.contrib.auth.models import User from django.db import models, transaction -from django.db.models import Q +from django.db.models import Q, CheckConstraint from django.db.models.expressions import F -from django.db.models.functions import Upper from django.forms.widgets import Input as InputWidget from django_fsm import FSMField, transition @@ -225,9 +224,7 @@ class Meta: verbose_name_plural = "Projection Phenotypes" -class AnatomicalEntity(models.Model): - """Anatomical Entity""" - +class AnatomicalEntityMeta(models.Model): name = models.CharField(max_length=200, db_index=True) ontology_uri = models.URLField(unique=True) @@ -236,11 +233,61 @@ def __str__(self): class Meta: ordering = ["name"] + verbose_name = "Anatomical Entity" verbose_name_plural = "Anatomical Entities" +class Layer(AnatomicalEntityMeta): + ... + + +class Region(AnatomicalEntityMeta): + ... + layers = models.ManyToManyField(Layer, through='AnatomicalEntityIntersection') + + +class AnatomicalEntityIntersection(models.Model): + layer = models.ForeignKey(Layer, on_delete=models.CASCADE) + region = models.ForeignKey(Region, on_delete=models.CASCADE) + + class Meta: + verbose_name = "Region/Layer Combination" + verbose_name_plural = "Region/Layer Combinations" + + +class AnatomicalEntity(models.Model): + simple_entity = models.OneToOneField(AnatomicalEntityMeta, on_delete=models.CASCADE, null=True, blank=True) + region_layer = models.ForeignKey(AnatomicalEntityIntersection, on_delete=models.CASCADE, null=True, blank=True) + + @property + def name(self): + return self.simple_entity.name if self.simple_entity \ + else f'{self.region_layer.region.name},{self.region_layer.layer.name}' + + @property + def ontology_uri(self): + return self.simple_entity.ontology_uri if self.simple_entity \ + else f'{self.region_layer.region.ontology_uri},{self.region_layer.layer.ontology_uri}' + + + def __str__(self): + return self.name + + class Meta: + constraints = [ + CheckConstraint( + check=( + Q(simple_entity__isnull=False, region_layer__isnull=True) | + Q(simple_entity__isnull=True, region_layer__isnull=False) + ), + name='check_anatomical_entity_exclusivity' + ) + ] + + class Synonym(models.Model): - anatomical_entity = models.ForeignKey(AnatomicalEntity, on_delete=models.CASCADE, related_name="synonyms") + anatomical_entity = models.ForeignKey(AnatomicalEntity, on_delete=models.CASCADE, + related_name="synonyms", null=True) name = models.CharField(max_length=200, db_index=True) class Meta: @@ -630,6 +677,7 @@ class Destination(AbstractConnectionLayer): on_delete=models.CASCADE, related_name="destinations" # Overridden related_name ) + anatomical_entities = models.ManyToManyField(AnatomicalEntity, blank=True, related_name='destination_connection_layers') @@ -651,7 +699,8 @@ class Meta: class Via(AbstractConnectionLayer): - anatomical_entities = models.ManyToManyField(AnatomicalEntity, blank=True, related_name='via_connection_layers') + anatomical_entities = models.ManyToManyField(AnatomicalEntity, blank=True, + related_name='via_connection_layers') objects = ViaManager() diff --git a/backend/composer/services/cs_ingestion/cs_ingestion_services.py b/backend/composer/services/cs_ingestion/cs_ingestion_services.py index a540cfeb..da128ac9 100644 --- a/backend/composer/services/cs_ingestion/cs_ingestion_services.py +++ b/backend/composer/services/cs_ingestion/cs_ingestion_services.py @@ -1,45 +1,38 @@ import logging -from datetime import datetime -from typing import List, Dict, Optional, Tuple, Set, Any from django.db import transaction -from composer.models import AnatomicalEntity, Sentence, ConnectivityStatement, Sex, FunctionalCircuitRole, \ - ProjectionPhenotype, Phenotype, Specie, Provenance, Via, Note, User, Destination -from .helpers import get_value_or_none, found_entity, \ - ORIGINS, DESTINATIONS, VIAS, LABEL, SEX, SPECIES, ID, FORWARD_CONNECTION, SENTENCE_NUMBER, \ - FUNCTIONAL_CIRCUIT_ROLE, CIRCUIT_TYPE, CIRCUIT_TYPE_MAPPING, PHENOTYPE, OTHER_PHENOTYPE, NOTE_ALERT, PROVENANCE, \ - VALIDATION_ERRORS, STATE -from .logging_service import LoggerService, STATEMENT_INCORRECT_STATE, SENTENCE_INCORRECT_STATE -from .models import LoggableAnomaly, ValidationErrors, Severity +from composer.services.cs_ingestion.helpers.overwritable_helper import get_overwritable_statements +from composer.services.cs_ingestion.helpers.sentence_helper import get_or_create_sentence +from composer.services.cs_ingestion.helpers.validators import validate_statements +from .helpers.statement_helper import create_or_update_connectivity_statement, update_forward_connections +from .helpers.upstream_changes_helper import update_upstream_statements +from .logging_service import LoggerService +from .models import LoggableAnomaly, Severity from .neurondm_script import main as get_statements_from_neurondm -from ...enums import ( - CircuitType, - NoteType, - CSState, SentenceState -) - -NOW = datetime.now().strftime("%Y%m%d%H%M%S") logger_service = LoggerService() -def ingest_statements(update_upstream=False): +def ingest_statements(update_upstream=False, update_anatomical_entities=False): statements_list = get_statements_from_neurondm(logger_service_param=logger_service) overridable_statements = get_overwritable_statements(statements_list) - statements = validate_statements(overridable_statements) + statements = validate_statements(overridable_statements, update_anatomical_entities) successful_transaction = True try: with transaction.atomic(): for statement in statements: sentence, _ = get_or_create_sentence(statement) - create_or_update_connectivity_statement(statement, sentence) + create_or_update_connectivity_statement(statement, sentence, update_anatomical_entities) update_forward_connections(statements) + except Exception as e: logger_service.add_anomaly( - LoggableAnomaly(statement_id=None, entity_id=None, message=str(e), severity=Severity.ERROR)) + LoggableAnomaly(statement_id=None, entity_id=None, message=str(e), + severity=Severity.ERROR) + ) successful_transaction = False logging.error(f"Ingestion aborted due to {e}") @@ -49,515 +42,3 @@ def ingest_statements(update_upstream=False): if update_upstream: update_upstream_statements() logger_service.write_ingested_statements_to_file(statements) - - -def get_overwritable_statements(statements_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - overwritable_statements = [ - statement for statement in statements_list - if not has_invalid_sentence(statement) and not has_invalid_statement(statement) - ] - return overwritable_statements - - -def has_invalid_sentence(statement: Dict) -> bool: - try: - sentence = Sentence.objects.get(doi__iexact=statement[ID]) - except Sentence.DoesNotExist: - return False - return not can_sentence_be_overwritten(sentence, statement) - - -def has_invalid_statement(statement: Dict) -> bool: - try: - connectivity_statement = ConnectivityStatement.objects.get(reference_uri=statement[ID]) - except ConnectivityStatement.DoesNotExist: - return False - return not can_statement_be_overwritten(connectivity_statement, statement) - - -def can_statement_be_overwritten(connectivity_statement: ConnectivityStatement, statement) -> bool: - if connectivity_statement.state != CSState.EXPORTED and connectivity_statement.state != CSState.INVALID: - logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, STATEMENT_INCORRECT_STATE)) - return False - - return True - - -def can_sentence_be_overwritten(sentence: Sentence, statement: Dict) -> bool: - if sentence.state != SentenceState.COMPOSE_NOW: - logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, SENTENCE_INCORRECT_STATE)) - return False - return True - - -def validate_statements(statements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - db_reference_uris = set(ConnectivityStatement.objects.values_list('reference_uri', flat=True)) - input_statement_ids = {statement[ID] for statement in statements} - statement_ids = input_statement_ids.union(db_reference_uris) - - for statement in statements: - # Initialize validation_errors if not already present - if VALIDATION_ERRORS not in statement: - statement[VALIDATION_ERRORS] = ValidationErrors() - - # Validate entities, sex, and species, updating validation_errors accordingly - annotate_invalid_entities(statement) - annotate_invalid_sex(statement) - annotate_invalid_species(statement) - - # Validate forward connection - annotate_invalid_forward_connections(statement, statement_ids) - - return statements - - -def annotate_invalid_entities(statement: Dict) -> bool: - has_invalid_entities = False - - # Consolidate all URIs to check - uris_to_check = list(statement[ORIGINS].anatomical_entities) - uris_to_check.extend(uri for dest in statement[DESTINATIONS] for uri in dest.anatomical_entities) - uris_to_check.extend(uri for via in statement[VIAS] for uri in via.anatomical_entities) - - # Check all URIs and log if not found - for uri in uris_to_check: - if not found_entity(uri): - statement[VALIDATION_ERRORS].entities.add(uri) - has_invalid_entities = True - - return has_invalid_entities - - -def annotate_invalid_sex(statement: Dict) -> bool: - if statement[SEX]: - if len(statement[SEX]) > 1: - logger_service.add_anomaly( - LoggableAnomaly(statement[ID], None, f'Multiple sexes found in statement.')) - - first_sex_uri = statement[SEX][0] - if not Sex.objects.filter(ontology_uri=first_sex_uri).exists(): - statement[VALIDATION_ERRORS].sex.add(first_sex_uri) - return True - return False - - -def annotate_invalid_species(statement: Dict) -> bool: - has_invalid_species = False - for species_uri in statement[SPECIES]: - if not Specie.objects.filter(ontology_uri=species_uri).exists(): - statement[VALIDATION_ERRORS].species.add(species_uri) - has_invalid_species = True - return has_invalid_species - - -def annotate_invalid_forward_connections(statement: Dict, statement_ids: Set[str]) -> bool: - has_invalid_forward_connection = False - for reference_uri in statement[FORWARD_CONNECTION]: - if reference_uri not in statement_ids: - statement[VALIDATION_ERRORS].forward_connection.add(reference_uri) - has_invalid_forward_connection = True - return has_invalid_forward_connection - - -def get_or_create_sentence(statement: Dict) -> Tuple[Sentence, bool]: - text = f'{statement[LABEL]} created from neurondm on {NOW}' - has_sentence_reference = len(statement[SENTENCE_NUMBER]) > 0 - - if len(statement[SENTENCE_NUMBER]) > 1: - logger_service.add_anomaly( - LoggableAnomaly(statement[ID], None, f'Multiple sentence numbers found.')) - - sentence, created = Sentence.objects.get_or_create( - doi__iexact=statement[ID], - defaults={"title": text[0:185], - "text": text, - "doi": statement[ID], - "external_ref": statement[SENTENCE_NUMBER][0] if has_sentence_reference else None, - "batch_name": f"neurondm-{NOW}" if has_sentence_reference else None, - "state": SentenceState.COMPOSE_NOW - }, - ) - if created: - logging.info(f"Sentence for neuron {statement[LABEL]} created.") - - return sentence, created - - -def create_or_update_connectivity_statement(statement: Dict, sentence: Sentence) -> Tuple[ConnectivityStatement, bool]: - reference_uri = statement[ID] - defaults = { - "sentence": sentence, - "knowledge_statement": statement[LABEL], - "sex": get_sex(statement), - "circuit_type": get_circuit_type(statement), - "functional_circuit_role": get_functional_circuit_role(statement), - "phenotype": get_phenotype(statement), - "projection_phenotype": get_projection_phenotype(statement), - "reference_uri": statement[ID], - "state": CSState.EXPORTED, - } - - connectivity_statement, created = ConnectivityStatement.objects.get_or_create( - reference_uri=reference_uri, - defaults=defaults - ) - if not created: - if has_changes(connectivity_statement, statement, defaults): - ConnectivityStatement.objects.filter(reference_uri=reference_uri).update(**defaults) - fields_to_refresh = [field for field in defaults.keys() if field != 'state'] - connectivity_statement.refresh_from_db(fields=fields_to_refresh) - add_ingestion_system_note(connectivity_statement) - - validation_errors = statement.get(VALIDATION_ERRORS, ValidationErrors()) - - if validation_errors.has_errors(): - error_message = validation_errors.to_string() - if connectivity_statement.state != CSState.INVALID: - do_transition_to_invalid_with_note(connectivity_statement, error_message) - else: - create_invalid_note(connectivity_statement, error_message) - - update_many_to_many_fields(connectivity_statement, statement) - statement[STATE] = connectivity_statement.state - - return connectivity_statement, created - - -def has_changes(connectivity_statement, statement, defaults): - validation_errors = statement.get(VALIDATION_ERRORS, ValidationErrors()) - - for field, value in defaults.items(): - if field == 'state': - continue - - if field in ['sex', 'functional_circuit_role', 'phenotype', 'projection_phenotype']: - current_fk_id = getattr(connectivity_statement, f'{field}_id') - new_fk_id = value.id if value is not None else None - if current_fk_id != new_fk_id: - return True - else: - # For simple fields, directly compare the values - if getattr(connectivity_statement, field) != value: - return True - - # Check for changes in species - current_species = set(species.ontology_uri for species in connectivity_statement.species.all()) - new_species = set(uri for uri in statement.get(SPECIES, []) if uri not in validation_errors.species) - if current_species != new_species: - return True - - # Check for changes in provenance - current_provenance = set(provenance.uri for provenance in connectivity_statement.provenance_set.all()) - new_provenance = set(statement.get(PROVENANCE) or [statement[ID]]) - if current_provenance != new_provenance: - return True - - # Check for changes in forward_connection - current_forward_connections = set( - connection.reference_uri for connection in connectivity_statement.forward_connection.all()) - new_forward_connections = set( - uri for uri in statement.get(FORWARD_CONNECTION, []) if uri not in validation_errors.forward_connection) - if current_forward_connections != new_forward_connections: - return True - - # Check for changes in origins - current_origins = set(origin.ontology_uri for origin in connectivity_statement.origins.all()) - new_origins = set(uri for uri in statement[ORIGINS].anatomical_entities if uri not in validation_errors.entities) - if current_origins != new_origins: - return True - - # Check for changes in vias - current_vias = [ - { - 'anatomical_entities': set(via.anatomical_entities.all().values_list('ontology_uri', flat=True)), - 'from_entities': set(via.from_entities.all().values_list('ontology_uri', flat=True)) - } - for via in connectivity_statement.via_set.order_by('order').all() - ] - new_vias = statement[VIAS] - - if len(current_vias) != len(new_vias): - return True - - for current_via, new_via in zip(current_vias, new_vias): - new_via_anatomical_entities = set( - uri for uri in new_via.anatomical_entities if uri not in validation_errors.entities) - - new_via_from_entities = set(uri for uri in new_via.from_entities if uri not in validation_errors.entities) - - if (new_via_anatomical_entities != current_via['anatomical_entities'] or - new_via_from_entities != current_via['from_entities']): - return True - - # Check for changes in destinations - current_destinations = connectivity_statement.destinations.all() - new_destinations = statement[DESTINATIONS] - - if len(current_destinations) != len(new_destinations): - return True - - # We may need to change this algorithm when multi-destination is supported by neurondm - - current_destinations_anatomical_entities = set( - uri for destination in current_destinations - for uri in destination.anatomical_entities.all().values_list('ontology_uri', flat=True) - ) - current_destinations_from_entities = set( - uri for destination in current_destinations - for uri in destination.from_entities.all().values_list('ontology_uri', flat=True) - ) - - new_destinations_anatomical_entities = {uri for new_dest in statement[DESTINATIONS] for uri in - new_dest.anatomical_entities if uri not in validation_errors.entities} - - new_destinations_from_entities = {uri for new_dest in statement[DESTINATIONS] for uri in new_dest.from_entities if - uri not in validation_errors.entities} - - if (current_destinations_anatomical_entities != new_destinations_anatomical_entities or - current_destinations_from_entities != new_destinations_from_entities): - return True - - # Not checking the Notes because they are kept - - return False - - -def get_sex(statement: Dict) -> Sex: - return get_value_or_none(Sex, statement[SEX][0] if statement[SEX] else None) - - -def get_functional_circuit_role(statement: Dict) -> Optional[FunctionalCircuitRole]: - if len(statement[FUNCTIONAL_CIRCUIT_ROLE]) > 1: - logger_service.add_anomaly( - LoggableAnomaly(statement[ID], None, f'Multiple functional circuit roles found.')) - - return get_value_or_none( - FunctionalCircuitRole, statement[FUNCTIONAL_CIRCUIT_ROLE][0]) if statement[FUNCTIONAL_CIRCUIT_ROLE] else None - - -def get_circuit_type(statement: Dict): - if statement[CIRCUIT_TYPE]: - if len(statement[CIRCUIT_TYPE]) > 1: - logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'Multiple circuit types found')) - return CIRCUIT_TYPE_MAPPING.get(statement[CIRCUIT_TYPE][0], None) - else: - logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'No circuit type found.')) - return None - - -def get_phenotype(statement: Dict) -> Optional[Phenotype]: - if statement[PHENOTYPE]: - if len(statement[PHENOTYPE]) > 1: - logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'Multiple phenotypes found.')) - - for p in statement[PHENOTYPE]: - try: - phenotype = Phenotype.objects.get(ontology_uri=p) - return phenotype - except Phenotype.DoesNotExist: - pass - - logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'No valid phenotype found.')) - - return None - - -def get_projection_phenotype(statement: Dict) -> Optional[ProjectionPhenotype]: - if statement[OTHER_PHENOTYPE]: - last_phenotype_uri = statement[OTHER_PHENOTYPE][-1] - try: - projection_phenotype = ProjectionPhenotype.objects.get(ontology_uri=last_phenotype_uri) - return projection_phenotype - except ProjectionPhenotype.DoesNotExist: - pass - else: - logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'No projection phenotypes found.')) - return None - - -def do_transition_to_invalid_with_note(connectivity_statement: ConnectivityStatement, note: str): - system_user = User.objects.get(username="system") - connectivity_statement.invalid(by=system_user) - connectivity_statement.save() - - create_invalid_note(connectivity_statement, note) - - -def create_invalid_note(connectivity_statement: ConnectivityStatement, note: str): - Note.objects.create( - connectivity_statement=connectivity_statement, - user=User.objects.get(username="system"), - type=NoteType.ALERT, - note=f"Invalidated due to the following reason(s): {note}" - ) - - -def update_many_to_many_fields(connectivity_statement: ConnectivityStatement, statement: Dict): - connectivity_statement.origins.clear() - connectivity_statement.species.clear() - # Notes are not cleared because they should be kept - - for provenance in connectivity_statement.provenance_set.all(): - provenance.delete() - - for destination in connectivity_statement.destinations.all(): - destination.delete() - - for via in connectivity_statement.via_set.all(): - via.delete() - - add_origins(connectivity_statement, statement) - add_vias(connectivity_statement, statement) - add_destinations(connectivity_statement, statement) - add_species(connectivity_statement, statement) - add_provenances(connectivity_statement, statement) - add_notes(connectivity_statement, statement) - - -def add_origins(connectivity_statement: ConnectivityStatement, statement: Dict): - origin_uris = statement[ORIGINS].anatomical_entities - origins = [] - for uri in origin_uris: - anatomical_entity = AnatomicalEntity.objects.filter(ontology_uri=uri).first() - if anatomical_entity: - origins.append(anatomical_entity) - else: - assert connectivity_statement.state == CSState.INVALID - - if origins: - connectivity_statement.origins.add(*origins) - - -def add_vias(connectivity_statement: ConnectivityStatement, statement: Dict): - vias_data = [ - Via(connectivity_statement=connectivity_statement, type=via.type, order=via.order) - for via in statement[VIAS] - ] - created_vias = Via.objects.bulk_create(vias_data) - - for via_instance, via_data in zip(created_vias, statement[VIAS]): - for uri in via_data.anatomical_entities: - anatomical_entity = AnatomicalEntity.objects.filter(ontology_uri=uri).first() - if anatomical_entity: - via_instance.anatomical_entities.add(anatomical_entity) - else: - assert connectivity_statement.state == CSState.INVALID - - for uri in via_data.from_entities: - from_entity = AnatomicalEntity.objects.filter(ontology_uri=uri).first() - if from_entity: - via_instance.from_entities.add(from_entity) - else: - assert connectivity_statement.state == CSState.INVALID - - -def add_destinations(connectivity_statement: ConnectivityStatement, statement: Dict): - destinations_data = [ - Destination(connectivity_statement=connectivity_statement, type=dest.type) - for dest in statement[DESTINATIONS] - ] - created_destinations = Destination.objects.bulk_create(destinations_data) - - for destination_instance, dest_data in zip(created_destinations, statement[DESTINATIONS]): - for uri in dest_data.anatomical_entities: - anatomical_entity = AnatomicalEntity.objects.filter(ontology_uri=uri).first() - if anatomical_entity: - destination_instance.anatomical_entities.add(anatomical_entity) - else: - assert connectivity_statement.state == CSState.INVALID - - for uri in dest_data.from_entities: - from_entity = AnatomicalEntity.objects.filter(ontology_uri=uri).first() - if from_entity: - destination_instance.from_entities.add(from_entity) - else: - assert connectivity_statement.state == CSState.INVALID - - -def add_notes(connectivity_statement: ConnectivityStatement, statement: Dict): - for note in statement[NOTE_ALERT]: - Note.objects.create(connectivity_statement=connectivity_statement, - user=User.objects.get(username="system"), - type=NoteType.ALERT, - note=note) - - -def add_provenances(connectivity_statement: ConnectivityStatement, statement: Dict): - provenances_list = statement[PROVENANCE] if statement[PROVENANCE] else [statement[ID]] - provenances = (Provenance(connectivity_statement=connectivity_statement, uri=provenance) for provenance in - provenances_list) - Provenance.objects.bulk_create(provenances) - - -def add_species(connectivity_statement: ConnectivityStatement, statement: Dict): - species = Specie.objects.filter(ontology_uri__in=statement[SPECIES]) - connectivity_statement.species.add(*species) - - -def add_ingestion_system_note(connectivity_statement: ConnectivityStatement): - Note.objects.create(connectivity_statement=connectivity_statement, - user=User.objects.get(username="system"), - type=NoteType.ALERT, - note=f"Overwritten by manual ingestion") - - -def update_forward_connections(statements: List): - for statement in statements: - connectivity_statement = ConnectivityStatement.objects.get(reference_uri=statement[ID]) - connectivity_statement.forward_connection.clear() - for uri in statement[FORWARD_CONNECTION]: - try: - forward_statement = ConnectivityStatement.objects.get(reference_uri=uri) - except ConnectivityStatement.DoesNotExist: - assert connectivity_statement.state == CSState.INVALID - continue - connectivity_statement.forward_connection.add(forward_statement) - - -def update_upstream_statements(): - invalid_visited = set() - connectivity_statements_invalid_reasons = {} - - initial_invalid_statements = ConnectivityStatement.objects.filter(state=CSState.INVALID) - - for statement in initial_invalid_statements: - propagate_invalid_state(statement, invalid_visited, connectivity_statements_invalid_reasons) - - for statement_uri, (connectivity_statement, reasons) in connectivity_statements_invalid_reasons.items(): - all_reasons = '; '.join(reasons) - - # Perform transition and create a note only if not already invalid - if connectivity_statement.state != CSState.INVALID: - do_transition_to_invalid_with_note(connectivity_statement, all_reasons) - else: - create_invalid_note(connectivity_statement, all_reasons) - - -def propagate_invalid_state(connectivity_statement: ConnectivityStatement, invalid_visited: Set, - connectivity_statements_invalid_reasons: Dict, previous_reason: str = ''): - statement_uri = connectivity_statement.reference_uri - - if statement_uri in invalid_visited: - return - - invalid_visited.add(statement_uri) - - # Fetch backward connections directly from the database - backward_connections = ConnectivityStatement.objects.filter( - forward_connection=connectivity_statement - ) - - for backward_cs in backward_connections: - # Build the reason string - current_reason = (f"statement with id {backward_cs.id} is invalid because its " - f"forward connection with id {connectivity_statement.id} is invalid") - if previous_reason: - current_reason += f" because {previous_reason}" - - # Accumulate reasons in connectivity_statements_invalid_reasons, store ConnectivityStatement object with reasons - if backward_cs.reference_uri not in connectivity_statements_invalid_reasons: - connectivity_statements_invalid_reasons[backward_cs.reference_uri] = (backward_cs, []) - connectivity_statements_invalid_reasons[backward_cs.reference_uri][1].append(current_reason) - - # Recursively propagate invalid state - propagate_invalid_state(backward_cs, invalid_visited, connectivity_statements_invalid_reasons, current_reason) diff --git a/backend/composer/services/cs_ingestion/exceptions.py b/backend/composer/services/cs_ingestion/exceptions.py index 40d875b8..1ae89f89 100644 --- a/backend/composer/services/cs_ingestion/exceptions.py +++ b/backend/composer/services/cs_ingestion/exceptions.py @@ -4,3 +4,7 @@ def __init__(self, statement_id, entity_id, message): self.entity_id = entity_id self.message = message super().__init__(f"StatementID: {statement_id}, EntityID: {entity_id}, Error: {message}") + + +class EntityNotFoundException(Exception): + pass diff --git a/backend/composer/services/cs_ingestion/helpers/__init__.py b/backend/composer/services/cs_ingestion/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py new file mode 100644 index 00000000..81f5f3fa --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/anatomical_entities_helper.py @@ -0,0 +1,133 @@ +from typing import Dict + +from django.db import IntegrityError, transaction +from neurondm import orders + +from composer.enums import CSState +from composer.models import ConnectivityStatement, AnatomicalEntityMeta, Via, Destination, AnatomicalEntity, Layer, \ + Region, AnatomicalEntityIntersection +from composer.services.cs_ingestion.exceptions import EntityNotFoundException +from composer.services.cs_ingestion.helpers.common_helpers import ORIGINS, VIAS, DESTINATIONS + + +def add_origins(connectivity_statement: ConnectivityStatement, statement: Dict, update_anatomic_entities: bool): + for entity in statement[ORIGINS].anatomical_entities: + try: + add_entity_to_instance(connectivity_statement, 'origins', entity, update_anatomic_entities) + except (EntityNotFoundException, AnatomicalEntityMeta.DoesNotExist): + assert connectivity_statement.state == CSState.INVALID, f"connectivity_statement {connectivity_statement} should be invalid due to entity {entity} not found but it isn't" + except IntegrityError as e: + raise e + + +def add_vias(connectivity_statement: ConnectivityStatement, statement: Dict, update_anatomic_entities: bool): + for neurondm_via in statement[VIAS]: + via_instance = Via.objects.create(connectivity_statement=connectivity_statement, + type=neurondm_via.type, + order=neurondm_via.order) + add_entities_to_connection(via_instance, + neurondm_via.anatomical_entities, + neurondm_via.from_entities, + connectivity_statement, update_anatomic_entities) + + +def add_destinations(connectivity_statement: ConnectivityStatement, statement: Dict, update_anatomic_entities: bool): + for neurondm_destination in statement[DESTINATIONS]: + destination_instance = Destination.objects.create(connectivity_statement=connectivity_statement, + type=neurondm_destination.type) + add_entities_to_connection(destination_instance, + neurondm_destination.anatomical_entities, + neurondm_destination.from_entities, + connectivity_statement, update_anatomic_entities) + + +def add_entities_to_connection(instance, anatomical_entities, from_entities, connectivity_statement, + update_anatomic_entities: bool): + try: + for entity in anatomical_entities: + add_entity_to_instance(instance, 'anatomical_entities', entity, update_anatomic_entities) + + for entity in from_entities: + add_entity_to_instance(instance, 'from_entities', entity, update_anatomic_entities) + + except (EntityNotFoundException, AnatomicalEntity.DoesNotExist): + assert connectivity_statement.state == CSState.INVALID, \ + f"connectivity_statement {connectivity_statement} should be invalid due to entity {entity} not found but it isn't" + except IntegrityError as e: + raise e + + +def add_entity_to_instance(instance, entity_field, entity, update_anatomic_entities: bool): + if isinstance(entity, orders.rl): + complex_anatomical_entity, _ = get_or_create_complex_entity(str(entity.region), str(entity.layer), + update_anatomic_entities) + getattr(instance, entity_field).add(complex_anatomical_entity) + else: + anatomical_entity, _ = get_or_create_simple_entity(str(entity)) + getattr(instance, entity_field).add(anatomical_entity) + + +def get_or_create_complex_entity(region_uri, layer_uri, update_anatomical_entities=False): + try: + layer = Layer.objects.get(ontology_uri=layer_uri) + except Layer.DoesNotExist: + layer = None + + try: + region = Region.objects.get(ontology_uri=region_uri) + except Region.DoesNotExist: + region = None + + if update_anatomical_entities: + if not layer: + try: + layer_meta = AnatomicalEntityMeta.objects.get(ontology_uri=layer_uri) + layer, _ = convert_anatomical_entity_to_specific_type(layer_meta, Layer) + except AnatomicalEntityMeta.DoesNotExist: + raise EntityNotFoundException(f"Layer meta not found for URI: {layer_uri}") + + if not region: + try: + region_meta = AnatomicalEntityMeta.objects.get(ontology_uri=region_uri) + region, _ = convert_anatomical_entity_to_specific_type(region_meta, Region) + except AnatomicalEntityMeta.DoesNotExist: + raise EntityNotFoundException(f"Region meta not found for URI: {layer_uri}") + else: + if not layer or not region: + raise EntityNotFoundException("Required Layer or Region not found.") + + intersection, _ = AnatomicalEntityIntersection.objects.get_or_create(layer=layer, region=region) + anatomical_entity, created = AnatomicalEntity.objects.get_or_create(region_layer=intersection) + + return anatomical_entity, created + + +def get_or_create_simple_entity(ontology_uri: str): + try: + anatomical_entity_meta = AnatomicalEntityMeta.objects.get(ontology_uri=ontology_uri) + anatomical_entity, created = AnatomicalEntity.objects.get_or_create(simple_entity=anatomical_entity_meta) + return anatomical_entity, created + except AnatomicalEntityMeta.DoesNotExist: + raise EntityNotFoundException(f"Anatomical entity meta not found for URI: {ontology_uri}") + + +def convert_anatomical_entity_to_specific_type(entity_meta, target_model): + """ + Convert an AnatomicalEntityMeta instance to a more specific subclass type (Layer or Region). + Attempts to delete the original instance and create the new specific instance atomically. + """ + defaults = {'name': entity_meta.name, 'ontology_uri': entity_meta.ontology_uri} + + try: + with transaction.atomic(): + # Delete the anatomical entity meta in the incorrect type + entity_meta.delete() + # Create a new anatomical entity in the new specific type + specific_entity, created = target_model.objects.get_or_create( + ontology_uri=entity_meta.ontology_uri, + defaults=defaults + ) + return specific_entity, created + except IntegrityError as e: + raise IntegrityError( + f"Failed to convert AnatomicalEntityMeta to {target_model.__name__} due to integrity error: {e}") diff --git a/backend/composer/services/cs_ingestion/helpers/changes_detector.py b/backend/composer/services/cs_ingestion/helpers/changes_detector.py new file mode 100644 index 00000000..301d5eae --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/changes_detector.py @@ -0,0 +1,127 @@ +import rdflib +from neurondm import orders + +from composer.models import AnatomicalEntity +from composer.services.cs_ingestion.helpers.common_helpers import VALIDATION_ERRORS, SPECIES, PROVENANCE, ID, \ + FORWARD_CONNECTION, ORIGINS, VIAS, DESTINATIONS +from composer.services.cs_ingestion.models import ValidationErrors + + +def has_changes(connectivity_statement, statement, defaults): + validation_errors = statement.get(VALIDATION_ERRORS, ValidationErrors()) + + for field, value in defaults.items(): + if field == 'state': + continue + + if field in ['sex', 'functional_circuit_role', 'phenotype', 'projection_phenotype']: + current_fk_id = getattr(connectivity_statement, f'{field}_id') + new_fk_id = value.id if value is not None else None + if current_fk_id != new_fk_id: + return True + else: + # For simple fields, directly compare the values + if getattr(connectivity_statement, field) != value: + return True + + # Check for changes in species + current_species = set(species.ontology_uri for species in connectivity_statement.species.all()) + new_species = set(uri for uri in statement.get(SPECIES, []) if uri not in validation_errors.species) + if current_species != new_species: + return True + + # Check for changes in provenance + current_provenance = set(provenance.uri for provenance in connectivity_statement.provenance_set.all()) + new_provenance = set(statement.get(PROVENANCE) or [statement[ID]]) + if current_provenance != new_provenance: + return True + + # Check for changes in forward_connection + current_forward_connections = set( + connection.reference_uri for connection in connectivity_statement.forward_connection.all()) + new_forward_connections = set( + uri for uri in statement.get(FORWARD_CONNECTION, []) if uri not in validation_errors.forward_connection) + if current_forward_connections != new_forward_connections: + return True + + # Check for changes in origins + current_origins = set( + get_anatomical_entity_identifier_composer(origin) for origin in connectivity_statement.origins.all()) + new_origins = set( + get_anatomical_entity_identifier_neurondm(uri) for uri in statement[ORIGINS].anatomical_entities if + uri not in validation_errors.entities) + if current_origins != new_origins: + return True + + # Check for changes in vias + current_vias = [ + { + 'anatomical_entities': set( + get_anatomical_entity_identifier_composer(ae) for ae in via.anatomical_entities.all()), + 'from_entities': set(get_anatomical_entity_identifier_composer(ae) for ae in via.from_entities.all()) + } + for via in connectivity_statement.via_set.order_by('order').all() + ] + new_vias = statement[VIAS] + + if len(current_vias) != len(new_vias): + return True + + for current_via, new_via in zip(current_vias, new_vias): + new_via_anatomical_entities = set( + get_anatomical_entity_identifier_neurondm(uri) for uri in new_via.anatomical_entities if + uri not in validation_errors.entities) + + new_via_from_entities = set(get_anatomical_entity_identifier_neurondm(uri) for uri in new_via.from_entities if + uri not in validation_errors.entities) + + if (new_via_anatomical_entities != current_via['anatomical_entities'] or + new_via_from_entities != current_via['from_entities']): + return True + + # Check for changes in destinations + current_destinations = connectivity_statement.destinations.all() + new_destinations = statement[DESTINATIONS] + + if len(current_destinations) != len(new_destinations): + return True + + # We may need to change this algorithm when multi-destination is supported by neurondm + + current_destinations_anatomical_entities = set( + get_anatomical_entity_identifier_composer(uri) for destination in current_destinations + for uri in destination.anatomical_entities.all() + ) + current_destinations_from_entities = set( + get_anatomical_entity_identifier_composer(uri) for destination in current_destinations + for uri in destination.from_entities.all() + ) + + new_destinations_anatomical_entities = {get_anatomical_entity_identifier_neurondm(uri) for new_dest in + statement[DESTINATIONS] for uri in + new_dest.anatomical_entities if uri not in validation_errors.entities} + + new_destinations_from_entities = {get_anatomical_entity_identifier_neurondm(uri) for new_dest in + statement[DESTINATIONS] for uri in new_dest.from_entities if + uri not in validation_errors.entities} + + if (current_destinations_anatomical_entities != new_destinations_anatomical_entities or + current_destinations_from_entities != new_destinations_from_entities): + return True + + # Not checking the Notes because they are kept + return False + + +def get_anatomical_entity_identifier_composer(entity: AnatomicalEntity): + if entity.region_layer: + layer_uri = entity.region_layer.layer.ontology_uri + region_uri = entity.region_layer.region.ontology_uri + return f"{region_uri}:{layer_uri}" + return entity.simple_entity.ontology_uri + + +def get_anatomical_entity_identifier_neurondm(entity: rdflib.term): + if isinstance(entity, orders.rl): + return f"{str(entity.region)}:{str(entity.layer)}" + return str(entity) diff --git a/backend/composer/services/cs_ingestion/helpers.py b/backend/composer/services/cs_ingestion/helpers/common_helpers.py similarity index 89% rename from backend/composer/services/cs_ingestion/helpers.py rename to backend/composer/services/cs_ingestion/helpers/common_helpers.py index 7016e6ad..16eba1b0 100644 --- a/backend/composer/services/cs_ingestion/helpers.py +++ b/backend/composer/services/cs_ingestion/helpers/common_helpers.py @@ -1,7 +1,7 @@ import logging from composer.enums import CircuitType -from composer.models import AnatomicalEntity +from composer.models import AnatomicalEntityMeta ID = "id" ORIGINS = "origins" @@ -41,7 +41,3 @@ def get_value_or_none(model, prop: str): return None else: return None - - -def found_entity(uri: str) -> bool: - return AnatomicalEntity.objects.filter(ontology_uri=uri).exists() diff --git a/backend/composer/services/cs_ingestion/helpers/getters.py b/backend/composer/services/cs_ingestion/helpers/getters.py new file mode 100644 index 00000000..e0c3f9d8 --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/getters.py @@ -0,0 +1,62 @@ +from typing import Dict, Optional + +from composer.models import Sex, FunctionalCircuitRole, Phenotype, ProjectionPhenotype +from composer.services.cs_ingestion.helpers.common_helpers import get_value_or_none, SEX, FUNCTIONAL_CIRCUIT_ROLE, ID, \ + CIRCUIT_TYPE, CIRCUIT_TYPE_MAPPING, PHENOTYPE, OTHER_PHENOTYPE +from composer.services.cs_ingestion.logging_service import LoggerService +from composer.services.cs_ingestion.models import LoggableAnomaly + +logger_service = LoggerService() + + +def get_sex(statement: Dict) -> Sex: + return get_value_or_none(Sex, statement[SEX][0] if statement[SEX] else None) + + +def get_functional_circuit_role(statement: Dict) -> Optional[FunctionalCircuitRole]: + if len(statement[FUNCTIONAL_CIRCUIT_ROLE]) > 1: + logger_service.add_anomaly( + LoggableAnomaly(statement[ID], None, f'Multiple functional circuit roles found.')) + + return get_value_or_none( + FunctionalCircuitRole, statement[FUNCTIONAL_CIRCUIT_ROLE][0]) if statement[FUNCTIONAL_CIRCUIT_ROLE] else None + + +def get_circuit_type(statement: Dict): + if statement[CIRCUIT_TYPE]: + if len(statement[CIRCUIT_TYPE]) > 1: + logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'Multiple circuit types found')) + return CIRCUIT_TYPE_MAPPING.get(statement[CIRCUIT_TYPE][0], None) + else: + logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'No circuit type found.')) + return None + + +def get_phenotype(statement: Dict) -> Optional[Phenotype]: + if statement[PHENOTYPE]: + if len(statement[PHENOTYPE]) > 1: + logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'Multiple phenotypes found.')) + + for p in statement[PHENOTYPE]: + try: + phenotype = Phenotype.objects.get(ontology_uri=p) + return phenotype + except Phenotype.DoesNotExist: + pass + + logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'No valid phenotype found.')) + + return None + + +def get_projection_phenotype(statement: Dict) -> Optional[ProjectionPhenotype]: + if statement[OTHER_PHENOTYPE]: + last_phenotype_uri = statement[OTHER_PHENOTYPE][-1] + try: + projection_phenotype = ProjectionPhenotype.objects.get(ontology_uri=last_phenotype_uri) + return projection_phenotype + except ProjectionPhenotype.DoesNotExist: + pass + else: + logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, f'No projection phenotypes found.')) + return None diff --git a/backend/composer/services/cs_ingestion/helpers/notes_helper.py b/backend/composer/services/cs_ingestion/helpers/notes_helper.py new file mode 100644 index 00000000..f89c2249 --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/notes_helper.py @@ -0,0 +1,28 @@ +from django.contrib.auth.models import User + +from composer.enums import NoteType +from composer.models import ConnectivityStatement, Note + + +def add_ingestion_system_note(connectivity_statement: ConnectivityStatement): + Note.objects.create(connectivity_statement=connectivity_statement, + user=User.objects.get(username="system"), + type=NoteType.ALERT, + note=f"Overwritten by manual ingestion") + + +def do_transition_to_invalid_with_note(connectivity_statement: ConnectivityStatement, note: str): + system_user = User.objects.get(username="system") + connectivity_statement.invalid(by=system_user) + connectivity_statement.save() + + create_invalid_note(connectivity_statement, note) + + +def create_invalid_note(connectivity_statement: ConnectivityStatement, note: str): + Note.objects.create( + connectivity_statement=connectivity_statement, + user=User.objects.get(username="system"), + type=NoteType.ALERT, + note=f"Invalidated due to the following reason(s): {note}" + ) diff --git a/backend/composer/services/cs_ingestion/helpers/overwritable_helper.py b/backend/composer/services/cs_ingestion/helpers/overwritable_helper.py new file mode 100644 index 00000000..c21ad928 --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/overwritable_helper.py @@ -0,0 +1,49 @@ +from typing import List, Any, Dict + +from composer.enums import CSState, SentenceState +from composer.models import Sentence, ConnectivityStatement +from composer.services.cs_ingestion.helpers.common_helpers import ID +from composer.services.cs_ingestion.logging_service import STATEMENT_INCORRECT_STATE, SENTENCE_INCORRECT_STATE, \ + LoggerService +from composer.services.cs_ingestion.models import LoggableAnomaly + +logger_service = LoggerService() + + +def get_overwritable_statements(statements_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + overwritable_statements = [ + statement for statement in statements_list + if not has_invalid_sentence(statement) and not has_invalid_statement(statement) + ] + return overwritable_statements + + +def has_invalid_sentence(statement: Dict) -> bool: + try: + sentence = Sentence.objects.get(doi__iexact=statement[ID]) + except Sentence.DoesNotExist: + return False + return not can_sentence_be_overwritten(sentence, statement) + + +def has_invalid_statement(statement: Dict) -> bool: + try: + connectivity_statement = ConnectivityStatement.objects.get(reference_uri=statement[ID]) + except ConnectivityStatement.DoesNotExist: + return False + return not can_statement_be_overwritten(connectivity_statement, statement) + + +def can_statement_be_overwritten(connectivity_statement: ConnectivityStatement, statement) -> bool: + if connectivity_statement.state != CSState.EXPORTED and connectivity_statement.state != CSState.INVALID: + logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, STATEMENT_INCORRECT_STATE)) + return False + + return True + + +def can_sentence_be_overwritten(sentence: Sentence, statement: Dict) -> bool: + if sentence.state != SentenceState.COMPOSE_NOW: + logger_service.add_anomaly(LoggableAnomaly(statement[ID], None, SENTENCE_INCORRECT_STATE)) + return False + return True diff --git a/backend/composer/services/cs_ingestion/helpers/sentence_helper.py b/backend/composer/services/cs_ingestion/helpers/sentence_helper.py new file mode 100644 index 00000000..80cf324d --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/sentence_helper.py @@ -0,0 +1,36 @@ +import logging +from datetime import datetime +from typing import Dict, Tuple + +from composer.enums import SentenceState +from composer.models import Sentence +from composer.services.cs_ingestion.helpers.common_helpers import SENTENCE_NUMBER, LABEL, ID +from composer.services.cs_ingestion.logging_service import LoggerService +from composer.services.cs_ingestion.models import LoggableAnomaly + +NOW = datetime.now().strftime("%Y%m%d%H%M%S") +logger_service = LoggerService() + + +def get_or_create_sentence(statement: Dict) -> Tuple[Sentence, bool]: + text = f'{statement[LABEL]} created from neurondm on {NOW}' + has_sentence_reference = len(statement[SENTENCE_NUMBER]) > 0 + + if len(statement[SENTENCE_NUMBER]) > 1: + logger_service.add_anomaly( + LoggableAnomaly(statement[ID], None, f'Multiple sentence numbers found.')) + + sentence, created = Sentence.objects.get_or_create( + doi__iexact=statement[ID], + defaults={"title": text[0:185], + "text": text, + "doi": statement[ID], + "external_ref": statement[SENTENCE_NUMBER][0] if has_sentence_reference else None, + "batch_name": f"neurondm-{NOW}" if has_sentence_reference else None, + "state": SentenceState.COMPOSE_NOW + }, + ) + if created: + logging.info(f"Sentence for neuron {statement[LABEL]} created.") + + return sentence, created diff --git a/backend/composer/services/cs_ingestion/helpers/statement_helper.py b/backend/composer/services/cs_ingestion/helpers/statement_helper.py new file mode 100644 index 00000000..fafd734f --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/statement_helper.py @@ -0,0 +1,114 @@ +from typing import Dict, Tuple, List + +from django.contrib.auth.models import User + +from composer.enums import CSState, NoteType +from composer.management.commands.ingest_nlp_sentence import ID +from composer.models import Sentence, ConnectivityStatement, Note, Specie, Provenance +from composer.services.cs_ingestion.helpers.anatomical_entities_helper import add_origins, add_vias, add_destinations +from composer.services.cs_ingestion.helpers.changes_detector import has_changes +from composer.services.cs_ingestion.helpers.common_helpers import LABEL, VALIDATION_ERRORS, STATE, NOTE_ALERT, \ + PROVENANCE, SPECIES, FORWARD_CONNECTION +from composer.services.cs_ingestion.helpers.getters import get_sex, get_circuit_type, get_functional_circuit_role, \ + get_phenotype, get_projection_phenotype +from composer.services.cs_ingestion.helpers.notes_helper import do_transition_to_invalid_with_note, create_invalid_note, \ + add_ingestion_system_note +from composer.services.cs_ingestion.models import ValidationErrors + + +def create_or_update_connectivity_statement(statement: Dict, sentence: Sentence, update_anatomical_entities: bool) -> \ + Tuple[ConnectivityStatement, bool]: + reference_uri = statement[ID] + defaults = { + "sentence": sentence, + "knowledge_statement": statement[LABEL], + "sex": get_sex(statement), + "circuit_type": get_circuit_type(statement), + "functional_circuit_role": get_functional_circuit_role(statement), + "phenotype": get_phenotype(statement), + "projection_phenotype": get_projection_phenotype(statement), + "reference_uri": statement[ID], + "state": CSState.EXPORTED, + } + + connectivity_statement, created = ConnectivityStatement.objects.get_or_create( + reference_uri=reference_uri, + defaults=defaults + ) + if not created: + if has_changes(connectivity_statement, statement, defaults): + ConnectivityStatement.objects.filter(reference_uri=reference_uri).update(**defaults) + fields_to_refresh = [field for field in defaults.keys() if field != 'state'] + connectivity_statement.refresh_from_db(fields=fields_to_refresh) + add_ingestion_system_note(connectivity_statement) + + validation_errors = statement.get(VALIDATION_ERRORS, ValidationErrors()) + + if validation_errors.has_errors(): + error_message = validation_errors.to_string() + if connectivity_statement.state != CSState.INVALID: + do_transition_to_invalid_with_note(connectivity_statement, error_message) + else: + create_invalid_note(connectivity_statement, error_message) + + update_many_to_many_fields(connectivity_statement, statement, update_anatomical_entities) + statement[STATE] = connectivity_statement.state + + return connectivity_statement, created + + +def update_many_to_many_fields(connectivity_statement: ConnectivityStatement, statement: Dict, + update_anatomical_entities: bool): + connectivity_statement.origins.clear() + connectivity_statement.species.clear() + # Notes are not cleared because they should be kept + + for provenance in connectivity_statement.provenance_set.all(): + provenance.delete() + + for destination in connectivity_statement.destinations.all(): + destination.delete() + + for via in connectivity_statement.via_set.all(): + via.delete() + + add_origins(connectivity_statement, statement, update_anatomical_entities) + add_vias(connectivity_statement, statement, update_anatomical_entities) + add_destinations(connectivity_statement, statement, update_anatomical_entities) + add_species(connectivity_statement, statement) + add_provenances(connectivity_statement, statement) + add_notes(connectivity_statement, statement) + + +def add_notes(connectivity_statement: ConnectivityStatement, statement: Dict): + for note in statement[NOTE_ALERT]: + Note.objects.create(connectivity_statement=connectivity_statement, + user=User.objects.get(username="system"), + type=NoteType.ALERT, + note=note) + + +def add_provenances(connectivity_statement: ConnectivityStatement, statement: Dict): + provenances_list = statement[PROVENANCE] if statement[PROVENANCE] else [statement[ID]] + provenances = (Provenance(connectivity_statement=connectivity_statement, uri=provenance) for provenance in + provenances_list) + Provenance.objects.bulk_create(provenances) + + +def add_species(connectivity_statement: ConnectivityStatement, statement: Dict): + species = Specie.objects.filter(ontology_uri__in=statement[SPECIES]) + connectivity_statement.species.add(*species) + + +def update_forward_connections(statements: List): + for statement in statements: + connectivity_statement = ConnectivityStatement.objects.get(reference_uri=statement[ID]) + connectivity_statement.forward_connection.clear() + for uri in statement[FORWARD_CONNECTION]: + try: + forward_statement = ConnectivityStatement.objects.get(reference_uri=uri) + except ConnectivityStatement.DoesNotExist: + assert statement[STATE] == CSState.INVALID, \ + f"connectivity_statement {connectivity_statement} should be invalid due to forward connection {uri} not found but it isn't" + continue + connectivity_statement.forward_connection.add(forward_statement) diff --git a/backend/composer/services/cs_ingestion/helpers/upstream_changes_helper.py b/backend/composer/services/cs_ingestion/helpers/upstream_changes_helper.py new file mode 100644 index 00000000..fdf82774 --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/upstream_changes_helper.py @@ -0,0 +1,54 @@ +from typing import Set, Dict + +from composer.enums import CSState +from composer.models import ConnectivityStatement +from composer.services.cs_ingestion.helpers.notes_helper import do_transition_to_invalid_with_note, create_invalid_note + + +def update_upstream_statements(): + invalid_visited = set() + connectivity_statements_invalid_reasons = {} + + initial_invalid_statements = ConnectivityStatement.objects.filter(state=CSState.INVALID) + + for statement in initial_invalid_statements: + propagate_invalid_state(statement, invalid_visited, connectivity_statements_invalid_reasons) + + for statement_uri, (connectivity_statement, reasons) in connectivity_statements_invalid_reasons.items(): + all_reasons = '; '.join(reasons) + + # Perform transition and create a note only if not already invalid + if connectivity_statement.state != CSState.INVALID: + do_transition_to_invalid_with_note(connectivity_statement, all_reasons) + else: + create_invalid_note(connectivity_statement, all_reasons) + + +def propagate_invalid_state(connectivity_statement: ConnectivityStatement, invalid_visited: Set, + connectivity_statements_invalid_reasons: Dict, previous_reason: str = ''): + statement_uri = connectivity_statement.reference_uri + + if statement_uri in invalid_visited: + return + + invalid_visited.add(statement_uri) + + # Fetch backward connections directly from the database + backward_connections = ConnectivityStatement.objects.filter( + forward_connection=connectivity_statement + ) + + for backward_cs in backward_connections: + # Build the reason string + current_reason = (f"statement with id {backward_cs.id} is invalid because its " + f"forward connection with id {connectivity_statement.id} is invalid") + if previous_reason: + current_reason += f" because {previous_reason}" + + # Accumulate reasons in connectivity_statements_invalid_reasons, store ConnectivityStatement object with reasons + if backward_cs.reference_uri not in connectivity_statements_invalid_reasons: + connectivity_statements_invalid_reasons[backward_cs.reference_uri] = (backward_cs, []) + connectivity_statements_invalid_reasons[backward_cs.reference_uri][1].append(current_reason) + + # Recursively propagate invalid state + propagate_invalid_state(backward_cs, invalid_visited, connectivity_statements_invalid_reasons, current_reason) diff --git a/backend/composer/services/cs_ingestion/helpers/validators.py b/backend/composer/services/cs_ingestion/helpers/validators.py new file mode 100644 index 00000000..12312e96 --- /dev/null +++ b/backend/composer/services/cs_ingestion/helpers/validators.py @@ -0,0 +1,96 @@ +from typing import List, Dict, Any, Set, Type + +from neurondm import orders + +from composer.models import ConnectivityStatement, Sex, Specie, Region, AnatomicalEntityMeta, Layer +from composer.services.cs_ingestion.helpers.common_helpers import ID, VALIDATION_ERRORS, ORIGINS, DESTINATIONS, VIAS, \ + SEX, SPECIES, FORWARD_CONNECTION +from composer.services.cs_ingestion.logging_service import LoggerService +from composer.services.cs_ingestion.models import ValidationErrors, LoggableAnomaly +from django.db.models import Model as DjangoModel + +logger_service = LoggerService() + + +def validate_statements(statements: List[Dict[str, Any]], update_anatomical_entities: bool) -> List[Dict[str, Any]]: + db_reference_uris = set(ConnectivityStatement.objects.values_list('reference_uri', flat=True)) + input_statement_ids = {statement[ID] for statement in statements} + statement_ids = input_statement_ids.union(db_reference_uris) + + for statement in statements: + # Initialize validation_errors if not already present + if VALIDATION_ERRORS not in statement: + statement[VALIDATION_ERRORS] = ValidationErrors() + + # Validate entities, sex, and species, updating validation_errors accordingly + annotate_invalid_entities(statement, update_anatomical_entities) + annotate_invalid_sex(statement) + annotate_invalid_species(statement) + + # Validate forward connection + annotate_invalid_forward_connections(statement, statement_ids) + + return statements + + +def annotate_invalid_entities(statement: Dict, update_anatomical_entities: bool) -> bool: + has_invalid_entities = False + + entities_to_check = list(statement[ORIGINS].anatomical_entities) + entities_to_check.extend(entity for dest in statement[DESTINATIONS] for entity in dest.anatomical_entities) + entities_to_check.extend(entity for via in statement[VIAS] for entity in via.anatomical_entities) + + for entity in entities_to_check: + if isinstance(entity, orders.rl): + region_found = found_entity(entity.region, Region if not update_anatomical_entities else None) + layer_found = found_entity(entity.layer, Layer if not update_anatomical_entities else None) + if not region_found: + statement[VALIDATION_ERRORS].entities.add(entity.region) + has_invalid_entities = True + if not layer_found: + statement[VALIDATION_ERRORS].entities.add(entity.layer) + has_invalid_entities = True + else: + uri = str(entity) + if not found_entity(uri): + statement[VALIDATION_ERRORS].entities.add(uri) + has_invalid_entities = True + + return has_invalid_entities + + +def annotate_invalid_sex(statement: Dict) -> bool: + if statement[SEX]: + if len(statement[SEX]) > 1: + logger_service.add_anomaly( + LoggableAnomaly(statement[ID], None, f'Multiple sexes found in statement.')) + + first_sex_uri = statement[SEX][0] + if not Sex.objects.filter(ontology_uri=first_sex_uri).exists(): + statement[VALIDATION_ERRORS].sex.add(first_sex_uri) + return True + return False + + +def annotate_invalid_species(statement: Dict) -> bool: + has_invalid_species = False + for species_uri in statement[SPECIES]: + if not Specie.objects.filter(ontology_uri=species_uri).exists(): + statement[VALIDATION_ERRORS].species.add(species_uri) + has_invalid_species = True + return has_invalid_species + + +def annotate_invalid_forward_connections(statement: Dict, statement_ids: Set[str]) -> bool: + has_invalid_forward_connection = False + for reference_uri in statement[FORWARD_CONNECTION]: + if reference_uri not in statement_ids: + statement[VALIDATION_ERRORS].forward_connection.add(reference_uri) + has_invalid_forward_connection = True + return has_invalid_forward_connection + + +def found_entity(uri: str, Model: Type[DjangoModel] = None) -> bool: + if Model: + return Model.objects.filter(ontology_uri=uri).exists() + return AnatomicalEntityMeta.objects.filter(ontology_uri=uri).exists() diff --git a/backend/composer/services/cs_ingestion/logging_service.py b/backend/composer/services/cs_ingestion/logging_service.py index a525492c..1a0e7678 100644 --- a/backend/composer/services/cs_ingestion/logging_service.py +++ b/backend/composer/services/cs_ingestion/logging_service.py @@ -2,7 +2,7 @@ from typing import List, Dict from composer.enums import CSState, SentenceState -from composer.services.cs_ingestion.helpers import ID, LABEL, STATE, VALIDATION_ERRORS +from composer.services.cs_ingestion.helpers.common_helpers import ID, LABEL, STATE, VALIDATION_ERRORS from composer.services.cs_ingestion.models import LoggableAnomaly AXIOM_NOT_FOUND = "Entity not found in any axiom" @@ -12,7 +12,20 @@ INCONSISTENT_AXIOMS = "Region and layer found in different axioms" -class LoggerService: +class SingletonMeta(type): + """ + This is a thread-safe implementation of Singleton. + """ + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + + +class LoggerService(metaclass=SingletonMeta): def __init__(self, ingestion_anomalies_log_path='ingestion_anomalies_log.csv', ingested_log_path='ingested_log.csv'): self.anomalies_log_path = ingestion_anomalies_log_path diff --git a/backend/composer/services/cs_ingestion/models.py b/backend/composer/services/cs_ingestion/models.py index e46ff10c..0fd1cc40 100644 --- a/backend/composer/services/cs_ingestion/models.py +++ b/backend/composer/services/cs_ingestion/models.py @@ -3,21 +3,21 @@ class NeuronDMOrigin: - def __init__(self, anatomical_entities_uri: Set): - self.anatomical_entities = anatomical_entities_uri + def __init__(self, anatomical_entities: Set): + self.anatomical_entities = anatomical_entities class NeuronDMVia: - def __init__(self, anatomical_entities_uri: Set, from_entities: Set, order: int, type: str): - self.anatomical_entities = anatomical_entities_uri + def __init__(self, anatomical_entities: Set, from_entities: Set, order: int, type: str): + self.anatomical_entities = anatomical_entities self.from_entities = from_entities self.order = order self.type = type class NeuronDMDestination: - def __init__(self, anatomical_entities_uri: Set, from_entities: Set, type: str): - self.anatomical_entities = anatomical_entities_uri + def __init__(self, anatomical_entities: Set, from_entities: Set, type: str): + self.anatomical_entities = anatomical_entities self.from_entities = from_entities self.type = type diff --git a/backend/composer/services/cs_ingestion/neurondm_script.py b/backend/composer/services/cs_ingestion/neurondm_script.py index 2dbc5ed0..8b95e6d9 100644 --- a/backend/composer/services/cs_ingestion/neurondm_script.py +++ b/backend/composer/services/cs_ingestion/neurondm_script.py @@ -9,7 +9,7 @@ from pyontutils.namespaces import rdfs, ilxtr from composer.services.cs_ingestion.exceptions import NeuronDMInconsistency -from composer.services.cs_ingestion.helpers import VALIDATION_ERRORS, DESTINATIONS, VIAS, ORIGINS +from composer.services.cs_ingestion.helpers.common_helpers import VALIDATION_ERRORS, DESTINATIONS, VIAS, ORIGINS from composer.services.cs_ingestion.logging_service import LoggerService, AXIOM_NOT_FOUND from composer.services.cs_ingestion.models import NeuronDMVia, NeuronDMOrigin, NeuronDMDestination, LoggableAnomaly, \ AxiomType, ValidationErrors, Severity @@ -85,8 +85,7 @@ def get_connections(n, lpes): tmp_origins, tmp_vias, tmp_destinations, validation_errors = process_connections(partial_order, set(origins_from_axioms), vias_from_axioms, - destinations_from_axioms - ) + destinations_from_axioms) validation_errors = validate_partial_order_and_axioms(origins_from_axioms, vias_from_axioms, destinations_from_axioms, tmp_origins, @@ -109,9 +108,13 @@ def create_uri_type_dict(lpes_func, predicate_type_map): return uri_type_dict -def process_connections(path, origins_from_axioms: Set[str], vias_from_axioms: Dict[str, str], - destinations_from_axioms: Dict[str, str], from_entities: Optional[Set[str]] = None, - depth: int = 0, result: Optional[Dict] = None) -> Tuple[ +def process_connections(path, + origins_from_axioms: Set[str], + vias_from_axioms: Dict[str, str], + destinations_from_axioms: Dict[str, str], + from_entities: Optional[Set[str]] = None, + depth: int = 0, + result: Optional[Dict] = None) -> Tuple[ List[NeuronDMOrigin], List[NeuronDMVia], List[NeuronDMDestination], ValidationErrors]: if result is None: result = {ORIGINS: [], DESTINATIONS: [], VIAS: [], VALIDATION_ERRORS: ValidationErrors()} @@ -124,26 +127,27 @@ def process_connections(path, origins_from_axioms: Set[str], vias_from_axioms: D else: current_entity = path[0] - current_entity_uri, current_entity_axiom_types = get_current_entity_metadata(current_entity, - origins_from_axioms, - vias_from_axioms, - destinations_from_axioms) + current_entity_axiom_types = get_matched_axiom_types(current_entity, + origins_from_axioms, + vias_from_axioms, + destinations_from_axioms) - if not current_entity_uri or len(current_entity_axiom_types) == 0: - result[VALIDATION_ERRORS].axiom_not_found.add(entity_to_string(current_entity)) + if not current_entity or len(current_entity_axiom_types) == 0: + current_entity_representation = entity_to_string(current_entity) + result[VALIDATION_ERRORS].axiom_not_found.add(current_entity_representation) if logger_service: - logger_service.add_anomaly(LoggableAnomaly(None, current_entity, AXIOM_NOT_FOUND)) + logger_service.add_anomaly(LoggableAnomaly(None, current_entity_representation, AXIOM_NOT_FOUND)) else: from_entities = from_entities or set() axiom_type = get_axiom_type(current_entity_axiom_types, path, depth) - update_result(current_entity_uri, axiom_type, from_entities, depth, result, vias_from_axioms, + update_result(current_entity, axiom_type, from_entities, depth, result, vias_from_axioms, destinations_from_axioms) depth += 1 - next_from_entities = {current_entity_uri} if current_entity_uri else from_entities + next_from_entities = {current_entity} if current_entity else from_entities # Process the next level structures, carrying over from_entities as a set for remaining_path in path[1:]: process_connections(remaining_path, origins_from_axioms, vias_from_axioms, destinations_from_axioms, @@ -154,36 +158,36 @@ def process_connections(path, origins_from_axioms: Set[str], vias_from_axioms: D def entity_to_string(entity): if isinstance(entity, orders.rl): - return f"{entity.region} (region) nor {entity.layer} (layer)" + return f"{entity.region} (region), {entity.layer} (layer)" else: return str(entity) -def get_current_entity_metadata(current_entity, origins_from_axioms: Set[str], vias_from_axioms: Dict[str, str], - destinations_from_axioms: Dict[str, str]) -> Tuple[Optional[str], List[AxiomType]]: - primary_uri = current_entity.toPython() if not isinstance(current_entity, - orders.rl) else current_entity.region.toPython() - secondary_uri = current_entity.layer.toPython() if isinstance(current_entity, orders.rl) else None +def get_matched_axiom_types(current_entity: rdflib.term, + origins_from_axioms: Set[str], + vias_from_axioms: Dict[str, str], + destinations_from_axioms: Dict[str, str]) -> List[AxiomType]: + # Check if current_entity is a complex region-layer pair (orders.rl) and extract URIs accordingly + if isinstance(current_entity, orders.rl): + primary_uri = current_entity.region.toPython() + secondary_uri = current_entity.layer.toPython() + else: + primary_uri = current_entity.toPython() + secondary_uri = None + + matched_types = [] uris_in_axioms = [ (origins_from_axioms, AxiomType.ORIGIN), - (vias_from_axioms, AxiomType.VIA), - (destinations_from_axioms, AxiomType.DESTINATION), + (vias_from_axioms.keys(), AxiomType.VIA), + (destinations_from_axioms.keys(), AxiomType.DESTINATION), ] - uris_found = {} for uri_set, node_type in uris_in_axioms: - # Check if the URIs are in the current set of axioms if primary_uri in uri_set or secondary_uri in uri_set: - # Prefer layer if both region and layer URIs are found - matched_uri = secondary_uri if secondary_uri in uri_set else primary_uri - uris_found.setdefault(matched_uri, []).append(node_type) + matched_types.append(node_type) - if not uris_found: - return None, [] - - matched_uri, matched_types = next(iter(uris_found.items()), (None, [])) - return matched_uri, matched_types + return matched_types def get_axiom_type(current_entity_axiom_types: List[AxiomType], path, depth: int) -> Optional[AxiomType]: @@ -210,33 +214,48 @@ def get_axiom_type(current_entity_axiom_types: List[AxiomType], path, depth: int return None -def update_result(current_entity_uri: str, axiom_type: AxiomType, from_entities: Set[str], depth: int, result: Dict, +def update_result(current_entity: rdflib.term, axiom_type: AxiomType, from_entities: Set[str], depth: int, result: Dict, vias_from_axioms: Dict[str, str], destinations_from_axioms: Dict[str, str]) -> Dict: if axiom_type == AxiomType.ORIGIN: - result[ORIGINS].append(NeuronDMOrigin({current_entity_uri})) + result[ORIGINS].append(NeuronDMOrigin({current_entity})) elif axiom_type == AxiomType.VIA: result[VIAS].append( - NeuronDMVia({current_entity_uri}, from_entities, depth, vias_from_axioms.get(current_entity_uri))) + NeuronDMVia({current_entity}, from_entities, depth, get_entity_type(current_entity, vias_from_axioms))) elif axiom_type == AxiomType.DESTINATION: result[DESTINATIONS].append( - NeuronDMDestination({current_entity_uri}, from_entities, destinations_from_axioms.get(current_entity_uri))) + NeuronDMDestination({current_entity}, from_entities, + get_entity_type(current_entity, destinations_from_axioms))) return result -def validate_partial_order_and_axioms(origins_from_axioms, vias_from_axioms, destinations_from_axioms, tmp_origins, - tmp_vias, tmp_destinations, +def get_entity_type(current_entity, axiom_dict): + if isinstance(current_entity, orders.rl): + # Try to find the type based on the layer, then region + return axiom_dict.get(str(current_entity.layer)) or axiom_dict.get(str(current_entity.region)) + else: + return axiom_dict.get(str(current_entity)) + + +def validate_partial_order_and_axioms(origins_from_axioms, vias_from_axioms, destinations_from_axioms, + tmp_origins, tmp_vias, tmp_destinations, validation_errors: ValidationErrors) -> ValidationErrors: anatomical_uris_origins = extract_anatomical_uris(tmp_origins) anatomical_uris_vias = extract_anatomical_uris(tmp_vias) anatomical_uris_destinations = extract_anatomical_uris(tmp_destinations) - validate_partial_order_and_axioms_aux(set(origins_from_axioms), anatomical_uris_origins, "origins", + validate_partial_order_and_axioms_aux(set(origins_from_axioms), + anatomical_uris_origins, + "origins", validation_errors) - validate_partial_order_and_axioms_aux(set(vias_from_axioms.keys()), anatomical_uris_vias, "vias", validation_errors) + validate_partial_order_and_axioms_aux(set(vias_from_axioms.keys()), + anatomical_uris_vias, + "vias", + validation_errors) - validate_partial_order_and_axioms_aux(set(destinations_from_axioms.keys()), anatomical_uris_destinations, + validate_partial_order_and_axioms_aux(set(destinations_from_axioms.keys()), + anatomical_uris_destinations, "destinations", validation_errors) @@ -245,19 +264,54 @@ def validate_partial_order_and_axioms(origins_from_axioms, vias_from_axioms, des def validate_partial_order_and_axioms_aux(axiom_uris: Set[str], actual_uris: Set[str], category: str, validation_errors: ValidationErrors): - missing_uris = axiom_uris - actual_uris - extra_uris = actual_uris - axiom_uris - if missing_uris: - validation_errors.non_specified.append( - f"Missing anatomical URIs in {category}: {', '.join(missing_uris)}") - if extra_uris: - validation_errors.non_specified.append( - f"Unexpected anatomical URIs in {category}: {', '.join(extra_uris)}" - ) + unexpected_uris = get_unexpected_uris(actual_uris, axiom_uris) + missing_uris = get_missing_uris(actual_uris, axiom_uris) + + for uri in unexpected_uris: + uri_str = f"{uri[0]}, {uri[1]}" if isinstance(uri, tuple) else uri + validation_errors.non_specified.append(f"Unexpected {category} URI not in axioms: {uri_str}") + + for uri in missing_uris: + validation_errors.non_specified.append(f"Missing {category} URI not found in actual URIs: {uri}") + + return validation_errors + + +def get_missing_uris(actual_uris, axiom_uris): + flattened_actual_uris = set() + for uri in actual_uris: + if isinstance(uri, tuple): + flattened_actual_uris.update(uri) + else: + flattened_actual_uris.add(uri) + missing_uris = axiom_uris.difference(flattened_actual_uris) + return missing_uris + + +def get_unexpected_uris(actual_uris, axiom_uris): + unexpected_uris = set() + # Identify actual URIs that are unexpectedly present + for actual_uri in actual_uris: + if isinstance(actual_uri, tuple): # Complex entity case (region, layer pairs) + region, layer = actual_uri + # Count as unexpected if neither region nor layer are in axiom_uris + if not (region in axiom_uris or layer in axiom_uris): + unexpected_uris.add(actual_uri) + else: # Simple URI case + if actual_uri not in axiom_uris: + unexpected_uris.add(actual_uri) + return unexpected_uris def extract_anatomical_uris(entities_list): - return set(uri for entity in entities_list for uri in entity.anatomical_entities) + uris = set() + for entity in entities_list: + for anatomical_entity in entity.anatomical_entities: + if isinstance(anatomical_entity, orders.rl): + uris.add((anatomical_entity.region.toPython(), anatomical_entity.layer.toPython())) + else: # Simple URIRef + uris.add(anatomical_entity.toPython()) + return uris def merge_origins(origins: List[NeuronDMOrigin]) -> NeuronDMOrigin: diff --git a/backend/composer/signals.py b/backend/composer/signals.py index 7aa6e54b..70ae94a2 100644 --- a/backend/composer/signals.py +++ b/backend/composer/signals.py @@ -1,11 +1,12 @@ from django.core.exceptions import ValidationError from django.dispatch import receiver -from django.db.models.signals import post_save, m2m_changed +from django.db.models.signals import post_save, m2m_changed, pre_save from django_fsm.signals import post_transition from .enums import CSState, NoteType -from .models import ConnectivityStatement, ExportBatch, Note, Sentence +from .models import ConnectivityStatement, ExportBatch, Note, Sentence, AnatomicalEntity, AnatomicalEntityIntersection, \ + AnatomicalEntityMeta from .services.export_services import compute_metrics, ConnectivityStatementStateService @@ -41,8 +42,20 @@ def post_transition_callback(sender, instance, name, source, target, **kwargs): def post_transition_cs(sender, instance, name, source, target, **kwargs): if issubclass(sender, ConnectivityStatement): if target == CSState.COMPOSE_NOW and source in ( - CSState.NPO_APPROVED, - CSState.EXPORTED, + CSState.NPO_APPROVED, + CSState.EXPORTED, ): # add important tag to CS when transition to COMPOSE_NOW from NPO Approved or Exported instance = ConnectivityStatementStateService.add_important_tag(instance) + + +@receiver(post_save, sender=AnatomicalEntityIntersection) +def create_region_layer_anatomical_entity(sender, instance=None, created=False, **kwargs): + if created and instance: + AnatomicalEntity.objects.create(region_layer=instance) + + +@receiver(post_save, sender=AnatomicalEntityMeta) +def create_simple_anatomical_entity(sender, instance=None, created=False, **kwargs): + if created and instance: + AnatomicalEntity.objects.create(simple_entity=instance) diff --git a/docker-compose-db.yaml b/docker-compose-db.yaml new file mode 100644 index 00000000..3f2b557f --- /dev/null +++ b/docker-compose-db.yaml @@ -0,0 +1,15 @@ +version: "3.9" +services: + composer-db: + image: postgres:13 + restart: "on-failure" + volumes: + - composer-db-data:/var/lib/postgresql/data + ports: + - "5432:5432" + env_file: + - .env + +volumes: + composer-db-data: + composer-data: diff --git a/frontend/src/apiclient/backend/api.ts b/frontend/src/apiclient/backend/api.ts index c05ca04b..444a3e50 100644 --- a/frontend/src/apiclient/backend/api.ts +++ b/frontend/src/apiclient/backend/api.ts @@ -24,7 +24,7 @@ import type { RequestArgs } from './base'; import { BASE_PATH, COLLECTION_FORMATS, BaseAPI, RequiredError } from './base'; /** - * Anatomical Entity + * * @export * @interface AnatomicalEntity */ @@ -37,14 +37,64 @@ export interface AnatomicalEntity { 'id': number; /** * - * @type {string} + * @type {AnatomicalEntityMeta} + * @memberof AnatomicalEntity + */ + 'simple_entity': AnatomicalEntityMeta; + /** + * + * @type {AnatomicalEntityIntersection} * @memberof AnatomicalEntity */ + 'region_layer': AnatomicalEntityIntersection; +} +/** + * + * @export + * @interface AnatomicalEntityIntersection + */ +export interface AnatomicalEntityIntersection { + /** + * + * @type {number} + * @memberof AnatomicalEntityIntersection + */ + 'id': number; + /** + * + * @type {Layer} + * @memberof AnatomicalEntityIntersection + */ + 'layer': Layer; + /** + * + * @type {Region} + * @memberof AnatomicalEntityIntersection + */ + 'region': Region; +} +/** + * + * @export + * @interface AnatomicalEntityMeta + */ +export interface AnatomicalEntityMeta { + /** + * + * @type {number} + * @memberof AnatomicalEntityMeta + */ + 'id': number; + /** + * + * @type {string} + * @memberof AnatomicalEntityMeta + */ 'name': string; /** * * @type {string} - * @memberof AnatomicalEntity + * @memberof AnatomicalEntityMeta */ 'ontology_uri': string; } @@ -632,6 +682,31 @@ export const LateralityEnum = { export type LateralityEnum = typeof LateralityEnum[keyof typeof LateralityEnum]; +/** + * + * @export + * @interface Layer + */ +export interface Layer { + /** + * + * @type {number} + * @memberof Layer + */ + 'id': number; + /** + * + * @type {string} + * @memberof Layer + */ + 'name': string; + /** + * + * @type {string} + * @memberof Layer + */ + 'ontology_uri': string; +} /** * * @export @@ -1561,6 +1636,37 @@ export interface Provenance { */ 'connectivity_statement_id': number; } +/** + * + * @export + * @interface Region + */ +export interface Region { + /** + * + * @type {number} + * @memberof Region + */ + 'id': number; + /** + * + * @type {string} + * @memberof Region + */ + 'name': string; + /** + * + * @type {string} + * @memberof Region + */ + 'ontology_uri': string; + /** + * + * @type {Array} + * @memberof Region + */ + 'layers': Array; +} /** * Sentence * @export diff --git a/frontend/src/components/AnatomicalEntitiesField.tsx b/frontend/src/components/AnatomicalEntitiesField.tsx index e51c563e..44f1c775 100644 --- a/frontend/src/components/AnatomicalEntitiesField.tsx +++ b/frontend/src/components/AnatomicalEntitiesField.tsx @@ -5,6 +5,7 @@ import {composerApi as api} from "../services/apis"; import {autocompleteRows} from "../helpers/settings"; import theme from "../theme/Theme"; import Typography from "@mui/material/Typography"; +import { getName } from "../helpers/anatomicalEntityHelper"; function AnatomicalEntitiesField(props: any) { const {label, errors, isDisabled} = props.options; @@ -31,7 +32,7 @@ function AnatomicalEntitiesField(props: any) { api.composerAnatomicalEntityList([], autocompleteRows, inputValue, 0); let inputValue; - !props.value ? (inputValue = "") : (inputValue = entity?.name); + !props.value ? (inputValue = "") : (inputValue = entity? getName(entity) : ''); if (loading) { return
Loading...
; diff --git a/frontend/src/components/ProofingTab/GraphDiagram/GraphDiagram.tsx b/frontend/src/components/ProofingTab/GraphDiagram/GraphDiagram.tsx index 5a827684..c80f1ec6 100644 --- a/frontend/src/components/ProofingTab/GraphDiagram/GraphDiagram.tsx +++ b/frontend/src/components/ProofingTab/GraphDiagram/GraphDiagram.tsx @@ -13,6 +13,7 @@ import createEngine, { import {CanvasWidget} from '@projectstorm/react-canvas-core'; import {CustomNodeModel} from "./Models/CustomNodeModel"; import {CustomNodeFactory} from "./Factories/CustomNodeFactory"; +import {getURI, getName} from "../../../helpers/anatomicalEntityHelper"; export enum NodeTypes { @@ -43,14 +44,14 @@ function getExternalID(url: string) { return parts[parts.length - 1].replace('_', ':'); } -function getId(layerId: string, entity: AnatomicalEntity) { +function getOrderId(layerId: string, entity: AnatomicalEntity) { return layerId + entity.id.toString(); } function findNodeForEntity(entity: AnatomicalEntity, nodeMap: Map, maxLayerIndex: number) { for (let layerIndex = 0; layerIndex <= maxLayerIndex; layerIndex++) { let layerId = layerIndex === 0 ? NodeTypes.Origin : NodeTypes.Via + layerIndex - let id = getId(layerId, entity); + let id = getOrderId(layerId, entity); if (nodeMap.has(id)) { return nodeMap.get(id); } @@ -86,11 +87,11 @@ const processData = ( let xOrigin = 100 origins?.forEach(origin => { - const id = getId(NodeTypes.Origin, origin) + const id = getOrderId(NodeTypes.Origin, origin) const originNode = new CustomNodeModel( NodeTypes.Origin, - origin.name, - getExternalID(origin.ontology_uri), + getName(origin), + getExternalID(getURI(origin)), { to: [], } @@ -107,11 +108,11 @@ const processData = ( let xVia = 100 let yVia = layerIndex * yIncrement + yStart; via.anatomical_entities.forEach(entity => { - const id = getId(NodeTypes.Via + layerIndex, entity) + const id = getOrderId(NodeTypes.Via + layerIndex, entity) const viaNode = new CustomNodeModel( NodeTypes.Via, - entity.name, - getExternalID(entity.ontology_uri), + getName(entity), + getExternalID(getURI(entity)), { from: [], to: [], @@ -150,8 +151,8 @@ const processData = ( destination.anatomical_entities.forEach(entity => { const destinationNode = new CustomNodeModel( NodeTypes.Destination, - entity.name, - getExternalID(entity.ontology_uri), + getName(entity), + getExternalID(getURI(entity)), { from: [], anatomicalType: destination?.type ? DestinationTypeMapping[destination.type] : '' diff --git a/frontend/src/helpers/anatomicalEntityHelper.ts b/frontend/src/helpers/anatomicalEntityHelper.ts new file mode 100644 index 00000000..6d5690bb --- /dev/null +++ b/frontend/src/helpers/anatomicalEntityHelper.ts @@ -0,0 +1,15 @@ +import {AnatomicalEntity} from "../apiclient/backend"; + +export const getName = (anatomicalEntity: AnatomicalEntity) => { + if (anatomicalEntity.region_layer) { + return `${anatomicalEntity.region_layer.region.name},${anatomicalEntity.region_layer.layer.name}` + } + return anatomicalEntity.simple_entity.name +} + +export const getURI = (anatomicalEntity: AnatomicalEntity) => { + if (anatomicalEntity.region_layer) { + return `${anatomicalEntity.region_layer.region.ontology_uri},${anatomicalEntity.region_layer.layer.ontology_uri}` + } + return anatomicalEntity.simple_entity.ontology_uri +} \ No newline at end of file diff --git a/frontend/src/helpers/dropdownMappers.ts b/frontend/src/helpers/dropdownMappers.ts index 87b228dd..7e008214 100644 --- a/frontend/src/helpers/dropdownMappers.ts +++ b/frontend/src/helpers/dropdownMappers.ts @@ -4,6 +4,7 @@ import { ConnectivityStatementUpdate, } from "../apiclient/backend"; import {Option, OptionDetail} from "../types"; +import {getURI, getName } from "./anatomicalEntityHelper"; import {OriginsGroupLabel, ViasGroupLabel} from "./settings"; export const DROPDOWN_MAPPER_ONTOLOGY_URL = "Ontology URI"; @@ -20,16 +21,16 @@ export function mapAnatomicalEntitiesToOptions( } return entities.map((entity: any) => ({ id: entity.id.toString(), - label: entity.name, + label: getName(entity), group: groupLabel, content: [ { title: "Name", - value: entity.name, + value: getName(entity), }, { title: DROPDOWN_MAPPER_ONTOLOGY_URL, - value: entity.ontology_uri, + value: getURI(entity), }, ], })); diff --git a/frontend/src/helpers/helpers.ts b/frontend/src/helpers/helpers.ts index 304df2e9..8a3f185c 100644 --- a/frontend/src/helpers/helpers.ts +++ b/frontend/src/helpers/helpers.ts @@ -3,6 +3,7 @@ import { SentenceAvailableTransitionsEnum as sentenceStates, } from "../apiclient/backend/api"; import { ComposerConnectivityStatementListStateEnum as statementStates } from "../apiclient/backend/api"; +import { getName } from "./anatomicalEntityHelper"; export const hiddenWidget = (fields: string[]) => { let hiddenSchema = {}; @@ -213,9 +214,9 @@ export function searchAnatomicalEntities( return entities .filter((entity) => - entity.name.toLowerCase().includes(normalizedSearchValue), + getName(entity).toLowerCase().includes(normalizedSearchValue), ) - .sort((a, b) => a.name.localeCompare(b.name)); + .sort((a, b) => getName(a).localeCompare(getName(b))); } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 9b2b1d3d..74a8ca68 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -162,7 +162,6 @@ paths: - rejected - revise - to_be_reviewed - - invalid explode: true style: form - in: query @@ -1764,7 +1763,42 @@ components: schemas: AnatomicalEntity: type: object - description: Anatomical Entity + properties: + id: + type: integer + readOnly: true + simple_entity: + allOf: + - $ref: '#/components/schemas/AnatomicalEntityMeta' + readOnly: true + region_layer: + allOf: + - $ref: '#/components/schemas/AnatomicalEntityIntersection' + readOnly: true + required: + - id + - region_layer + - simple_entity + AnatomicalEntityIntersection: + type: object + properties: + id: + type: integer + readOnly: true + layer: + allOf: + - $ref: '#/components/schemas/Layer' + readOnly: true + region: + allOf: + - $ref: '#/components/schemas/Region' + readOnly: true + required: + - id + - layer + - region + AnatomicalEntityMeta: + type: object properties: id: type: integer @@ -1775,7 +1809,7 @@ components: ontology_uri: type: string format: uri - readOnly: true + maxLength: 200 required: - id - name @@ -2158,6 +2192,23 @@ components: - RIGHT - LEFT type: string + Layer: + type: object + properties: + id: + type: integer + readOnly: true + name: + type: string + maxLength: 200 + ontology_uri: + type: string + format: uri + maxLength: 200 + required: + - id + - name + - ontology_uri Login: type: object properties: @@ -2703,6 +2754,29 @@ components: - connectivity_statement_id - id - uri + Region: + type: object + properties: + id: + type: integer + readOnly: true + name: + type: string + maxLength: 200 + ontology_uri: + type: string + format: uri + maxLength: 200 + layers: + type: array + items: + $ref: '#/components/schemas/Layer' + readOnly: true + required: + - id + - layers + - name + - ontology_uri Sentence: type: object description: Sentence