Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/sckan-324 - journey computation #381

Open
wants to merge 13 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions backend/composer/api/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,20 +252,23 @@ class ModelRetrieveViewSet(
# mixins.DestroyModelMixin,
# mixins.ListModelMixin,
viewsets.GenericViewSet,
): ...
):
...


class ModelCreateRetrieveViewSet(
ModelRetrieveViewSet,
mixins.CreateModelMixin,
mixins.ListModelMixin,
): ...
):
...


class ModelNoDeleteViewSet(
ModelCreateRetrieveViewSet,
mixins.UpdateModelMixin,
): ...
):
...


class AnatomicalEntityViewSet(viewsets.ReadOnlyModelViewSet):
Expand Down Expand Up @@ -506,7 +509,8 @@ def my(self, request, *args, **kwargs):
msg = "User not logged in."
raise ValidationError(msg, code="authorization")

profile, created = Profile.objects.get_or_create(user=self.request.user)
profile, created = Profile.objects.get_or_create(
user=self.request.user)
return Response(self.get_serializer(profile).data)


Expand Down Expand Up @@ -547,6 +551,7 @@ class StatementAlertViewSet(viewsets.ModelViewSet):
serializer_class = StatementAlertSerializer
permission_classes = [IsOwnerOfConnectivityStatementOrReadOnly]


@extend_schema(
responses=OpenApiTypes.OBJECT,
)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 4.1.4 on 2024-12-18 08:25

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("composer", "0065_alter_statementalert_text"),
]

operations = [
migrations.AddField(
model_name="connectivitystatement",
name="journey_path",
field=models.JSONField(blank=True, null=True),
),
]
23 changes: 23 additions & 0 deletions backend/composer/migrations/0067_auto_20241218_0928.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Generated by Django 4.1.4 on 2024-12-18 08:28

from django.db import migrations
from django.db import migrations
from composer.services.graph_service import compile_journey


def update_journey_fields(apps, schema_editor):
ConnectivityStatement = apps.get_model('composer', 'ConnectivityStatement')
for cs in ConnectivityStatement.objects.all():
cs.journey_path = compile_journey(cs)
cs.save(update_fields=["journey_path"])


class Migration(migrations.Migration):

dependencies = [
("composer", "0066_connectivitystatement_journey_path"),
]

operations = [
migrations.RunPython(update_journey_fields),
]
9 changes: 4 additions & 5 deletions backend/composer/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from django.db.models.expressions import F
from django.forms.widgets import Input as InputWidget
from django_fsm import FSMField, transition
from composer.services.graph_service import build_journey_description, build_journey_entities

from composer.services.layers_service import update_from_entities_on_deletion
from composer.services.state_services import (
Expand All @@ -21,7 +22,6 @@
ViaType,
Projection,
)
from .services.graph_service import compile_journey
from .utils import doi_uri, pmcid_uri, pmid_uri, create_reference_uri


Expand Down Expand Up @@ -567,6 +567,7 @@ class ConnectivityStatement(models.Model):
)
created_date = models.DateTimeField(auto_now_add=True, db_index=True)
modified_date = models.DateTimeField(auto_now=True, db_index=True)
journey_path = models.JSONField(null=True, blank=True)

def __str__(self):
suffix = ""
Expand Down Expand Up @@ -683,11 +684,10 @@ def get_previous_layer_entities(self, via_order):
return set(self.via_set.get(order=via_order - 1).anatomical_entities.all())

def get_journey(self):
return compile_journey(self)['journey']
return build_journey_description(self.journey_path, self)

def get_entities_journey(self):
entities_journey = compile_journey(self)['entities']
return entities_journey
return build_journey_entities(self.journey_path, self)

def get_laterality_description(self):
laterality_map = {
Expand Down Expand Up @@ -716,7 +716,6 @@ def save(self, *args, **kwargs):
self.reference_uri = create_reference_uri(self.pk)
self.save(update_fields=["reference_uri"])


def set_origins(self, origin_ids):
self.origins.set(origin_ids, clear=True)
self.save()
Expand Down
75 changes: 49 additions & 26 deletions backend/composer/services/graph_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,7 @@ def consolidate_paths(paths):

paths = consolidated + [paths[i] for i in range(len(paths)) if i not in used_indices]

return paths, [[((node[1].replace(JOURNEY_DELIMITER, ' or '), node[2]) if (
node[2] == 0 or path.index(node) == len(path) - 1) else (
node[1].replace(JOURNEY_DELIMITER, ', '), node[2])) for node in path] for path in paths]
return paths


def can_merge(path1, path2):
Expand Down Expand Up @@ -144,7 +142,7 @@ def merge_paths(path1, path2):
return merged_path


def compile_journey(connectivity_statement) -> dict:
def compile_journey(connectivity_statement) -> List[str]:
"""
Generates a string of descriptions of journey paths for a given connectivity statement.

Expand All @@ -157,17 +155,55 @@ def compile_journey(connectivity_statement) -> dict:
# Extract origins, vias, and destinations from the connectivity statement
Via = apps.get_model('composer', 'Via')
Destination = apps.get_model('composer', 'Destination')
Origin = apps.get_model('composer', 'AnatomicalEntity')

origins = list(connectivity_statement.origins.all())

vias = list(Via.objects.filter(connectivity_statement=connectivity_statement))
destinations = list(Destination.objects.filter(connectivity_statement=connectivity_statement))
vias = list(Via.objects.filter(connectivity_statement__id=connectivity_statement.id))
destinations = list(Destination.objects.filter(connectivity_statement__id=connectivity_statement.id))
origins = list(Origin.objects.filter(origins_relations__id=connectivity_statement.id).distinct())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question: Why did we change from:
origins = list(connectivity_statement.origins.all())
to
origins = list(Origin.objects.filter(origins_relations__id=connectivity_statement.id).distinct())

Copy link
Contributor Author

@D-GopalKrishna D-GopalKrishna Dec 18, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reason is here in the code - https://github.com/MetaCell/sckan-composer/blob/feature/SCKAN-324/backend/composer/services/graph_service.py#L24

It says:

Exception has occurred: AttributeError       (note: full exception trace is shown but execution is paused at: _run_module_as_main)
'AnatomicalEntity' object has no attribute 'name'
  File "/home/dgk/metacell/sckan-composer/backend/composer/services/graph_service.py", line 24, in generate_paths
    paths.extend(create_paths_from_origin(origin, vias, destinations, [(str(origin.id), origin.name, 0)], destination_layer))
  File "/home/dgk/metacell/sckan-composer/backend/composer/services/graph_service.py", line 166, in compile_journey
    all_paths2 = generate_paths(origins, vias, destinations)
  File "/home/dgk/metacell/sckan-composer/backend/composer/migrations/0067_auto_20241218_0928.py", line 11, in update_journey_fields
    cs.journey_path = compile_journey(cs)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 193, in database_forwards
    self.code(from_state.apps, schema_editor)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/db/migrations/migration.py", line 130, in apply
    operation.database_forwards(
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/db/migrations/executor.py", line 252, in apply_migration
    state = migration.apply(state, schema_editor)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/db/migrations/executor.py", line 167, in _migrate_all_forwards
    state = self.apply_migration(
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/db/migrations/executor.py", line 135, in migrate
    state = self._migrate_all_forwards(
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/core/management/commands/migrate.py", line 349, in handle
    post_migrate_state = executor.migrate(
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/core/management/base.py", line 96, in wrapped
    res = handle_func(*args, **kwargs)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/core/management/base.py", line 448, in execute
    output = self.handle(*args, **options)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/core/management/base.py", line 402, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/core/management/__init__.py", line 440, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/site-packages/django/core/management/__init__.py", line 446, in execute_from_command_line
    utility.execute()
  File "/home/dgk/metacell/sckan-composer/backend/manage.py", line 18, in main
    execute_from_command_line(sys.argv)
  File "/home/dgk/metacell/sckan-composer/backend/manage.py", line 22, in <module>
    main()
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/home/dgk/miniconda3/envs/composer/lib/python3.9/runpy.py", line 197, in _run_module_as_main (Current frame)
    return _run_code(code, main_globals, None,
AttributeError: 'AnatomicalEntity' object has no attribute 'name'

so when generating the path - we do have the origins, but not the origin.name - which is a property in AnatomicalEntity.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@afonsobspinto did you do the check with the API call?

This happens only when we do the migration - possibly something with - apps.get_model('composer', 'ConnectivityStatement') in the migration file (ref - 0067 auto migration file). Also if you notice this is only for the @properties and not for the fields.
So if you put a breakpoint to the above mentioned code, and then run the migration to db - this should cause the error.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Understood, during migrations django uses historical models that don't include python level constructs like @Property.
I think the new instruction works, but I would remove the .distinct(), I don't think it's needed in our context

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Before removing it, I want to confirm again with you if you are okay to remove the distinct() here, the reason I had that is because unlike vias and destinations - which are FK, origins is a M2M.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ManyToManyField in Django automatically ensures that the relationship between two models is unique. In other words, we know that we have a unique pair of ConnectivityStatement and AnatomicalEntity. Here's a simple test:

image

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With that said, if we have the time I would apply the same suggestion I mentioned here:
#404 (comment)

# origins = list(connectivity_statement.origins.all())

# Generate all paths and then consolidate them
all_paths2 = generate_paths(origins, vias, destinations)
consolidated_paths, journey_paths = consolidate_paths(all_paths2)
consolidated_paths = consolidate_paths(all_paths2)
return consolidated_paths


def get_journey_path_from_consolidated_paths(consolidated_paths):
journey_paths = [[((node[1].replace(JOURNEY_DELIMITER, ' or '), node[2]) if (
node[2] == 0 or path.index(node) == len(path) - 1) else (
node[1].replace(JOURNEY_DELIMITER, ', '), node[2])) for node in path] for path in consolidated_paths]
return journey_paths


def build_journey_description(consolidated_paths, connectivity_statement):
journey_paths = get_journey_path_from_consolidated_paths(
consolidated_paths)
Via = apps.get_model('composer', 'Via')
vias = list(Via.objects.filter(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question: Why do we need to fetch the vias here? can't we rely entirely on the new journey paths attribute created?

Copy link
Contributor Author

@D-GopalKrishna D-GopalKrishna Jan 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand what you mean yes, this is only to get the len(vias) later with - 0<layer<len(vias) + 1]

here is the journey_paths

image

and in this case, len(vias from the CS) is 1.
[<Via: Via object (37)>]

where the value of that via is:

{'_state': <django.db.models.base.ModelState object at 0x7f006b7fd8e0>, 'id': 37, 'connectivity_statement_id': 13, 'type': 'AXON', 'order': 0, '_prefetched_objects_cache': {'anatomical_entities': <QuerySet [<AnatomicalEntity: submucosa>, <AnatomicalEntity: nose>]>, 'from_entities': <QuerySet [<AnatomicalEntity: pituitary gland>, <AnatomicalEntity: zone of skin>]>}}

I am unable to see the pattern to relate them. Do you think there is one @afonsobspinto that I might be missing?

Copy link
Member

@afonsobspinto afonsobspinto Jan 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I understand the code correctly we don't need the vias count; we can just do something on the lines of:

        destination_layer = path[-1][1]  # Destination's layer
        via_names = ' via '.join([node for node, layer in path if 0 < layer < destination_layer])

which identifies the vias by looking for nodes with layers between origin (0) and destination (highest layer)

connectivity_statement=connectivity_statement))

# Create sentences for each journey path
journey_descriptions = []
for path in journey_paths:
origin_names = path[0][0]
destination_names = path[-1][0]
via_names = ' via '.join([node for node, layer in path if 0 < layer < len(vias) + 1])

if via_names:
sentence = f"from {origin_names} to {destination_names} via {via_names}"
else:
sentence = f"from {origin_names} to {destination_names}"

journey_descriptions.append(sentence)
return journey_descriptions


def build_journey_entities(consolidated_paths, connectivity_statement):
entities = []
Via = apps.get_model('composer', 'Via')
vias = list(Via.objects.filter(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

connectivity_statement=connectivity_statement))

for path in consolidated_paths:
origin_splits = path[0][0].split(JOURNEY_DELIMITER)
destination_splits = path[-1][0].split(JOURNEY_DELIMITER)
Expand All @@ -181,22 +217,9 @@ def compile_journey(connectivity_statement) -> dict:
'vias': [{'label': node, 'id': node_id} for node_id, node, layer in path if 0 < layer < len(vias) + 1]
}
entities.append(entity)
return entities

# Create sentences for each journey path
journey_descriptions = []
for path in journey_paths:
origin_names = path[0][0]
destination_names = path[-1][0]
via_names = ' via '.join([node for node, layer in path if 0 < layer < len(vias) + 1])

if via_names:
sentence = f"from {origin_names} to {destination_names} via {via_names}"
else:
sentence = f"from {origin_names} to {destination_names}"

journey_descriptions.append(sentence)

return {
'journey': journey_descriptions,
'entities': entities
}
def recompile_journey_path(instance):
instance.journey_path = compile_journey(instance)
instance.save(update_fields=["journey_path"])
11 changes: 11 additions & 0 deletions backend/composer/signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
Via,
)
from .services.export_services import compute_metrics, ConnectivityStatementStateService
from .services.graph_service import recompile_journey_path



@receiver(post_save, sender=ExportBatch)
Expand Down Expand Up @@ -101,6 +103,8 @@ def connectivity_statement_origins_changed(sender, instance, action, pk_set, **k
pass
except ValueError:
pass
recompile_journey_path(instance)


# Call `update_from_entities_on_deletion` for each removed entity
if action == "post_remove" and pk_set:
Expand All @@ -118,6 +122,7 @@ def via_anatomical_entities_changed(sender, instance, action, pk_set, **kwargs):
pass
except ValueError:
pass
recompile_journey_path(instance.connectivity_statement)

# Call `update_from_entities_on_deletion` for each removed entity
if action == "post_remove" and pk_set:
Expand All @@ -135,6 +140,8 @@ def via_from_entities_changed(sender, instance, action, **kwargs):
pass
except ValueError:
pass
recompile_journey_path(instance.connectivity_statement)



# Signals for Destination anatomical_entities
Expand All @@ -147,6 +154,8 @@ def destination_anatomical_entities_changed(sender, instance, action, **kwargs):
pass
except ValueError:
pass
recompile_journey_path(instance.connectivity_statement)



# Signals for Destination from_entities
Expand All @@ -159,6 +168,8 @@ def destination_from_entities_changed(sender, instance, action, **kwargs):
pass
except ValueError:
pass
recompile_journey_path(instance.connectivity_statement)



# Signals for Via model changes
Expand Down
Loading
Loading