From 1bd2751987c1697e403e2e1e100f2db5b2ea0129 Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Tue, 17 Jul 2018 16:03:14 -0400 Subject: [PATCH 01/26] Represent volumes as TIN geometry in the database All geometries in the back-end are now stored as PostGIS TIN Geometry. This allows for more consistency and a cleaner API. Implemented together with @aschampion, @clbarnes, @willp24. See catmaid/CATMAID#1765 Fixes catmaid/CATMAID#1581 --- django/applications/catmaid/control/volume.py | 39 ++++-- .../0049_volume_tin_representation.py | 124 ++++++++++++++++++ 2 files changed, 150 insertions(+), 13 deletions(-) create mode 100644 django/applications/catmaid/migrations/0049_volume_tin_representation.py diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 6f9ed45fa3..3918a9854b 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -166,19 +166,32 @@ def __init__(self, project_id, user_id, options): self.max_z = get_req_coordinate(options, "max_z") def get_geometry(self): - return """ST_GeomFromEWKT('POLYHEDRALSURFACE ( - ((%(lx)s %(ly)s %(lz)s, %(lx)s %(hy)s %(lz)s, %(hx)s %(hy)s %(lz)s, - %(hx)s %(ly)s %(lz)s, %(lx)s %(ly)s %(lz)s)), - ((%(lx)s %(ly)s %(lz)s, %(lx)s %(hy)s %(lz)s, %(lx)s %(hy)s %(hz)s, - %(lx)s %(ly)s %(hz)s, %(lx)s %(ly)s %(lz)s)), - ((%(lx)s %(ly)s %(lz)s, %(hx)s %(ly)s %(lz)s, %(hx)s %(ly)s %(hz)s, - %(lx)s %(ly)s %(hz)s, %(lx)s %(ly)s %(lz)s)), - ((%(hx)s %(hy)s %(hz)s, %(hx)s %(ly)s %(hz)s, %(lx)s %(ly)s %(hz)s, - %(lx)s %(hy)s %(hz)s, %(hx)s %(hy)s %(hz)s)), - ((%(hx)s %(hy)s %(hz)s, %(hx)s %(ly)s %(hz)s, %(hx)s %(ly)s %(lz)s, - %(hx)s %(hy)s %(lz)s, %(hx)s %(hy)s %(hz)s)), - ((%(hx)s %(hy)s %(hz)s, %(hx)s %(hy)s %(lz)s, %(lx)s %(hy)s %(lz)s, - %(lx)s %(hy)s %(hz)s, %(hx)s %(hy)s %(hz)s)))')""" + return """ST_GeomFromEWKT('TIN ( + (({0}, {2}, {1}, {0})), + (({1}, {2}, {3}, {1})), + + (({0}, {1}, {5}, {0})), + (({0}, {5}, {4}, {0})), + + (({2}, {6}, {7}, {2})), + (({2}, {7}, {3}, {2})), + + (({4}, {7}, {6}, {4})), + (({4}, {5}, {7}, {4})), + + (({0}, {6}, {2}, {0})), + (({0}, {4}, {6}, {0})), + + (({1}, {3}, {5}, {1})), + (({3}, {7}, {5}, {3})))') + """.format(*[ + '%({a})s %({b})s %({c})s'.format(**{ + 'a': 'hx' if i & 0b001 else 'lx', + 'b': 'hy' if i & 0b010 else 'ly', + 'c': 'hz' if i & 0b100 else 'lz', + }) + for i in range(8) + ]) def get_params(self): return { diff --git a/django/applications/catmaid/migrations/0049_volume_tin_representation.py b/django/applications/catmaid/migrations/0049_volume_tin_representation.py new file mode 100644 index 0000000000..1b1fc572f3 --- /dev/null +++ b/django/applications/catmaid/migrations/0049_volume_tin_representation.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-07-17 15:45 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +# Construct a PostGIS TIN representation of a bounding box +bb_vertices = """ + (({0}, {2}, {1}, {0})), + (({1}, {2}, {3}, {1})), + + (({0}, {1}, {5}, {0})), + (({0}, {5}, {4}, {0})), + + (({2}, {6}, {7}, {2})), + (({2}, {7}, {3}, {2})), + + (({4}, {7}, {6}, {4})), + (({4}, {5}, {7}, {4})), + + (({0}, {6}, {2}, {0})), + (({0}, {4}, {6}, {0})), + + (({1}, {3}, {5}, {1})), + (({3}, {7}, {5}, {3})) +""".format(*[ + '%{a}$s %{b}$s %{c}$s'.format(**{ + 'a': 1 if i & 0b001 else 2, + 'b': 3 if i & 0b010 else 4, + 'c': 5 if i & 0b100 else 6, + }) + for i in range(8) +]) + +forward = """ + DO $$ + BEGIN + -- Make sure we only deal with polyhedeal surfaces and TINs + IF (SELECT COUNT(*) FROM catmaid_volume + WHERE ST_GeometryType(geometry) NOT IN ('ST_PolyhedralSurface', 'ST_Tin') + LIMIT 1) <> 0 + THEN + RAISE EXCEPTION 'Only geometries of type ST_PolyhedralSurface and ' + 'ST_Tin are supported by CATMAID. Please fix volumes manually.'; + END IF; + + -- Make sure that all polyhedral surfaces have 30 vertices and six faces, + -- in which case we assume it is a simple box. + IF (SELECT COUNT(*) FROM catmaid_volume + WHERE ST_GeometryType(geometry) = 'ST_PolyhedralSurface' + AND (ST_NPoints(geometry) <> 30 OR ST_NumGeometries(geometry) <> 6) + LIMIT 1) <> 0 + THEN + RAISE EXCEPTION 'All polyhedral surfaces need to be boxes, i.e. ' + 'have 30 vertices and 6 faces. Please fix volumes manually.'; + END IF; + END + $$; + + SELECT disable_history_tracking_for_table('catmaid_volume'::regclass, + get_history_table_name('catmaid_volume'::regclass)); + SELECT drop_history_view_for_table('catmaid_volume'::regclass); + + -- Convert polyhedral surfaces to TINs, assuming that we only deal with + -- boxes. + UPDATE catmaid_volume + SET geometry = ST_GeomFromEWKT(FORMAT('TINZ ({bb_vertices})', + ST_XMax(geometry), + ST_XMin(geometry), + ST_YMax(geometry), + ST_YMin(geometry), + ST_ZMax(geometry), + ST_ZMin(geometry))) + WHERE ST_GeometryType(geometry) = 'ST_PolyhedralSurface'; + + UPDATE catmaid_volume__history + SET geometry = ST_GeomFromEWKT(FORMAT('TINZ ({bb_vertices})', + ST_XMax(geometry), + ST_XMin(geometry), + ST_YMax(geometry), + ST_YMin(geometry), + ST_ZMax(geometry), + ST_ZMin(geometry))) + WHERE ST_GeometryType(geometry) = 'ST_PolyhedralSurface'; + + ALTER TABLE catmaid_volume + ALTER COLUMN geometry TYPE Geometry(TINZ); + + ALTER TABLE catmaid_volume__history + ALTER COLUMN geometry TYPE Geometry(TINZ); + + SELECT create_history_view_for_table('catmaid_volume'::regclass); + SELECT enable_history_tracking_for_table('catmaid_volume'::regclass, + get_history_table_name('catmaid_volume'::regclass), FALSE); +""".format(bb_vertices=bb_vertices) + +backward = """ + SELECT disable_history_tracking_for_table('catmaid_volume'::regclass, + get_history_table_name('catmaid_volume'::regclass)); + SELECT drop_history_view_for_table('catmaid_volume'::regclass); + + ALTER TABLE catmaid_volume + ALTER COLUMN geometry TYPE Geometry(GeometryZ); + + ALTER TABLE catmaid_volume__history + ALTER COLUMN geometry TYPE Geometry(GeometryZ); + + SELECT create_history_view_for_table('catmaid_volume'::regclass); + SELECT enable_history_tracking_for_table('catmaid_volume'::regclass, + get_history_table_name('catmaid_volume'::regclass), FALSE); +""" + +class Migration(migrations.Migration): + + dependencies = [ + ('catmaid', '0048_add_pointcloud_permissions'), + ] + + operations = [ + migrations.RunSQL(forward, backward) + ] From 479271490c9ee77a810510004ba415283d9bc9c0 Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Wed, 18 Jul 2018 11:58:45 -0400 Subject: [PATCH 02/26] Volumes: Import volumes Includes: - URL endpoint - API method - Test fixture - Simple test --- django/applications/catmaid/control/volume.py | 141 +++++++++++++++++- .../catmaid/tests/apis/test_volume.py | 33 ++++ .../catmaid/tests/fixtures/cube.stl | 86 +++++++++++ django/applications/catmaid/urls.py | 1 + 4 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 django/applications/catmaid/tests/fixtures/cube.stl diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 3918a9854b..bd9196afe9 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -1,14 +1,18 @@ # -*- coding: utf-8 -*- import json +import os import re +from xml.etree import ElementTree as ET + +from django.conf import settings from catmaid.control.authentication import requires_user_role, user_can_edit from catmaid.models import UserRole, Project, Volume from catmaid.serializers import VolumeSerializer from django.db import connection -from django.http import JsonResponse +from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest from django.shortcuts import get_object_or_404 from rest_framework.decorators import api_view @@ -205,6 +209,88 @@ def get_params(self): } +def _chunk(iterable, length, fn=None): + if not fn: + fn = lambda x: x + + items = [] + it = iter(iterable) + while True: + try: + items.append(fn(next(it))) + except StopIteration: + if items: + raise ValueError( + "Iterable did not have a multiple of {} items ({} spare)".format(length, len(items)) + ) + else: + return + else: + if len(items) == length: + yield tuple(items) + items = [] + + +def _x3d_to_points(self, fn=None): + + indexed_triangle_set = ET.fromstring(self.x3d) + assert indexed_triangle_set.tag == "IndexedTriangleSet" + assert len(indexed_triangle_set) == 1 + + coordinate = indexed_triangle_set[0] + assert coordinate.tag == "Coordinate" + assert len(coordinate) == 0 + points_str = coordinate.attrib["point"] + + for item in _chunk(points_str.split(' '), 3, fn): + yield item + + +def _x3d_to_stl_ascii(x3d): + solid_fmt = """ +solid +{} +endsolid + """.strip() + facet_fmt = """ +facet normal 0 0 0 +outer loop +{} +endloop +endfacet + """.strip() + vertex_fmt = "vertex {} {} {}" + + triangle_strs = [] + for triangle in _x3d_to_points(x3d): + vertices = '\n'.join(vertex_fmt.format(*point) for point in triangle) + triangle_strs.append(facet_fmt.format(vertices)) + + return solid_fmt.format('\n'.join(triangle_strs)) + + +VERTEX_RE = re.compile(r"\bvertex\s+(?P[-+]?\d*\.?\d+([eE][-+]?\d+)?)\s+(?P[-+]?\d*\.?\d+([eE][-+]?\d+)?)\s+(?P[-+]?\d*\.?\d+([eE][-+]?\d+)?)\b", re.MULTILINE) + + +def _stl_ascii_to_vertices(stl_str): + for match in VERTEX_RE.finditer(stl_str): + d = match.groupdict() + yield [float(d[dim]) for dim in 'xyz'] + + +def _stl_ascii_to_indexed_triangles(stl_str): + vertices = [] + triangles = [] + for triangle in _chunk(_stl_ascii_to_vertices(stl_str), 3): + this_triangle = [] + for vertex in triangle: + this_triangle.append(len(vertices)) + vertices.append(vertex) + triangles.append(this_triangle) + + return vertices, triangles + + volume_type = { "box": BoxVolume, "trimesh": TriangleMeshVolume @@ -454,6 +540,59 @@ def add_volume(request, project_id): "volume_id": volume_id }) + +@api_view(['POST']) +@requires_user_role([UserRole.Import]) +def import_volumes(request, project_id): + """Import a neuron modeled by a skeleton from an uploaded file. + + Currently only SWC representation is supported. + --- + consumes: multipart/form-data + parameters: + - name: file + required: true + description: A skeleton representation file to import. + paramType: body + dataType: File + type: + neuron_id: + type: integer + required: true + description: ID of the neuron used or created. + skeleton_id: + type: integer + required: true + description: ID of the imported skeleton. + node_id_map: + required: true + description: > + An object whose properties are node IDs in the import file and + whose values are IDs of the created nodes. + """ + fnames_to_id = dict() + for uploadedfile in request.FILES.values(): + if uploadedfile.size > settings.IMPORTED_SKELETON_FILE_MAXIMUM_SIZE: # todo: use different setting + return HttpResponse( + 'File too large. Maximum file size is {} bytes.'.format(settings.IMPORTED_SKELETON_FILE_MAXIMUM_SIZE), + status=413) + + filename = uploadedfile.name + name, extension = os.path.splitext(filename) + if extension.lower() == ".stl": + stl_str = uploadedfile.read().decode('utf-8') + vertices, triangles = _stl_ascii_to_indexed_triangles(stl_str) + mesh = TriangleMeshVolume( + project_id, request.user.id, + {"type": "trimesh", "title": name, "mesh": [vertices, triangles]} + ) + fnames_to_id[filename] = mesh.save() + else: + return HttpResponse('File type "{}" not understood. Known file types: stl'.format(extension), status=415) + + return JsonResponse(fnames_to_id) + + @api_view(['GET']) @requires_user_role([UserRole.Browse]) def intersects(request, project_id, volume_id): diff --git a/django/applications/catmaid/tests/apis/test_volume.py b/django/applications/catmaid/tests/apis/test_volume.py index 1318093ad2..9b57bfefc2 100644 --- a/django/applications/catmaid/tests/apis/test_volume.py +++ b/django/applications/catmaid/tests/apis/test_volume.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import json +import os + import pytz import datetime @@ -8,12 +10,17 @@ from dateutil.tz import tzutc from django.db import connection +from guardian.shortcuts import assign_perm from .common import CatmaidApiTestCase from catmaid.models import Volume from catmaid.control.volume import BoxVolume +FIXTURE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'fixtures') +CUBE_PATH = os.path.join(FIXTURE_DIR, 'cube.stl') + + class VolumeTests(CatmaidApiTestCase): def setUp(self): @@ -109,3 +116,29 @@ def test_volume_edit_comment_only(self): self.assertEqual(row[5], self.test_vol_1_data['name']) self.assertEqual(row[6], 'New comment') self.assertEqual(row[7], self.test_vol_1_data['geometry']) + + def test_import_trimesh_from_stl(self): + self.fake_authentication() + assign_perm('can_import', self.test_user, self.test_project) + with open(CUBE_PATH, 'rb') as f: + response = self.client.post( + "/{}/volumes/import".format(self.test_project_id), + {"cube.stl": f} + ) + + self.assertEqual(response.status_code, 200) + parsed_response = json.loads(response.content.decode('utf-8')) + self.assertEqual(len(parsed_response), 1) + self.assertTrue("cube.stl" in parsed_response) + + cube_id = parsed_response["cube.stl"] + + response = self.client.get("/{}/volumes/{}/".format(self.test_project_id, cube_id)) + + self.assertEqual(response.status_code, 200) + parsed_response = json.loads(response.content.decode('utf-8')) + self.assertEqual(parsed_response['name'], 'cube') + self.assertEqual(parsed_response['bbox'], { + 'min': {'x': 0, 'y': 0, 'z': 0}, + 'max': {'x': 1, 'y': 1, 'z': 1} + }) diff --git a/django/applications/catmaid/tests/fixtures/cube.stl b/django/applications/catmaid/tests/fixtures/cube.stl new file mode 100644 index 0000000000..b9904a3c7e --- /dev/null +++ b/django/applications/catmaid/tests/fixtures/cube.stl @@ -0,0 +1,86 @@ +solid cube + facet normal 0 0 0 + outer loop + vertex 0 0 0 + vertex 0 1 0 + vertex 1 1 0 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 0 0 + vertex 1 1 0 + vertex 1 0 0 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 0 0 + vertex 0 0 1 + vertex 0 1 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 0 0 + vertex 0 1 1 + vertex 0 1 0 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 0 0 + vertex 1 0 0 + vertex 1 0 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 0 0 + vertex 1 0 1 + vertex 0 0 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 0 1 + vertex 1 0 1 + vertex 1 1 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 0 1 + vertex 1 1 1 + vertex 0 1 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 1 0 0 + vertex 1 1 0 + vertex 1 1 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 1 0 0 + vertex 1 1 1 + vertex 1 0 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 1 0 + vertex 0 1 1 + vertex 1 1 1 + endloop + endfacet + facet normal 0 0 0 + outer loop + vertex 0 1 0 + vertex 1 1 1 + vertex 1 1 0 + endloop + endfacet +endsolid cube diff --git a/django/applications/catmaid/urls.py b/django/applications/catmaid/urls.py index 1d1d4cfa5e..cc1a789528 100644 --- a/django/applications/catmaid/urls.py +++ b/django/applications/catmaid/urls.py @@ -494,6 +494,7 @@ urlpatterns += [ url(r'^(?P\d+)/volumes/$', volume.volume_collection), url(r'^(?P\d+)/volumes/add$', record_view("volumes.create")(volume.add_volume)), + url(r'^(?P\d+)/volumes/import$', volume.import_volumes), url(r'^(?P\d+)/volumes/(?P\d+)/$', volume.volume_detail), url(r'^(?P\d+)/volumes/(?P\d+)/intersect$', volume.intersects), ] From e98163d87b99e498b9e6b0c44a5a9e9b8a8fde5d Mon Sep 17 00:00:00 2001 From: willp24 Date: Wed, 18 Jul 2018 14:48:16 -0400 Subject: [PATCH 03/26] Volume Export --- django/applications/catmaid/control/volume.py | 16 +++++++++++++++ .../catmaid/tests/apis/test_volume.py | 20 +++++++++++++++++++ django/applications/catmaid/urls.py | 1 + 3 files changed, 37 insertions(+) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index bd9196afe9..81f4f8ce64 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -593,6 +593,22 @@ def import_volumes(request, project_id): return JsonResponse(fnames_to_id) +@api_view(['GET']) +@requires_user_role([UserRole.Browse]) +def export_volume(request, project_id, volume_id): + acceptable = ['model/stl', 'model/x.stl-ascii'] + file_type = request.META['HTTP_ACCEPT'] + if file_type in acceptable: + details = get_volume_details(project_id, volume_id) + ascii_details = _x3d_to_stl_ascii(details['mesh']) + response = HttpResponse(content_type=file_type) + response.write(ascii_details) + return response + else: + return HttpResponse('File type "{}" not understood. Known file types: {}'.format(file_type, ', '.join(acceptable)), status=415) + + + @api_view(['GET']) @requires_user_role([UserRole.Browse]) def intersects(request, project_id, volume_id): diff --git a/django/applications/catmaid/tests/apis/test_volume.py b/django/applications/catmaid/tests/apis/test_volume.py index 9b57bfefc2..268c906a57 100644 --- a/django/applications/catmaid/tests/apis/test_volume.py +++ b/django/applications/catmaid/tests/apis/test_volume.py @@ -142,3 +142,23 @@ def test_import_trimesh_from_stl(self): 'min': {'x': 0, 'y': 0, 'z': 0}, 'max': {'x': 1, 'y': 1, 'z': 1} }) + + def test_export_stl(self): + self.fake_authentication() + assign_perm('can_import', self.test_user, self.test_project) + with open(CUBE_PATH, 'rb') as f: + response = self.client.post( + "/{}/volumes/import".format(self.test_project_id), + {"cube.stl": f} + ) + + self.assertEqual(response.status_code, 200) + parsed_response = json.loads(response.content.decode('utf-8')) + self.assertEqual(len(parsed_response), 1) + self.assertTrue("cube.stl" in parsed_response) + + cube_id = parsed_response["cube.stl"] + + response = self.client.get("/{}/volumes/{}/export".format(self.test_project_id, cube_id), accept="model/x.stl-ascii") + + self.assertEqual(response.status_code, 200) diff --git a/django/applications/catmaid/urls.py b/django/applications/catmaid/urls.py index cc1a789528..d30b3e016c 100644 --- a/django/applications/catmaid/urls.py +++ b/django/applications/catmaid/urls.py @@ -497,6 +497,7 @@ url(r'^(?P\d+)/volumes/import$', volume.import_volumes), url(r'^(?P\d+)/volumes/(?P\d+)/$', volume.volume_detail), url(r'^(?P\d+)/volumes/(?P\d+)/intersect$', volume.intersects), + url(r'^(?P\d+)/volumes/(?P\d+)/export', volume.export_volume), ] # Analytics From 9db0060a6d6d6a4731f5ead0d1fd6c45f569e7dc Mon Sep 17 00:00:00 2001 From: willp24 Date: Wed, 18 Jul 2018 15:41:14 -0400 Subject: [PATCH 04/26] Included stl file format for uploading volumes --- .../catmaid/static/js/widgets/volumewidget.js | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index 8bb54db73c..b5b7d27780 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -67,12 +67,20 @@ if (0 === files.length) { CATMAID.error("Choose at least one file!"); } else { - Array.from(files).forEach(this.addVolumeFromFile); + this.addVolumesFromSTL(Array.from(files).filter(function(file){ + if (file.name.endsWith("stl")){ + return true; + } else { + this.addVolumeFromFile(file); + } + },this)); } }).bind(this)); + hiddenFileButton.setAttribute('multiple', true); controls.appendChild(hiddenFileButton); var openFile = document.createElement('button'); + openFile.setAttribute('title','Supports Json and ascii-stl files'); openFile.appendChild(document.createTextNode('Add from file')); openFile.onclick = hiddenFileButton.click.bind(hiddenFileButton); controls.appendChild(openFile); @@ -564,6 +572,21 @@ reader.readAsText(file); }; + VolumeManagerWidget.prototype.addVolumesFromSTL = function(files) { + var self = this; + var data = new FormData(); + files.forEach(function(file){ + data.append(file.name, file, file.name) + }); + return new Promise(function(resolve, reject) { + CATMAID.fetch(project.id + "/volumes/import", "POST", data, undefined, undefined, undefined, undefined, {"Content-type" : null}) + .then(function(data){ + self.redraw(); + }) + .catch(CATMAID.handleError); + }); + } + /** * Add a new volume. Edit it its properties directly in the widget. */ From e666c1393864d4227a123a48c2ceb33e7369d8e2 Mon Sep 17 00:00:00 2001 From: willp24 Date: Wed, 18 Jul 2018 16:38:55 -0400 Subject: [PATCH 05/26] Volume widget: add STL export button for volumes --- .../catmaid/static/js/widgets/volumewidget.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index b5b7d27780..7c57ab6263 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -183,7 +183,8 @@ orderable: false, defaultContent: 'Remove ' + 'List skeletons ' + - 'List connectors' + 'List connectors' + + 'Export STL' } ], }); @@ -302,6 +303,21 @@ return false; }); + // Connector intersection list + $(table).on('click', 'a[data-action="export-STL"]', function() { + var tr = $(this).closest("tr"); + var volume = self.datatable.row(tr).data(); + CATMAID.fetch("/" + project.id + "/volumes/" + volume.id + "/export", "GET", undefined, true) + .then(function(volume_file) { + var blob = new Blob([volume_file], {type: 'model/x.stl-ascii'}) + saveAs(blob, volume.name + '.stl'); + }) + .catch(CATMAID.handleError); + + // Prevent event from bubbling up. + return false; + }); + // Display a volume if clicked var self = this; $(table).on('click', 'tbody td', function() { From 042f34ae85b778700670a532f3b2f915823f1cd6 Mon Sep 17 00:00:00 2001 From: willp24 Date: Thu, 19 Jul 2018 10:35:18 -0400 Subject: [PATCH 06/26] Some semicolons --- .../catmaid/static/js/widgets/volumewidget.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index 7c57ab6263..20d2e8f3b1 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -307,9 +307,9 @@ $(table).on('click', 'a[data-action="export-STL"]', function() { var tr = $(this).closest("tr"); var volume = self.datatable.row(tr).data(); - CATMAID.fetch("/" + project.id + "/volumes/" + volume.id + "/export", "GET", undefined, true) + CATMAID.fetch("/" + project.id + "/volumes/" + volume.id + "/export", "GET", undefined, true, undefined, undefined, 'model/x.stl-ascii') .then(function(volume_file) { - var blob = new Blob([volume_file], {type: 'model/x.stl-ascii'}) + var blob = new Blob([volume_file], {type: 'model/x.stl-ascii'}); saveAs(blob, volume.name + '.stl'); }) .catch(CATMAID.handleError); @@ -592,7 +592,7 @@ var self = this; var data = new FormData(); files.forEach(function(file){ - data.append(file.name, file, file.name) + data.append(file.name, file, file.name); }); return new Promise(function(resolve, reject) { CATMAID.fetch(project.id + "/volumes/import", "POST", data, undefined, undefined, undefined, undefined, {"Content-type" : null}) @@ -601,7 +601,7 @@ }) .catch(CATMAID.handleError); }); - } + }; /** * Add a new volume. Edit it its properties directly in the widget. From 9d168aad60445fc784765175070a65fb61975cbe Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Thu, 19 Jul 2018 10:47:29 -0400 Subject: [PATCH 07/26] Volumes: use extension and accept header for export When exporting, use both the extension of the URL and the HTTP ACCEPT header to determine which format to return. This supports both extensions with multiple formats (e.g., ASCII and binary STL) and extensions without a distinctive media type (e.g., OBJ is `text/plain`). --- django/applications/catmaid/control/volume.py | 28 +++++++++++-------- .../catmaid/tests/apis/test_volume.py | 4 ++- django/applications/catmaid/urls.py | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 81f4f8ce64..9fa49aba8a 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -595,18 +595,24 @@ def import_volumes(request, project_id): @api_view(['GET']) @requires_user_role([UserRole.Browse]) -def export_volume(request, project_id, volume_id): - acceptable = ['model/stl', 'model/x.stl-ascii'] - file_type = request.META['HTTP_ACCEPT'] - if file_type in acceptable: - details = get_volume_details(project_id, volume_id) - ascii_details = _x3d_to_stl_ascii(details['mesh']) - response = HttpResponse(content_type=file_type) - response.write(ascii_details) - return response +def export_volume(request, project_id, volume_id, extension): + acceptable = { + 'stl': ['model/stl', 'model/x.stl-ascii'], + } + if extension.lower() in acceptable: + media_type = request.META['HTTP_ACCEPT'] + if media_type in acceptable[extension]: + details = get_volume_details(project_id, volume_id) + ascii_details = _x3d_to_stl_ascii(details['mesh']) + response = HttpResponse(content_type=media_type) + response.write(ascii_details) + return response + else: + return HttpResponse('Media type "{}" not understood. Known types for {}: {}'.format( + media_type, extension, ', '.join(acceptable[extension])), status=415) else: - return HttpResponse('File type "{}" not understood. Known file types: {}'.format(file_type, ', '.join(acceptable)), status=415) - + return HttpResponse('File type "{}" not understood. Known file types: {}'.format( + extension, ', '.join(acceptable.values())), status=415) @api_view(['GET']) diff --git a/django/applications/catmaid/tests/apis/test_volume.py b/django/applications/catmaid/tests/apis/test_volume.py index 268c906a57..7f4c016dfd 100644 --- a/django/applications/catmaid/tests/apis/test_volume.py +++ b/django/applications/catmaid/tests/apis/test_volume.py @@ -159,6 +159,8 @@ def test_export_stl(self): cube_id = parsed_response["cube.stl"] - response = self.client.get("/{}/volumes/{}/export".format(self.test_project_id, cube_id), accept="model/x.stl-ascii") + response = self.client.get( + "/{}/volumes/{}/export.stl".format(self.test_project_id, cube_id), + accept="model/x.stl-ascii") self.assertEqual(response.status_code, 200) diff --git a/django/applications/catmaid/urls.py b/django/applications/catmaid/urls.py index d30b3e016c..3cf6100240 100644 --- a/django/applications/catmaid/urls.py +++ b/django/applications/catmaid/urls.py @@ -497,7 +497,7 @@ url(r'^(?P\d+)/volumes/import$', volume.import_volumes), url(r'^(?P\d+)/volumes/(?P\d+)/$', volume.volume_detail), url(r'^(?P\d+)/volumes/(?P\d+)/intersect$', volume.intersects), - url(r'^(?P\d+)/volumes/(?P\d+)/export', volume.export_volume), + url(r'^(?P\d+)/volumes/(?P\d+)/export\.(?P\w+)', volume.export_volume), ] # Analytics From a9d51db7f0c34ce3c37f6d18dcbd55e59b0ba161 Mon Sep 17 00:00:00 2001 From: willp24 Date: Thu, 19 Jul 2018 11:01:15 -0400 Subject: [PATCH 08/26] Add STL headers and export URL format --- django/applications/catmaid/static/js/widgets/volumewidget.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index 20d2e8f3b1..e6b2171901 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -307,7 +307,8 @@ $(table).on('click', 'a[data-action="export-STL"]', function() { var tr = $(this).closest("tr"); var volume = self.datatable.row(tr).data(); - CATMAID.fetch("/" + project.id + "/volumes/" + volume.id + "/export", "GET", undefined, true, undefined, undefined, 'model/x.stl-ascii') + var headers = {Accept: ['model/x.stl-ascii', 'model/stl']}; + CATMAID.fetch("/" + project.id + "/volumes/" + volume.id + "/export.stl", "GET", undefined, true, undefined, undefined, undefined, headers) .then(function(volume_file) { var blob = new Blob([volume_file], {type: 'model/x.stl-ascii'}); saveAs(blob, volume.name + '.stl'); From afb434dca7ed09b71c090f110b0f36f27be0c1d4 Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Thu, 19 Jul 2018 11:38:39 -0400 Subject: [PATCH 09/26] Volumes: document import/export API endpoints --- django/applications/catmaid/control/volume.py | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 9fa49aba8a..5cdda1dfa7 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -544,32 +544,25 @@ def add_volume(request, project_id): @api_view(['POST']) @requires_user_role([UserRole.Import]) def import_volumes(request, project_id): - """Import a neuron modeled by a skeleton from an uploaded file. - - Currently only SWC representation is supported. - --- - consumes: multipart/form-data - parameters: - - name: file - required: true - description: A skeleton representation file to import. - paramType: body - dataType: File - type: - neuron_id: - type: integer - required: true - description: ID of the neuron used or created. - skeleton_id: - type: integer - required: true - description: ID of the imported skeleton. - node_id_map: - required: true - description: > - An object whose properties are node IDs in the import file and - whose values are IDs of the created nodes. - """ + """Import triangle mesh volumes from an uploaded files. + + Currently only STL representation is supported. + --- + consumes: multipart/form-data + parameters: + - name: file + required: true + description: > + Triangle mesh file to import. Multiple files can be provided, with + each being imported as a mesh named by its base filename. + paramType: body + dataType: File + type: + '{base_filename}': + description: ID of the volume created from this file + type: integer + required: true + """ fnames_to_id = dict() for uploadedfile in request.FILES.values(): if uploadedfile.size > settings.IMPORTED_SKELETON_FILE_MAXIMUM_SIZE: # todo: use different setting @@ -596,6 +589,16 @@ def import_volumes(request, project_id): @api_view(['GET']) @requires_user_role([UserRole.Browse]) def export_volume(request, project_id, volume_id, extension): + """Export volume as a triangle mesh file. + + The extension of the endpoint and `ACCEPT` header media type are both used + to determine the format of the export. + + Supported formats by extension and media type: + ##### STL + - `model/stl`, `model/x.stl-ascii`: ASCII STL + + """ acceptable = { 'stl': ['model/stl', 'model/x.stl-ascii'], } From 309c040cb1b15b65aeee9424f950fa031abb9cdf Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Thu, 19 Jul 2018 13:18:44 -0400 Subject: [PATCH 10/26] Volumes: fix STL export - Bypass DRF response accent content negotiation - Fix chunking of triangles in STL serialization - Fix accept header in tests --- django/applications/catmaid/control/volume.py | 24 ++++++++++++++----- .../catmaid/tests/apis/test_volume.py | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 5cdda1dfa7..10d4183850 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -15,7 +15,8 @@ from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest from django.shortcuts import get_object_or_404 -from rest_framework.decorators import api_view +from rest_framework import renderers +from rest_framework.decorators import api_view, renderer_classes from rest_framework.response import Response from six.moves import map @@ -231,9 +232,8 @@ def _chunk(iterable, length, fn=None): items = [] -def _x3d_to_points(self, fn=None): - - indexed_triangle_set = ET.fromstring(self.x3d) +def _x3d_to_points(x3d, fn=None): + indexed_triangle_set = ET.fromstring(x3d) assert indexed_triangle_set.tag == "IndexedTriangleSet" assert len(indexed_triangle_set) == 1 @@ -262,7 +262,7 @@ def _x3d_to_stl_ascii(x3d): vertex_fmt = "vertex {} {} {}" triangle_strs = [] - for triangle in _x3d_to_points(x3d): + for triangle in _chunk(_x3d_to_points(x3d), 3): vertices = '\n'.join(vertex_fmt.format(*point) for point in triangle) triangle_strs.append(facet_fmt.format(vertices)) @@ -586,7 +586,19 @@ def import_volumes(request, project_id): return JsonResponse(fnames_to_id) +class AnyRenderer(renderers.BaseRenderer): + """A DRF renderer that returns the data directly with a wildcard media type. + + This is useful for bypassing response content type negotiation. + """ + media_type = '*/*' + + def render(self, data, media_type=None, renderer_context=None): + return data + + @api_view(['GET']) +@renderer_classes((AnyRenderer,)) @requires_user_role([UserRole.Browse]) def export_volume(request, project_id, volume_id, extension): """Export volume as a triangle mesh file. @@ -603,7 +615,7 @@ def export_volume(request, project_id, volume_id, extension): 'stl': ['model/stl', 'model/x.stl-ascii'], } if extension.lower() in acceptable: - media_type = request.META['HTTP_ACCEPT'] + media_type = request.META.get('HTTP_ACCEPT') if media_type in acceptable[extension]: details = get_volume_details(project_id, volume_id) ascii_details = _x3d_to_stl_ascii(details['mesh']) diff --git a/django/applications/catmaid/tests/apis/test_volume.py b/django/applications/catmaid/tests/apis/test_volume.py index 7f4c016dfd..1d85c05f56 100644 --- a/django/applications/catmaid/tests/apis/test_volume.py +++ b/django/applications/catmaid/tests/apis/test_volume.py @@ -161,6 +161,6 @@ def test_export_stl(self): response = self.client.get( "/{}/volumes/{}/export.stl".format(self.test_project_id, cube_id), - accept="model/x.stl-ascii") + HTTP_ACCEPT="model/x.stl-ascii") self.assertEqual(response.status_code, 200) From f3642c98fa0ab7b1d32b23c1ab87ba777c05d7aa Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Thu, 19 Jul 2018 13:30:00 -0400 Subject: [PATCH 11/26] Volumes: support multiple accept types in export --- django/applications/catmaid/control/volume.py | 20 +++++++++---------- .../catmaid/tests/apis/test_volume.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 10d4183850..8401bfe592 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -615,16 +615,16 @@ def export_volume(request, project_id, volume_id, extension): 'stl': ['model/stl', 'model/x.stl-ascii'], } if extension.lower() in acceptable: - media_type = request.META.get('HTTP_ACCEPT') - if media_type in acceptable[extension]: - details = get_volume_details(project_id, volume_id) - ascii_details = _x3d_to_stl_ascii(details['mesh']) - response = HttpResponse(content_type=media_type) - response.write(ascii_details) - return response - else: - return HttpResponse('Media type "{}" not understood. Known types for {}: {}'.format( - media_type, extension, ', '.join(acceptable[extension])), status=415) + media_types = request.META.get('HTTP_ACCEPT', '').split(',') + for media_type in media_types: + if media_type in acceptable[extension]: + details = get_volume_details(project_id, volume_id) + ascii_details = _x3d_to_stl_ascii(details['mesh']) + response = HttpResponse(content_type=media_type) + response.write(ascii_details) + return response + return HttpResponse('Media types "{}" not understood. Known types for {}: {}'.format( + ', '.join(media_types), extension, ', '.join(acceptable[extension])), status=415) else: return HttpResponse('File type "{}" not understood. Known file types: {}'.format( extension, ', '.join(acceptable.values())), status=415) diff --git a/django/applications/catmaid/tests/apis/test_volume.py b/django/applications/catmaid/tests/apis/test_volume.py index 1d85c05f56..7ff5d54655 100644 --- a/django/applications/catmaid/tests/apis/test_volume.py +++ b/django/applications/catmaid/tests/apis/test_volume.py @@ -161,6 +161,6 @@ def test_export_stl(self): response = self.client.get( "/{}/volumes/{}/export.stl".format(self.test_project_id, cube_id), - HTTP_ACCEPT="model/x.stl-ascii") + HTTP_ACCEPT="model/x.stl-ascii,model/stl") self.assertEqual(response.status_code, 200) From c7a6063879b2cff6114b174267ceb4875703fc94 Mon Sep 17 00:00:00 2001 From: willp24 Date: Thu, 19 Jul 2018 14:07:41 -0400 Subject: [PATCH 12/26] Volumes: error handling on volume import --- django/applications/catmaid/static/js/widgets/volumewidget.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index e6b2171901..067e28e1e6 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -73,7 +73,7 @@ } else { this.addVolumeFromFile(file); } - },this)); + },this)).catch(CATMAID.handleError); } }).bind(this)); hiddenFileButton.setAttribute('multiple', true); From 8900fae1b20b5ad70d994e721cd103ff3c606c4e Mon Sep 17 00:00:00 2001 From: Chris Barnes Date: Thu, 19 Jul 2018 15:17:39 -0400 Subject: [PATCH 13/26] Volume: better STL parser --- django/applications/catmaid/control/volume.py | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 8401bfe592..c877d5e373 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -269,23 +269,35 @@ def _x3d_to_stl_ascii(x3d): return solid_fmt.format('\n'.join(triangle_strs)) -VERTEX_RE = re.compile(r"\bvertex\s+(?P[-+]?\d*\.?\d+([eE][-+]?\d+)?)\s+(?P[-+]?\d*\.?\d+([eE][-+]?\d+)?)\s+(?P[-+]?\d*\.?\d+([eE][-+]?\d+)?)\b", re.MULTILINE) - - -def _stl_ascii_to_vertices(stl_str): - for match in VERTEX_RE.finditer(stl_str): - d = match.groupdict() - yield [float(d[dim]) for dim in 'xyz'] +class InvalidSTLError(ValueError): + pass def _stl_ascii_to_indexed_triangles(stl_str): + stl_items = stl_str.strip().split() + if stl_items[0] != "solid" or "endsolid" not in stl_items[-2:]: + raise InvalidSTLError("Malformed solid header/ footer") + start = 1 if stl_items[1] == "facet" else 2 + stop = -1 if stl_items[-2] == "endfacet" else -2 vertices = [] triangles = [] - for triangle in _chunk(_stl_ascii_to_vertices(stl_str), 3): + for facet in _chunk(stl_items[start:stop], 21): + if any([ + facet[:2] != ("facet", "normal"), + facet[5:7] != ("outer", "loop"), + facet[-2:] != ("endloop", "endfacet") + ]): + raise InvalidSTLError("Malformed facet/loop header/footer") + this_triangle = [] - for vertex in triangle: - this_triangle.append(len(vertices)) - vertices.append(vertex) + for vertex in _chunk(facet[7:-2], 4): + if vertex[0] != "vertex": + raise InvalidSTLError("Malformed vertex") + vertex_id = len(vertices) + vertices.append([float(item) for item in vertex[1:]]) + this_triangle.append(vertex_id) + if len(this_triangle) != 3: + raise InvalidSTLError("Expected triangle, got {} points".format(this_triangle)) triangles.append(this_triangle) return vertices, triangles @@ -574,7 +586,12 @@ def import_volumes(request, project_id): name, extension = os.path.splitext(filename) if extension.lower() == ".stl": stl_str = uploadedfile.read().decode('utf-8') - vertices, triangles = _stl_ascii_to_indexed_triangles(stl_str) + + try: + vertices, triangles = _stl_ascii_to_indexed_triangles(stl_str) + except InvalidSTLError as e: + raise ValueError("Invalid STL file ({})".format(str(e))) + mesh = TriangleMeshVolume( project_id, request.user.id, {"type": "trimesh", "title": name, "mesh": [vertices, triangles]} From bdb41dce721a131626ee59f2f44bc0b1f10d2b52 Mon Sep 17 00:00:00 2001 From: willp24 Date: Thu, 19 Jul 2018 16:46:03 -0400 Subject: [PATCH 14/26] Volumes: Added a message on successful importing of meshes --- django/applications/catmaid/static/js/widgets/volumewidget.js | 1 + 1 file changed, 1 insertion(+) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index 067e28e1e6..e8ad21edc8 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -598,6 +598,7 @@ return new Promise(function(resolve, reject) { CATMAID.fetch(project.id + "/volumes/import", "POST", data, undefined, undefined, undefined, undefined, {"Content-type" : null}) .then(function(data){ + CATMAID.msg("success", Object.keys(data).length + " mesh(s) loaded"); self.redraw(); }) .catch(CATMAID.handleError); From 3188561db18426856cd729a95bf49cca27c629b1 Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Fri, 20 Jul 2018 14:31:44 -0400 Subject: [PATCH 15/26] Volumes: add volume ontology class Add an ontology class for volumes, create class instances for all existing volumes, and relate volume class instances to volume rows via a `volume_class_instance` table. Also move the `model_of` relation from the needed classes for the tracing tool to the default project needed classes. Written with @tomka, @clbarnes, and @willp24. See catmaid/CATMAID#1765. --- .../applications/catmaid/control/project.py | 4 +- .../applications/catmaid/control/tracing.py | 1 - .../0050_create_volume_class_and_instances.py | 157 ++++++++++++++++++ django/applications/catmaid/models.py | 11 ++ 4 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 django/applications/catmaid/migrations/0050_create_volume_class_and_instances.py diff --git a/django/applications/catmaid/control/project.py b/django/applications/catmaid/control/project.py index c207bdd598..74a77f9948 100644 --- a/django/applications/catmaid/control/project.py +++ b/django/applications/catmaid/control/project.py @@ -23,7 +23,8 @@ 'annotation': "An arbitrary annotation", 'stack_property': 'A property which a stack has', 'landmark': "A particular type of location", - "landmarkgroup": "A type of collection that groups landmarks" + "landmarkgroup": "A type of collection that groups landmarks", + 'volume': 'A region of space' } # All relations needed by the tracing system alongside their @@ -34,6 +35,7 @@ 'annotated_with': "Something is annotated by something else.", 'has_property': 'A thing which has an arbitrary property', 'close_to': 'Something is spatially in the neighborhood of something else', + 'model_of': "Marks something as a model of something else." } # All client datastores needed by the tracing system along their descriptions. diff --git a/django/applications/catmaid/control/tracing.py b/django/applications/catmaid/control/tracing.py index 943367b7d1..2762770099 100644 --- a/django/applications/catmaid/control/tracing.py +++ b/django/applications/catmaid/control/tracing.py @@ -28,7 +28,6 @@ needed_relations = { 'labeled_as': "Something is labeled by sth. else.", 'element_of': "A generic element-of relationship", - 'model_of': "Marks something as a model of something else.", 'presynaptic_to': "Something is presynaptic to something else.", 'postsynaptic_to': "Something is postsynaptic to something else.", 'abutting': "Two things abut against each other", diff --git a/django/applications/catmaid/migrations/0050_create_volume_class_and_instances.py b/django/applications/catmaid/migrations/0050_create_volume_class_and_instances.py new file mode 100644 index 0000000000..81eb6a1626 --- /dev/null +++ b/django/applications/catmaid/migrations/0050_create_volume_class_and_instances.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.14 on 2018-07-20 14:27 +import logging + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + +from catmaid.apps import get_system_user + + +logger = logging.getLogger(__name__) + + +def forwards(apps, schema_editor): + """Make sure all required class and relations are existing for all + projects. We can't use the regular model classes, but have to get + them through the migration system. + """ + from catmaid.control.project import validate_project_setup + + Class = apps.get_model('catmaid', 'Class') + Project = apps.get_model('catmaid', 'Project') + Relation = apps.get_model('catmaid', 'Relation') + User = apps.get_model('auth', 'User') + ClientDatastore = apps.get_model('catmaid', 'ClientDatastore') + Volume = apps.get_model('catmaid', 'Volume') + + projects = Project.objects.all() + # If there are no projects, don't continue, because there is nothing to + # migrate. + if 0 == len(projects) or 0 == User.objects.count(): + return + + try: + system_user = get_system_user(User) + for p in projects: + validate_project_setup(p.id, system_user.id, True, Class, Relation, ClientDatastore) + except ImproperlyConfigured as e: + if Volume.objects.count() > 0: + logger.warn("Couldn't find a configured system user and will therefore " + "skip a configuration update of all existing projects. This is " + "okay during the initial setup of a CATMAID database. In this " + "case nothing needs to be done. Otherwise, please run " + "`manage.py catmaid_update_project_configuration` manually " + "after this migration call is finished and rerun this migration.") + raise e + +forward_create_relations = """ + CREATE TABLE volume_class_instance ( + volume_id bigint NOT NULL, + class_instance_id integer NOT NULL + ) + INHERITS (relation_instance); + + -- Volume Class Instance constraints + ALTER TABLE ONLY volume_class_instance + ADD CONSTRAINT volume_class_instance_pkey PRIMARY KEY (id); + + ALTER TABLE ONLY volume_class_instance + ADD CONSTRAINT volume_class_instance_sa_id + FOREIGN KEY (volume_id) + REFERENCES catmaid_volume(id) DEFERRABLE INITIALLY DEFERRED; + + ALTER TABLE ONLY volume_class_instance + ADD CONSTRAINT volume_class_instance_id_fkey + FOREIGN KEY (class_instance_id) + REFERENCES class_instance(id) DEFERRABLE INITIALLY DEFERRED; + + WITH new_cis AS ( + INSERT INTO class_instance (user_id, project_id, name, class_id) + SELECT + v.user_id, + v.project_id, + v.name, + c.id + FROM catmaid_volume v + JOIN class c ON (c.project_id = v.project_id AND c.class_name = 'volume') + RETURNING id, user_id, project_id, name + ) + INSERT INTO volume_class_instance (user_id, project_id, relation_id, volume_id, class_instance_id) + SELECT + v.user_id, + v.project_id, + r.id, + v.id, + ci.id + FROM catmaid_volume v + JOIN relation r ON (r.project_id = v.project_id AND r.relation_name = 'model_of') + JOIN new_cis ci ON ( + ci.user_id = v.user_id AND + ci.project_id = v.project_id AND + ci.name = v.name + ); + + -- Create history tables + SELECT create_history_table('volume_class_instance'::regclass, 'edition_time', 'txid'); +""" + +backward_create_relations = """ + SELECT disable_history_tracking_for_table('volume_class_instance'::regclass, + get_history_table_name('volume_class_instance'::regclass)); + SELECT drop_history_table('volume_class_instance'::regclass); + + DROP TABLE volume_class_instance; + + DELETE FROM class_instance_class_instance cici + USING class ca, class cb, class_instance cia, class_instance cib + WHERE (cici.class_instance_a = cia.id + OR cici.class_instance_b = cib.id) + AND cia.class_id = ca.id + AND cib.class_id = cb.id + AND ca.class_name = 'volume' + AND cb.class_name = 'volume'; + + DELETE FROM class_instance + USING class c + WHERE class_instance.class_id = c.id + AND c.class_name = 'volume'; + +""" + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('catmaid', '0049_volume_tin_representation'), + ] + + operations = [ + migrations.RunPython(forwards, migrations.RunPython.noop), + migrations.RunSQL( + forward_create_relations, + backward_create_relations, + [ + migrations.CreateModel( + name='VolumeClassInstance', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_time', models.DateTimeField(default=django.utils.timezone.now)), + ('edition_time', models.DateTimeField(default=django.utils.timezone.now)), + ('class_instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='catmaid.ClassInstance')), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='catmaid.Project')), + ('relation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='catmaid.Relation')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('volume', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='catmaid.Volume')), + ], + options={ + 'db_table': 'volume_class_instance', + }, + ), + ] + ), + ] diff --git a/django/applications/catmaid/models.py b/django/applications/catmaid/models.py index dc3ffd096e..73586bed13 100644 --- a/django/applications/catmaid/models.py +++ b/django/applications/catmaid/models.py @@ -719,6 +719,17 @@ class Volume(UserFocusedModel): geometry = spatial_models.GeometryField(dim=3, srid=0) +class VolumeClassInstance(UserFocusedModel): + # Repeat the columns inherited from 'relation_instance' + relation = models.ForeignKey(Relation, on_delete=models.CASCADE) + # Now new columns: + volume = models.ForeignKey(Volume, on_delete=models.CASCADE) + class_instance = models.ForeignKey(ClassInstance, on_delete=models.CASCADE) + + class Meta: + db_table = "volume_class_instance" + + class RegionOfInterest(UserFocusedModel): # Repeat the columns inherited from 'location' editor = models.ForeignKey(User, on_delete=models.CASCADE, From 80506f9a026a623ddf01d8f70ebf493149cd941d Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Fri, 20 Jul 2018 14:51:38 -0400 Subject: [PATCH 16/26] Volumes: create and remove volume class instances See catmaid/CATMAID#1765. --- django/applications/catmaid/control/volume.py | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index c877d5e373..d09d4cf306 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -108,11 +108,33 @@ def save(self): raise ValueError("Can't create new volume without mesh") cursor.execute(""" - INSERT INTO catmaid_volume (user_id, project_id, editor_id, name, - comment, creation_time, edition_time, geometry) - VALUES (%(uid)s, %(pid)s, %(uid)s, %(t)s, %(c)s, now(), now(), """ + - surface + """) - RETURNING id;""", params) + WITH v AS ( + INSERT INTO catmaid_volume (user_id, project_id, editor_id, name, + comment, creation_time, edition_time, geometry) + VALUES (%(uid)s, %(pid)s, %(uid)s, %(t)s, %(c)s, now(), now(), """ + + surface + """) + RETURNING user_id, project_id, id + ), ci AS ( + INSERT INTO class_instance (user_id, project_id, name, class_id) + SELECT %(uid)s, project_id, %(t)s, id + FROM class + WHERE project_id = %(pid)s AND class_name = 'volume' + RETURNING id + ), r AS ( + SELECT id FROM relation + WHERE project_id = %(pid)s AND relation_name = 'model_of' + ) + INSERT INTO volume_class_instance + (user_id, project_id, relation_id, volume_id, class_instance_id) + SELECT + v.user_id, + v.project_id, + r.id, + v.id, + ci.id + FROM v, ci, r + RETURNING volume_id + """, params) return cursor.fetchone()[0] @@ -405,7 +427,17 @@ def remove_volume(request, project_id, volume_id): raise Exception("You don't have permissions to delete this volume") cursor.execute(""" - DELETE FROM catmaid_volume WHERE id=%s + WITH v AS ( + DELETE FROM catmaid_volume WHERE id=%s RETURNING id + ), vci AS ( + DELETE FROM volume_class_instance + USING v + WHERE volume_id = v.id + RETURNING class_instance_id + ) + DELETE FROM class_instance + USING vci + WHERE id = vci.class_instance_id; """, (volume_id,)) return Response({ From 6f6b643b7c60ba056c75a61665b0c7d236040395 Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Fri, 20 Jul 2018 15:08:02 -0400 Subject: [PATCH 17/26] Tests: fix volume class instance history test --- django/applications/catmaid/tests/test_history_tables.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django/applications/catmaid/tests/test_history_tables.py b/django/applications/catmaid/tests/test_history_tables.py index 960b8a42a5..32e41366df 100644 --- a/django/applications/catmaid/tests/test_history_tables.py +++ b/django/applications/catmaid/tests/test_history_tables.py @@ -94,6 +94,7 @@ def run(self, *args): 'point_set', 'image_data', 'pointcloud_image_data', + 'volume_class_instance', # Non-CATMAID tables 'auth_group', @@ -181,6 +182,7 @@ def run(self, *args): 'treenode__history', 'treenode_class_instance__history', 'treenode_connector__history', + 'volume_class_instance__history', # History tables of versioned non-CATMAID tables 'auth_group__history', From 00f9dd7a6851d4eb6cdde05d815113d885530feb Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Fri, 20 Jul 2018 13:44:50 -0400 Subject: [PATCH 18/26] Volume manager: represent each volume with model class --- .../catmaid/static/js/widgets/volumewidget.js | 22 ++++++++++++++----- .../catmaid/static/libs/catmaid/volumes.js | 8 +++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index e8ad21edc8..43c3751ddd 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -156,23 +156,33 @@ container.appendChild(tableContainer); this.datatable = $(table).DataTable({ lengthMenu: [CATMAID.pageLengthOptions, CATMAID.pageLengthLabels], - ajax: { - url: CATMAID.makeURL(project.id + "/volumes/"), - dataSrc: "" + ajax: function(data, callback, settings) { + + CATMAID.fetch(project.id + "/volumes/") + .then(function(volumeData) { + let volumes = volumeData.map(function(volume) { + return new CATMAID.Volume(volume); + }); + callback({ + draw: data.draw, + data: volumes + }); + }) + .catch(CATMAID.handleError); }, columns: [ - {data: "name"}, + {data: "title"}, {data: "id"}, {data: "comment"}, { - data: "user", + data: "user_id", render: function(data, type, row, meta) { return CATMAID.User.safe_get(data).login; } }, {data: "creation_time"}, { - data: "editor", + data: "editor_id", render: function(data, type, row, meta) { return CATMAID.User.safe_get(data).login; } diff --git a/django/applications/catmaid/static/libs/catmaid/volumes.js b/django/applications/catmaid/static/libs/catmaid/volumes.js index e17195b0ac..ffd2ba6b13 100644 --- a/django/applications/catmaid/static/libs/catmaid/volumes.js +++ b/django/applications/catmaid/static/libs/catmaid/volumes.js @@ -12,6 +12,14 @@ CATMAID.Volume = function(options) { options = options || {}; this.id = options.id || null; + this.project_id = options.project || null; + this.user_id = options.user || null; + this.editor_id = options.editor || null; + this.title = options.name || ''; + this.comment = options.comment || ''; + this.edition_time = options.edition_time || null; + this.creation_time = options.creation_time || null; + this.selected = options.selected || false; }; CATMAID.Volume.prototype = {}; From ae6f07d714d4881a4ea3bb8bf979cc810925724f Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Fri, 20 Jul 2018 14:19:53 -0400 Subject: [PATCH 19/26] Volume manager: add annotate functionality Add front-end and back-end functionality to add annotations to a set of volumes. A new checkbox column has been added to the volume table and an "Annotate" button, that allows to annotate all selected volumes. Worked on together with @aschampion, @clbarnes and @willp24. See catmaid/CATMAID#1765 --- django/applications/catmaid/control/volume.py | 32 +++++++++++ .../catmaid/static/js/widgets/volumewidget.js | 55 ++++++++++++++++++- django/applications/catmaid/urls.py | 1 + 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index d09d4cf306..1a2f79f3c2 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -8,6 +8,7 @@ from django.conf import settings from catmaid.control.authentication import requires_user_role, user_can_edit +from catmaid.control.common import get_request_list from catmaid.models import UserRole, Project, Volume from catmaid.serializers import VolumeSerializer @@ -728,3 +729,34 @@ def intersects(request, project_id, volume_id): return JsonResponse({ 'intersects': result[0] }) + +@api_view(['POST']) +@requires_user_role([UserRole.Browse]) +def get_volume_entities(request, project_id): + """Retrieve a mapping of volume IDs to entity (class instance) IDs. + --- + parameters: + - name: volume_ids + description: A list of volume IDs to map + paramType: query + """ + volume_ids = get_request_list(request.POST, 'volume_ids', map_fn=int) + + cursor = connection.cursor() + cursor.execute(""" + SELECT vci.volume_id, vci.class_instance_id + FROM volume_class_instance vci + JOIN UNNEST(%(volume_ids)s::int[]) volume(id) + ON volume.id = vci.volume_id + WHERE project_id = %(project_id)s + AND relation_id = ( + SELECT id FROM relation + WHERE relation_name = 'model_of' + AND project_id = %(project_id)s + ) + """, { + 'volume_ids': volume_ids, + 'project_id': project_id + }) + + return JsonResponse(dict(cursor.fetchall())) diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index 43c3751ddd..efe517942d 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -85,6 +85,12 @@ openFile.onclick = hiddenFileButton.click.bind(hiddenFileButton); controls.appendChild(openFile); + var annotate = document.createElement('button'); + annotate.appendChild(document.createTextNode('Annotate')); + annotate.setAttribute('title', 'Annotate all selected volumes'); + annotate.onclick = this.annotateSelectedVolumes.bind(this); + controls.appendChild(annotate); + let self = this; CATMAID.DOM.appendNumericField( controls, @@ -144,7 +150,7 @@ table.style.width = "100%"; var header = table.createTHead(); var hrow = header.insertRow(0); - var columns = ['Name', 'Id', 'Comment', 'User', 'Creation time', + var columns = ['', 'Name', 'Id', 'Comment', 'User', 'Creation time', 'Editor', 'Edition time', 'Action']; columns.forEach(function(c) { hrow.insertCell().appendChild(document.createTextNode(c)); @@ -171,6 +177,12 @@ .catch(CATMAID.handleError); }, columns: [ + { + render: function(data, type, row, meta) { + return ''; + } + }, {data: "title"}, {data: "id"}, {data: "comment"}, @@ -197,6 +209,12 @@ 'Export STL' } ], + }) + .on('change', 'input[data-role=select]', function() { + var table = $(this).closest('table'); + var tr = $(this).closest('tr'); + var data = $(table).DataTable().row(tr).data(); + data.selected = this.checked; }); // Remove volume if 'remove' was clicked @@ -331,7 +349,7 @@ // Display a volume if clicked var self = this; - $(table).on('click', 'tbody td', function() { + $(table).on('dblclick', 'tbody td', function() { var tr = $(this).closest("tr"); var volume = self.datatable.row(tr).data(); self.loadVolume(volume.id) @@ -623,6 +641,39 @@ }; + /** + * Annotate all currently selected volumes. + */ + VolumeManagerWidget.prototype.annotateSelectedVolumes = function() { + if (!this.datatable) { + return; + } + + let allVolumes = this.datatable.rows({'search': 'applied' }).data().toArray(); + let selectedVolumeIds = allVolumes.filter(function(v) { + return v.selected; + }).map(function(v) { + return v.id; + }); + + if (selectedVolumeIds.length === 0) { + CATMAID.warn("No volumes selected"); + return; + } + + // Retrieve class instance IDs for volumes + CATMAID.fetch(project.id + '/volumes/entities/', 'POST', { + volume_ids: selectedVolumeIds + }) + .then(function(ciMapping) { + return CATMAID.annotate(Object.values(ciMapping)); + }) + .then(function() { + CATMAID.msg("Success", "Annotations added"); + }) + .catch(CATMAID.handleError); + }; + var getVolumeType = function(volume) { if (volume instanceof CATMAID.AlphaShapeVolume) { return "alphashape"; diff --git a/django/applications/catmaid/urls.py b/django/applications/catmaid/urls.py index 3cf6100240..ef758e1679 100644 --- a/django/applications/catmaid/urls.py +++ b/django/applications/catmaid/urls.py @@ -495,6 +495,7 @@ url(r'^(?P\d+)/volumes/$', volume.volume_collection), url(r'^(?P\d+)/volumes/add$', record_view("volumes.create")(volume.add_volume)), url(r'^(?P\d+)/volumes/import$', volume.import_volumes), + url(r'^(?P\d+)/volumes/entities/$', volume.get_volume_entities), url(r'^(?P\d+)/volumes/(?P\d+)/$', volume.volume_detail), url(r'^(?P\d+)/volumes/(?P\d+)/intersect$', volume.intersects), url(r'^(?P\d+)/volumes/(?P\d+)/export\.(?P\w+)', volume.export_volume), From 410500250c1e242f249ad03fa4f06b862baf3ac3 Mon Sep 17 00:00:00 2001 From: Andrew Champion Date: Fri, 20 Jul 2018 16:52:35 -0400 Subject: [PATCH 20/26] Volumes: list annotations in manager Note that this also changes the response of the volumes listing endpoint from an array of objects to an array of arrays, with a separate `columns` field. --- django/applications/catmaid/control/volume.py | 29 ++++++++++++++++--- .../applications/catmaid/static/js/tools.js | 10 +++++++ .../catmaid/static/js/widgets/volumewidget.js | 16 ++++++++-- .../static/libs/catmaid/models/volumes.js | 6 +++- .../catmaid/static/libs/catmaid/volumes.js | 7 +++-- 5 files changed, 57 insertions(+), 11 deletions(-) diff --git a/django/applications/catmaid/control/volume.py b/django/applications/catmaid/control/volume.py index 1a2f79f3c2..6886e5d9ce 100644 --- a/django/applications/catmaid/control/volume.py +++ b/django/applications/catmaid/control/volume.py @@ -352,10 +352,31 @@ def volume_collection(request, project_id): # FIXME: Parsing our PostGIS geometry with GeoDjango doesn't work # anymore since Django 1.8. Therefore, the geometry fields isn't read. # See: https://github.com/catmaid/CATMAID/issues/1250 - fields = ('id', 'name', 'comment', 'user', 'editor', 'project', - 'creation_time', 'edition_time') - volumes = Volume.objects.filter(project_id=project_id).values(*fields) - return Response(volumes) + + cursor = connection.cursor() + cursor.execute(""" + SELECT v.id, v.name, v.comment, v.user_id, v.editor_id, v.project_id, + v.creation_time, v.edition_time, + JSON_AGG(ann.name) FILTER (WHERE ann.name IS NOT NULL) AS annotations + FROM catmaid_volume v + LEFT JOIN volume_class_instance vci ON vci.volume_id = v.id + LEFT JOIN class_instance_class_instance cici + ON cici.class_instance_a = vci.class_instance_id + LEFT JOIN class_instance ann ON ann.id = cici.class_instance_b + WHERE v.project_id = %(pid)s + AND ( + cici.relation_id IS NULL OR + cici.relation_id = ( + SELECT id FROM relation + WHERE project_id = %(pid)s AND relation_name = 'annotated_with' + ) + ) + GROUP BY v.id + """, {'pid': project_id}) + return JsonResponse({ + 'columns': [r[0] for r in cursor.description], + 'data': cursor.fetchall() + }) def get_volume_details(project_id, volume_id): cursor = connection.cursor() diff --git a/django/applications/catmaid/static/js/tools.js b/django/applications/catmaid/static/js/tools.js index 867807d279..d498bdc840 100644 --- a/django/applications/catmaid/static/js/tools.js +++ b/django/applications/catmaid/static/js/tools.js @@ -822,4 +822,14 @@ CATMAID.tools = CATMAID.tools || {}; return path.substring(start, end); }; + /** + * Create an object from matched arrays of keys and values. + */ + tools.buildObject = function (keys, values) { + return keys.reduce(function (obj, k, i) { + obj[k] = values[i]; + return obj; + }, {}); + }; + })(CATMAID.tools); diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index efe517942d..33d4549ab5 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -150,7 +150,7 @@ table.style.width = "100%"; var header = table.createTHead(); var hrow = header.insertRow(0); - var columns = ['', 'Name', 'Id', 'Comment', 'User', 'Creation time', + var columns = ['', 'Name', 'Id', 'Comment', 'Annotations', 'User', 'Creation time', 'Editor', 'Edition time', 'Action']; columns.forEach(function(c) { hrow.insertCell().appendChild(document.createTextNode(c)); @@ -166,8 +166,8 @@ CATMAID.fetch(project.id + "/volumes/") .then(function(volumeData) { - let volumes = volumeData.map(function(volume) { - return new CATMAID.Volume(volume); + let volumes = volumeData.data.map(function(volume) { + return new CATMAID.Volume(CATMAID.tools.buildObject(volumeData.columns, volume)); }); callback({ draw: data.draw, @@ -186,6 +186,16 @@ {data: "title"}, {data: "id"}, {data: "comment"}, + { + data: "annotations", + render: function (data, type, row, meta) { + if (type === 'display') { + return data.join(', '); + } else { + return data; + } + } + }, { data: "user_id", render: function(data, type, row, meta) { diff --git a/django/applications/catmaid/static/libs/catmaid/models/volumes.js b/django/applications/catmaid/static/libs/catmaid/models/volumes.js index ce3cec04e6..d25004df39 100644 --- a/django/applications/catmaid/static/libs/catmaid/models/volumes.js +++ b/django/applications/catmaid/static/libs/catmaid/models/volumes.js @@ -21,7 +21,11 @@ */ listAll: function(projectId) { var url = projectId + '/volumes/'; - return CATMAID.fetch(url, 'GET'); + return CATMAID.fetch(url, 'GET').then(function (volumes) { + return volumes.data.map(function (vol) { + return CATMAID.tools.buildObject(volumes.columns, vol); + }); + }); }, /** diff --git a/django/applications/catmaid/static/libs/catmaid/volumes.js b/django/applications/catmaid/static/libs/catmaid/volumes.js index ffd2ba6b13..98e1dfe63b 100644 --- a/django/applications/catmaid/static/libs/catmaid/volumes.js +++ b/django/applications/catmaid/static/libs/catmaid/volumes.js @@ -12,14 +12,15 @@ CATMAID.Volume = function(options) { options = options || {}; this.id = options.id || null; - this.project_id = options.project || null; - this.user_id = options.user || null; - this.editor_id = options.editor || null; + this.project_id = options.project_id || null; + this.user_id = options.user_id || null; + this.editor_id = options.editor_id || null; this.title = options.name || ''; this.comment = options.comment || ''; this.edition_time = options.edition_time || null; this.creation_time = options.creation_time || null; this.selected = options.selected || false; + this.annotations = options.annotations || []; }; CATMAID.Volume.prototype = {}; From 7a22f3042c2af232287b407f6014ea2d33cf932a Mon Sep 17 00:00:00 2001 From: willp24 Date: Fri, 20 Jul 2018 17:33:23 -0400 Subject: [PATCH 21/26] Volumes: Standardized the volume selection dropdown Changes made with @tomka see catmaid/CATMAID#1765 --- .../catmaid/static/js/WindowMaker.js | 75 ++++++------------ .../catmaid/static/js/helpers/volumes.js | 77 +++++++++++++++++++ .../catmaid/static/js/widgets/settings.js | 45 +++-------- .../catmaid/static/js/widgets/volumewidget.js | 38 ++++----- .../catmaid/static/libs/catmaid/filter.js | 35 +++------ 5 files changed, 141 insertions(+), 129 deletions(-) create mode 100644 django/applications/catmaid/static/js/helpers/volumes.js diff --git a/django/applications/catmaid/static/js/WindowMaker.js b/django/applications/catmaid/static/js/WindowMaker.js index 17c26317f6..50b52f861a 100644 --- a/django/applications/catmaid/static/js/WindowMaker.js +++ b/django/applications/catmaid/static/js/WindowMaker.js @@ -708,49 +708,6 @@ var WindowMaker = new function() } } - // Update volume list - var initVolumeList = function() { - return CATMAID.Volumes.listAll(project.id).then(function(json) { - var volumes = json.sort(function(a, b) { - return CATMAID.tools.compareStrings(a.name, b.name); - }).map(function(volume) { - return { - title: volume.name, - value: volume.id - }; - }); - var selectedVolumes = WA.getLoadedVolumeIds(); - // Create actual element based on the returned data - var node = DOM.createCheckboxSelect('Volumes', volumes, - selectedVolumes, true, function(row, id, visible) { - let loadedVolume = WA.loadedVolumes.get(id); - let faces, color, alpha, subdiv, bb; - if (loadedVolume) { - faces = loadedVolume.faces; - color = loadedVolume.color; - alpha = loadedVolume.opacity; - subdiv = loadedVolume.subdiv; - bb = loadedVolume.boundingBox; - } - setVolumeEntryVisible(row, id, visible, faces, color, alpha, subdiv, bb); - }); - - // Add a selection handler - node.onchange = function(e) { - var visible = e.target.checked; - var volumeId = parseInt(e.target.value, 10); - WA.showVolume(volumeId, visible, undefined, undefined, - o.meshes_faces) - .catch(CATMAID.handleError); - - setVolumeEntryVisible(e.target.closest('li'), volumeId, visible, - o.meshes_faces, o.meshes_color, o.meshes_opacity, - o.meshes_subdiv, o.meshes_boundingbox); - }; - return node; - }); - }; - // Update landmark list var initLandmarkList = function() { return CATMAID.Landmarks.listGroups(project.id).then(function(json) { @@ -859,9 +816,29 @@ var WindowMaker = new function() // Create async selection and wrap it in container to have handle on initial // DOM location - var volumeSelection = DOM.createAsyncPlaceholder(initVolumeList()); - var volumeSelectionWrapper = document.createElement('span'); - volumeSelectionWrapper.appendChild(volumeSelection); + var volumeSelectionWrapper = CATMAID.createVolumeSelector({ + mode: "checkbox", + selectedVolumeIds: WA.getLoadedVolumeIds(), + select: function(volumeId, visible, element){ + WA.showVolume(volumeId, visible, undefined, undefined, o.meshes_faces) + .catch(CATMAID.handleError); + + setVolumeEntryVisible(element.closest('li'), volumeId, visible, + o.meshes_faces, o.meshes_color, o.meshes_opacity, + o.meshes_subdiv, o.meshes_boundingbox); + }, + rowCallback: function(row, id, visible) { + let loadedVolume = WA.loadedVolumes.get(id); + let faces, color, alpha, subdiv, bb; + if (loadedVolume) { + faces = loadedVolume.faces; + color = loadedVolume.color; + alpha = loadedVolume.opacity; + subdiv = loadedVolume.subdiv; + bb = loadedVolume.boundingBox; + } + } + }); // Create async selection and wrap it in container to have handle on initial // DOM location @@ -877,11 +854,7 @@ var WindowMaker = new function() // Replace volume selection wrapper children with new select var refreshVolumeList = function() { - while (0 !== volumeSelectionWrapper.children.length) { - volumeSelectionWrapper.removeChild(volumeSelectionWrapper.children[0]); - } - var volumeSelection = DOM.createAsyncPlaceholder(initVolumeList()); - volumeSelectionWrapper.appendChild(volumeSelection); + volumeSelectionWrapper.refresh(WA.getLoadedVolumeIds()); }; // Replace point cloud selection wrapper children with new select diff --git a/django/applications/catmaid/static/js/helpers/volumes.js b/django/applications/catmaid/static/js/helpers/volumes.js new file mode 100644 index 0000000000..7674a7a1d7 --- /dev/null +++ b/django/applications/catmaid/static/js/helpers/volumes.js @@ -0,0 +1,77 @@ +/* -*- mode: espresso; espresso-indent-level: 2; indent-tabs-mode: nil -*- */ +/* vim: set softtabstop=2 shiftwidth=2 tabstop=2 expandtab: */ + +(function (CATMAID) { + + "use strict"; + + // Update volume list + let initVolumeList = function (options, newSelectedIds) { + return CATMAID.Volumes.listAll(project.id).then(function (json) { + let volumes = json.sort(function (a, b) { + return CATMAID.tools.compareStrings(a.name, b.name); + }).map(function (volume) { + return { + title: volume.name, + value: volume.id + }; + }); + if (options.mode === "radio") { + if (newSelectedIds.length > 1){ + throw new CATMAID.ValueError("Radio select only takes one selected volume"); + } + let selectedVolume = newSelectedIds[0] || options.selectedVolumeIds[0]; + // Create actual element based on the returned data + let node = CATMAID.DOM.createRadioSelect('Volumes', volumes, + selectedVolume, true); + // Add a selection handler + node.onchange = function (e) { + let volumeId = e.target.value; + let selected = true; + + if (CATMAID.tools.isFn(options.select)) { + options.select(volumeId, selected, e.target); + } + }; + return node; + } else { + let selectedVolumes = newSelectedIds || options.selectedVolumeIds; + // Create actual element based on the returned data + let node = CATMAID.DOM.createCheckboxSelect('Volumes', volumes, + selectedVolumes, true, options.rowCallback); + + // Add a selection handler + node.onchange = function (e) { + let selected = e.target.checked; + let volumeId = parseInt(e.target.value, 10); + + if (CATMAID.tools.isFn(options.select)) { + options.select(volumeId, selected, e.target); + } + }; + return node; + } + }); + }; + + CATMAID.createVolumeSelector = function (options) { + var volumeSelectionWrapper = document.createElement('span'); + let volumeSelection; + if (options.label){ + volumeSelection = CATMAID.DOM.createLabeledAsyncPlaceholder(options.label, initVolumeList(options), options.title); + } else { + volumeSelection = CATMAID.DOM.createAsyncPlaceholder(initVolumeList(options)); + } + volumeSelectionWrapper.appendChild(volumeSelection); + volumeSelectionWrapper.refresh = function(newSelectedIds){ + while (0 !== volumeSelectionWrapper.children.length) { + volumeSelectionWrapper.removeChild(volumeSelectionWrapper.children[0]); + } + var volumeSelection = CATMAID.DOM.createAsyncPlaceholder(initVolumeList(options, newSelectedIds)); + volumeSelectionWrapper.appendChild(volumeSelection); + }; + return volumeSelectionWrapper; + }; + + +})(CATMAID); \ No newline at end of file diff --git a/django/applications/catmaid/static/js/widgets/settings.js b/django/applications/catmaid/static/js/widgets/settings.js index 28e9e83e27..ffc5cb17f9 100644 --- a/django/applications/catmaid/static/js/widgets/settings.js +++ b/django/applications/catmaid/static/js/widgets/settings.js @@ -1735,42 +1735,19 @@ var dsTracingWarnings = CATMAID.DOM.addSettingsContainer(ds, "Warnings", true); - // Volume warning - let initVolumeList = function() { - return CATMAID.Volumes.listAll(project.id) - .then(function(json) { - var volumes = json.sort(function(a, b) { - return CATMAID.tools.compareStrings(a.name, b.name); - }).map(function(volume) { - return { - title: volume.name + " (#" + volume.id + ")", - value: volume.id - }; - }); - var selectedVolumeId = SkeletonAnnotations.getNewNodeVolumeWarning(); - // Create actual element based on the returned data - var node = CATMAID.DOM.createRadioSelect('Volumes', volumes, - selectedVolumeId, true); - // Add a selection handler - node.onchange = function(e) { - let volumeId = null; - if (e.srcElement.value !== "none") { - volumeId = parseInt(e.srcElement.value, 10); - } - // Remove existing handler and new one if selected - SkeletonAnnotations.setNewNodeVolumeWarning(volumeId); - }; - - return node; - }); - }; - // Create async selection and wrap it in container to have handle on initial // DOM location - var volumeSelectionSetting = CATMAID.DOM.createLabeledAsyncPlaceholder( - "New nodes not in volume", initVolumeList(), - "A warning will be shown when new nodes are created outside of the selected volume"); - dsTracingWarnings.append(volumeSelectionSetting); + var volumeSelectionWrapper = CATMAID.createVolumeSelector({ + mode: "radio", + label: "New nodes not in volume", + title: "A warning will be shown when new nodes are created outside of the selected volume", + selectedVolumeIds: [SkeletonAnnotations.getNewNodeVolumeWarning()], + select: function(volumeId, selected, element){ + // Remove existing handler and new one if selected + SkeletonAnnotations.setNewNodeVolumeWarning(element.value !== "none"? volumeId : null); + } + }); + dsTracingWarnings.append(volumeSelectionWrapper); // Skeleton length warning diff --git a/django/applications/catmaid/static/js/widgets/volumewidget.js b/django/applications/catmaid/static/js/widgets/volumewidget.js index 33d4549ab5..48c605dcb5 100644 --- a/django/applications/catmaid/static/js/widgets/volumewidget.js +++ b/django/applications/catmaid/static/js/widgets/volumewidget.js @@ -71,7 +71,7 @@ if (file.name.endsWith("stl")){ return true; } else { - this.addVolumeFromFile(file); + this.addVolumeFromFile(file).catch(CATMAID.handleError); } },this)).catch(CATMAID.handleError); } @@ -608,23 +608,25 @@ * @param {String} files The file to load */ VolumeManagerWidget.prototype.addVolumeFromFile = function(file) { - var self = this; - var reader = new FileReader(); - reader.onload = function(e) { - var volumes = JSON.parse(e.target.result); - // Try to load volumes and record invalid ones - var invalidVolumes = volumes.filter(function(v) { - var volumeType = volumeTypes[v.type]; - var properties = v.properties; - if (volumeType && properties) { - volumeType.createVolume(properties); - } else { - // Return true for invalid volume types - return !volumeType; - } - }); - }; - reader.readAsText(file); + return new Promise(function(resolve, reject) { + var self = this; + var reader = new FileReader(); + reader.onload = function(e) { + var volumes = JSON.parse(e.target.result); + // Try to load volumes and record invalid ones + var invalidVolumes = volumes.filter(function(v) { + var volumeType = volumeTypes[v.type]; + var properties = v.properties; + if (volumeType && properties) { + volumeType.createVolume(properties); + } else { + // Return true for invalid volume types + return !volumeType; + } + }); + }; + reader.readAsText(file); + }); }; VolumeManagerWidget.prototype.addVolumesFromSTL = function(files) { diff --git a/django/applications/catmaid/static/libs/catmaid/filter.js b/django/applications/catmaid/static/libs/catmaid/filter.js index dc28824713..9959705284 100644 --- a/django/applications/catmaid/static/libs/catmaid/filter.js +++ b/django/applications/catmaid/static/libs/catmaid/filter.js @@ -635,34 +635,17 @@ // Take all has no additional options }, 'volume': function(container, options) { - // Update volume list - var initVolumeList = function() { - return CATMAID.Volumes.listAll(project.id).then(function(json) { - var volumes = json.sort(function(a, b) { - return CATMAID.tools.compareStrings(a.name, b.name); - }).map(function(volume) { - return { - title: volume.name, - value: volume.id - }; - }); - var selectedVolume = options.volumeId; - // Create actual element based on the returned data - var node = CATMAID.DOM.createRadioSelect('Volumes', volumes, - selectedVolume, true); - // Add a selection handler - node.onchange = function(e) { - options.volumeId = e.target.value; - }; - return node; - }); - }; - // Create async selection and wrap it in container to have handle on initial // DOM location - var volumeSelection = CATMAID.DOM.createAsyncPlaceholder(initVolumeList()); - var volumeSelectionWrapper = document.createElement('span'); - $(container).append(volumeSelection); + var volumeSelectionWrapper = CATMAID.createVolumeSelector({ + mode: "radio", + selectedVolumeIds: [options.volumeId], + select: function(volumeId, selected, element){ + options.volumeId = volumeId; + } + }); + + container.appendChild(volumeSelectionWrapper); }, }; From bae8dbfb63bb1bf0b70e36eb65732f61c0bc5185 Mon Sep 17 00:00:00 2001 From: willp24 Date: Fri, 20 Jul 2018 17:45:35 -0400 Subject: [PATCH 22/26] Volumes: handle error in volume list better. --- django/applications/catmaid/static/js/helpers/volumes.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django/applications/catmaid/static/js/helpers/volumes.js b/django/applications/catmaid/static/js/helpers/volumes.js index 7674a7a1d7..662f8c6c3b 100644 --- a/django/applications/catmaid/static/js/helpers/volumes.js +++ b/django/applications/catmaid/static/js/helpers/volumes.js @@ -17,13 +17,13 @@ }; }); if (options.mode === "radio") { - if (newSelectedIds.length > 1){ + let selectedVolume = newSelectedIds || options.selectedVolumeIds; + if (selectedVolume.length > 1){ throw new CATMAID.ValueError("Radio select only takes one selected volume"); } - let selectedVolume = newSelectedIds[0] || options.selectedVolumeIds[0]; // Create actual element based on the returned data let node = CATMAID.DOM.createRadioSelect('Volumes', volumes, - selectedVolume, true); + selectedVolume[0], true); // Add a selection handler node.onchange = function (e) { let volumeId = e.target.value; From 341164d86f4c50cedbb3251d9d6db816ccfe06a1 Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Tue, 16 Oct 2018 21:51:24 -0400 Subject: [PATCH 23/26] API changelog: add volume endpoint updates --- API_CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/API_CHANGELOG.md b/API_CHANGELOG.md index 7a81f49d0c..0857abde3e 100644 --- a/API_CHANGELOG.md +++ b/API_CHANGELOG.md @@ -28,6 +28,12 @@ included in this changelog. - `GET /{project_id}/labels/detail`: Returns a list of of label objects, each with a name field and an ID field. +- `POST /{project_id}/volumes/import`: + Import volumes as STL files. + +- `GET /{project_id}/volumes/{volume_id}/export.{extension}`: + Export a particular volume. Currentl only exports AS STL are supported. + ### Modifications - `POST /{project_id}/skeletons/node-label`: @@ -54,6 +60,13 @@ included in this changelog. marked as soma in the SWC export. The first matching condition in this order wins. +- `GET /{project_id}/volumes/`: + The return format changed. Instead of a list of volume objects an object with + a 'columns' field and a 'data' field are returned. The data fields contains a + list of lists, with each inner list being a volume. The entries are described + by the 'columns' field. Along with the already returned fields, annotations + are now retuned as well. + ### Deprecations None. @@ -75,6 +88,9 @@ None. Accepts the same parameters as the GET variant, but allows for larger skeleton_ids list. +- `GET /{project_id}/volumes/entities`: + Return a mapping of volume IDs to their respective class instance ID. + ### Modifications - `GET /{project_id}/skeletons/in-bounding-box`: From dce6ccc2bd57b94250c8c119edf726440e80c93c Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Wed, 17 Oct 2018 13:09:03 -0400 Subject: [PATCH 24/26] DB inegrity check: add test if all mesh faces are triangles This is run now by default for the catmaid_check_db_integrity management command. The --tracing [true|false] and --volumes [true|false] command line parameters now allow to explicitly test only some parts of the database. By default all is tested. Please enter the commit message for your changes. Lines starting See catmaid/CATMAID#1765 --- CHANGELOG.md | 5 ++ .../commands/catmaid_check_db_integrity.py | 52 +++++++++++++++++-- django/applications/catmaid/util.py | 9 ++++ 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c96b3bded..c16b092404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,11 @@ Miscellaneous: - Admin: a user import view is now available to import users from other CATMAID instances. It requires superuser permissions on the remote instance. +- DB integrity check management command: volumes are now checked to make sure + all faces are triangles. The --tracing [true|false] and --volumes [true|false] + command line parameters now allow to explicitly test only some parts of the + database. By default all is tested. + ### Bug fixes diff --git a/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py b/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py index 78883eb624..e676143915 100644 --- a/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py +++ b/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py @@ -7,6 +7,7 @@ from django.core.management.base import BaseCommand, CommandError from django.db import connection from catmaid.models import Project +from catmaid.util import str2bool class Command(BaseCommand): help = ''' @@ -15,6 +16,10 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument('--project_id', nargs='*', type=int, default=[]) + parser.add_argument("--tracing", type=str2bool, nargs='?', + const=True, default=True, help="Check tracing data.") + parser.add_argument("--volumes", type=str2bool, nargs='?', + const=True, default=True, help="Check volumes data.") def handle(self, *args, **options): project_ids = options['project_id'] @@ -23,17 +28,29 @@ def handle(self, *args, **options): passed = True for project_id in project_ids: - passed = passed and self.check_project(project_id) + passed = passed and self.check_project(project_id, options) if not passed: sys.exit(1) - def check_project(self, project_id): + def check_project(self, project_id, options): if not Project.objects.filter(id=project_id).exists(): raise CommandError('Project with id %s does not exist.' % project_id) - project_passed = True self.stdout.write('Checking integrity of project %s' % project_id) + passed = True + if options['tracing']: + passed = passed and self.check_tracing_data(project_id) + + if options['volumes']: + passed = passed and self.check_volumes(project_id) + + self.stdout.write('') + + return passed + + + def check_tracing_data(self, project_id): self.stdout.write('Check that no connected treenodes are in different skeletons...', ending='') cursor = connection.cursor() cursor.execute(''' @@ -102,9 +119,34 @@ def check_project(self, project_id): project_passed = False row = cursor.fetchone() self.stdout.write('FAILED: node %s in skeleton %s has no path to root' % row) + if test_passed: self.stdout.write('OK') - self.stdout.write('') - return project_passed + def check_volumes(self, project_id): + passed = True + self.stdout.write('Check if all meshes consist only of triangles...', ending='') + cursor = connection.cursor() + cursor.execute(""" + SELECT g.volume_id, + (g.gdump).path[1] as triangle_id, + COUNT(*) as n_points + FROM ( + SELECT v.id AS volume_id, + ST_DumpPoints(geometry) AS gdump + FROM catmaid_volume v + ) AS g + GROUP BY volume_id, (g.gdump).path[1] + HAVING COUNT(*) <> 4; + """) + n_non_triangles = len(list(cursor.fetchall())) + if n_non_triangles > 0: + self.stdout.write('FAILED: found {} non-triangle meshes in project {}'.format( + n_non_triangles, project_id)) + passed = False + else: + self.stdout.write('OK') + passed = passed and True + + return passed diff --git a/django/applications/catmaid/util.py b/django/applications/catmaid/util.py index fc56d16b16..005dd3346c 100644 --- a/django/applications/catmaid/util.py +++ b/django/applications/catmaid/util.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import argparse import math from django.utils.encoding import python_2_unicode_compatible @@ -72,3 +73,11 @@ def is_collinear(a, b, c, between=False, eps=epsilon): return not (min(tx, ty, tz) < 0.0 or max(tx, ty, tz) > 1.0) else: return True + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') From 3b29a4afd7db2e03266ab0bead09fa6daaa0914d Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Wed, 17 Oct 2018 16:04:56 -0400 Subject: [PATCH 25/26] DB integrity check: improve triangle test Like before, the triangle check tests if each triangle has four vertices (the line is explicitly closed) and now also checks if the start and end point are actually the same. --- .../commands/catmaid_check_db_integrity.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py b/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py index e676143915..37f66023cc 100644 --- a/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py +++ b/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py @@ -129,21 +129,35 @@ def check_volumes(self, project_id): self.stdout.write('Check if all meshes consist only of triangles...', ending='') cursor = connection.cursor() cursor.execute(""" - SELECT g.volume_id, - (g.gdump).path[1] as triangle_id, - COUNT(*) as n_points - FROM ( - SELECT v.id AS volume_id, - ST_DumpPoints(geometry) AS gdump - FROM catmaid_volume v - ) AS g - GROUP BY volume_id, (g.gdump).path[1] - HAVING COUNT(*) <> 4; + SELECT volume_id, triangle_id, path, txtpoints + FROM ( + SELECT volume_id, + (v.gdump).path[1], + array_agg((v.gdump).path order by (v.gdump).path[3] ASC), + array_agg((v.gdump).geom order by (v.gdump).path[3] ASC) as points, + array_agg(ST_AsText((v.gdump).geom) ORDER BY (v.gdump).path[3] ASC) as txtpoints + FROM ( + SELECT volume_id, gdump + FROM ( + SELECT v.id AS volume_id, + ST_DumpPoints(geometry) AS gdump + FROM catmaid_volume v + ) v(volume_id, gdump) + ) v(volume_id, gdump) + GROUP BY v.volume_id, (v.gdump).path[1] + ) triangle(volume_id, triangle_id, path, points, txtpoints) + WHERE array_length(points, 1) <> 4 + OR ST_X(points[1]) <> ST_X(points[4]) + OR ST_Y(points[1]) <> ST_Y(points[4]) + OR ST_Z(points[1]) <> ST_Z(points[4]); """) - n_non_triangles = len(list(cursor.fetchall())) + non_triangles = list(cursor.fetchall()) + n_non_triangles = len(non_triangles) if n_non_triangles > 0: self.stdout.write('FAILED: found {} non-triangle meshes in project {}'.format( n_non_triangles, project_id)) + self.stdout.write('\tThe following volumes contain those geometries: {}'.format( + ', '.join(nt[0] for nt in non_triangles))) passed = False else: self.stdout.write('OK') From a421f8b5f5a745b5864e5b96aeec8d6c06f1e039 Mon Sep 17 00:00:00 2001 From: Tom Kazimiers Date: Wed, 17 Oct 2018 21:55:14 -0400 Subject: [PATCH 26/26] DB integrity check: add consistency test for volume triangle winding If the 'trimesh' library is installed, the 'check_db_integretiy' management command will now also check if all triangles that make up CATMAID volumes have the same orientation (per volume). See catmaid/CATMAID#1765 --- .../commands/catmaid_check_db_integrity.py | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py b/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py index 37f66023cc..8c7d552f1b 100644 --- a/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py +++ b/django/applications/catmaid/management/commands/catmaid_check_db_integrity.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- +import logging import sys - +import numpy as np import progressbar from django.core.management.base import BaseCommand, CommandError @@ -9,6 +10,18 @@ from catmaid.models import Project from catmaid.util import str2bool + +logger = logging.getLogger(__name__) + +winding_check_enabled = True +try: + from trimesh import Trimesh + from trimesh.grouping import merge_vertices_hash +except ImportError: + logger.warn('Optional depedency "trimesh" not found. Won\'t be able to check volume triange winding.') + winding_check_enabled = False + + class Command(BaseCommand): help = ''' Tests the integrity of the specified projects with several sanity checks @@ -163,4 +176,64 @@ def check_volumes(self, project_id): self.stdout.write('OK') passed = passed and True + + self.stdout.write('Check if all triangles have the same orientation...', ending='') + if winding_check_enabled: + cursor.execute(""" + SELECT volume_id, triangle_id, points + FROM ( + SELECT volume_id, + (v.gdump).path[1], + /* Points need to be ordered by index to be comparable. */ + array_agg(ARRAY[ST_X((v.gdump).geom), ST_Y((v.gdump).geom), ST_Z((v.gdump).geom)] order by (v.gdump).path[ 3] ASC) as points + FROM ( + SELECT volume_id, gdump + FROM ( + SELECT v.id AS volume_id, + ST_DumpPoints(geometry) AS gdump + FROM catmaid_volume v + ) v(volume_id, gdump) + ) v(volume_id, gdump) + GROUP BY v.volume_id, (v.gdump).path[1] + ) triangle(volume_id, triangle_id, points) + WHERE array_length(points, 1) = 4; + """) + volumes = {} + for tri in cursor.fetchall(): + entry = volumes.get(tri[0]) + if not entry: + entry = { + 'volume_id': tri[0], + 'vertices': [], + 'faces': [], + } + volumes[tri[0]] = entry + vertices = entry['vertices'] + faces = entry['faces'] + vertex_offset = len(vertices) + vertices.extend(tri[2]) + faces.append([vertex_offset, vertex_offset + 1, vertex_offset + 2]) + + volumes_with_inconsistent_winding = [] + for volume_id, details in volumes.items(): + mesh = Trimesh(vertices=np.array(vertices), faces=np.array(faces), + process=False) + # Merge all vertices in trimeshs + merge_vertices_hash(mesh) + # Check if the winding is consistent + if not mesh.is_winding_consistent: + volumes_with_inconsistent_winding.append(volume_id) + details['mesh'] = mesh + + if volumes_with_inconsistent_winding: + self.stdout.write('FAILED: The following volumes have an ' + + 'inconsistent winding: {}'.format(', '.join( + volumes_with_inconsistent_winding))) + else: + self.stdout.write('OK') + + passed = passed and not volumes_with_inconsistent_winding + else: + self.stdout.write('Not enabled (pip intall trimesh to enable)') + return passed