From c0d91348d75357f11729ef3fdc2a689ff5f8915c Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 28 Jun 2016 15:08:06 +0200 Subject: [PATCH] Added procedures for importing shapefiles --- src/main/assembly/server-plugin.xml | 1 - .../neo4j/gis/spatial/ShapefileImporter.java | 31 +++++++++------ .../spatial/procedures/SpatialProcedures.java | 38 ++++++++++++++++++- .../procedures/SpatialProceduresTest.java | 23 ++++++++++- 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/src/main/assembly/server-plugin.xml b/src/main/assembly/server-plugin.xml index d98edeb53..a9f9a2646 100644 --- a/src/main/assembly/server-plugin.xml +++ b/src/main/assembly/server-plugin.xml @@ -35,7 +35,6 @@ org.geotools:gt-process org.geotools:gt-render - org.geotools:gt-shapefile org.geotools:gt-coverage org.geotools:gt-cql javax.media:jai_imageio diff --git a/src/main/java/org/neo4j/gis/spatial/ShapefileImporter.java b/src/main/java/org/neo4j/gis/spatial/ShapefileImporter.java index 3b981b4aa..b430f5d25 100644 --- a/src/main/java/org/neo4j/gis/spatial/ShapefileImporter.java +++ b/src/main/java/org/neo4j/gis/spatial/ShapefileImporter.java @@ -20,23 +20,23 @@ package org.neo4j.gis.spatial; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import org.geotools.data.shapefile.ShpFiles; import org.geotools.data.shapefile.dbf.DbaseFileHeader; import org.geotools.data.shapefile.dbf.DbaseFileReader; import org.geotools.data.shapefile.prj.PrjFileReader; import org.geotools.data.shapefile.shp.JTSUtilities; -import org.geotools.data.shapefile.shp.ShapefileException; import org.geotools.data.shapefile.shp.ShapefileReader; import org.geotools.data.shapefile.shp.ShapefileReader.Record; import org.neo4j.gis.spatial.rtree.Listener; import org.neo4j.gis.spatial.rtree.NullListener; import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.opengis.referencing.crs.CoordinateReferenceSystem; @@ -117,15 +117,20 @@ public static void main(String[] args) throws Exception { // Public methods - - public void importFile(String dataset, String layerName) throws ShapefileException, FileNotFoundException, IOException { - importFile(dataset, layerName, Charset.defaultCharset()); - } - - public void importFile(String dataset, String layerName, Charset charset) throws ShapefileException, FileNotFoundException, IOException { - Class layerClass = maintainGeometryOrder ? OrderedEditableLayer.class : EditableLayerImpl.class; - EditableLayerImpl layer = (EditableLayerImpl)spatialDatabase.getOrCreateLayer(layerName, WKBGeometryEncoder.class, layerClass); - GeometryFactory geomFactory = layer.getGeometryFactory(); + + public List importFile(String dataset, String layerName) throws IOException { + return importFile(dataset, layerName, Charset.defaultCharset()); + } + + public List importFile(String dataset, String layerName, Charset charset) throws IOException { + Class layerClass = maintainGeometryOrder ? OrderedEditableLayer.class : EditableLayerImpl.class; + EditableLayerImpl layer = (EditableLayerImpl) spatialDatabase.getOrCreateLayer(layerName, WKBGeometryEncoder.class, layerClass); + return importFile(dataset, layer, charset); + } + + public List importFile(String dataset, EditableLayerImpl layer, Charset charset) throws IOException { + GeometryFactory geomFactory = layer.getGeometryFactory(); + ArrayList added = new ArrayList<>(); boolean strict = false; boolean shpMemoryMapped = true; @@ -206,7 +211,8 @@ record = shpReader.nextRecord(); } else { // TODO check geometry.isValid() // ? - layer.add(geometry, fieldsName, fields.toArray(values)); + SpatialDatabaseRecord spatial_record = layer.add(geometry, fieldsName, fields.toArray(values)); + added.add(spatial_record.getGeomNode()); } } else { filterCounter ++; @@ -242,6 +248,7 @@ record = shpReader.nextRecord(); long stopTime = System.currentTimeMillis(); log("info | elapsed time in seconds: " + (1.0 * (stopTime - startTime) / 1000)); + return added; } diff --git a/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java b/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java index aaf2fdaf5..27574f004 100644 --- a/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java +++ b/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java @@ -44,6 +44,9 @@ import org.neo4j.procedure.PerformsWrites; import org.neo4j.procedure.Procedure; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -324,6 +327,37 @@ public Stream updateGeometryFromWKT(@Name("layerName") String name, return null; } + @Procedure("spatial.importShapefileToLayer") + @PerformsWrites + public Stream importShapefile( + @Name("layerName") String name, + @Name("uri") String uri) throws IOException { + EditableLayerImpl layer = getEditableLayerOrThrow(name); + return importShapefileToLayer(uri, layer, 1000).stream().map(node -> new NodeResult(node)); + } + + @Procedure("spatial.importShapefile") + @PerformsWrites + public Stream importShapefile( + @Name("uri") String uri) throws IOException { + return importShapefileToLayer(uri, null, 1000).stream().map(node -> new NodeResult(node)); + } + + private List importShapefileToLayer(String shpPath, EditableLayerImpl layer, int commitInterval) throws IOException { + if (shpPath.toLowerCase().endsWith(".shp")) { + // remove extension + shpPath = shpPath.substring(0, shpPath.lastIndexOf(".")); + } + + ShapefileImporter importer = new ShapefileImporter(db, new ProgressLoggingListener("Importing " + shpPath, log.debugLogger()), commitInterval); + if (layer == null) { + String layerName = shpPath.substring(shpPath.lastIndexOf(File.separator) + 1); + return importer.importFile(shpPath, layerName); + } else { + return importer.importFile(shpPath, layer, Charset.defaultCharset()); + } + } + @Procedure("spatial.bbox") @PerformsWrites // TODO FIX public Stream findGeometriesInBBox( @@ -435,8 +469,8 @@ private Coordinate toCoordinate(Map map, String xName, String yName) { return null; } - private EditableLayer getEditableLayerOrThrow(String name) { - return (EditableLayer) getLayerOrThrow(wrap(db), name); + private EditableLayerImpl getEditableLayerOrThrow(String name) { + return (EditableLayerImpl) getLayerOrThrow(wrap(db), name); } private Layer getLayerOrThrow(String name) { diff --git a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java index aefbbf944..186f92fcc 100644 --- a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java +++ b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java @@ -100,11 +100,11 @@ public static void testCallCount(GraphDatabaseService db, String call, Map { int numLeft = count; while (numLeft > 0) { - assertTrue("Expected " + count + " results but found only " + numLeft, res.hasNext()); + assertTrue("Expected " + count + " results but found only " + (count - numLeft), res.hasNext()); res.next(); numLeft--; } - Assert.assertFalse("Expected " + count + " results but there are more " + numLeft, res.hasNext()); + Assert.assertFalse("Expected " + count + " results but there are more", res.hasNext()); }); } @@ -344,6 +344,25 @@ public void add_many_nodes_to_the_spatial_layer_using_addNode() throws Exception testCountQuery("addNode", query, count, "count(node)", map("count", count)); } + @Test + public void import_shapefile() throws Exception { + testCallCount(db, "CALL spatial.importShapefile('shp/highway.shp')", null, 143); + testCallCount(db, "CALL spatial.layers()", null, 1); + } + + @Test + public void import_shapefile_without_extension() throws Exception { + testCallCount(db, "CALL spatial.importShapefile('shp/highway')", null, 143); + testCallCount(db, "CALL spatial.layers()", null, 1); + } + + @Test + public void import_shapefile_to_layer() throws Exception { + execute("CALL spatial.addWKTLayer('geom','wkt')"); + testCallCount(db, "CALL spatial.importShapefileToLayer('geom','shp/highway.shp')", null, 143); + testCallCount(db, "CALL spatial.layers()", null, 1); + } + private void testCountQuery(String name, String query, long count, String column, Map params) { Result results = db.execute("EXPLAIN " + query); results.close();