Update for Neo4j 4.x

This apparently requires us to manually delete the temp DB after test runs, and every impermanent DB is 500 megs.
EHRI · Oct 18, 2021 · daf7447 · daf7447
1 parent bac5981
commit daf7447
Show file tree

Hide file tree

Showing 56 changed files with 411 additions and 465 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,7 @@
 sudo: false
 
 language: java
-jdk: openjdk8
+jdk: openjdk11
 
 cache:
   directories:

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Docker file for EHRI backend web service
-FROM neo4j:3.5.28
+FROM neo4j:4.2.9
 
 # Set git commit build revision as a label which
 # we can inspect to figure out the image version.

diff --git a/ehri-cli/src/main/java/eu/ehri/project/commands/GenSchema.java b/ehri-cli/src/main/java/eu/ehri/project/commands/GenSchema.java
@@ -24,6 +24,8 @@
 import eu.ehri.project.core.impl.Neo4jGraphManager;
 import eu.ehri.project.core.impl.neo4j.Neo4j2Graph;
 import org.apache.commons.cli.CommandLine;
+import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.graphdb.Transaction;
 
 /**
  * Command for generating the (Neo4j) graph schema.
@@ -47,8 +49,11 @@ public String getHelp() {
     public int execWithOptions(FramedGraph<?> graph, CommandLine cmdLine) throws Exception {
         Graph baseGraph = graph.getBaseGraph();
         if (baseGraph instanceof Neo4j2Graph) {
-            Neo4jGraphManager.createIndicesAndConstraints(
-                    ((Neo4j2Graph) baseGraph).getRawGraph());
+            GraphDatabaseService service = ((Neo4j2Graph) baseGraph).getRawGraph();
+            try (Transaction tx = service.beginTx()) {
+                Neo4jGraphManager.dropIndicesAndConstraints(tx);
+                Neo4jGraphManager.createIndicesAndConstraints(tx);
+            }
         } else {
             logger.warn("Cannot generate schema on a non-Neo4j2 graph");
         }

diff --git a/ehri-cli/src/test/java/eu/ehri/project/commands/GraphSONTest.java b/ehri-cli/src/test/java/eu/ehri/project/commands/GraphSONTest.java
@@ -66,6 +66,7 @@ public void testSaveDumpAndRead() throws Exception {
 
         assertEquals(0, graphSON.execWithOptions(graph1, outCmdLine));
         graph1.shutdown();
+        resetGraph();
 
         assertTrue(temp.exists());
         assertTrue(temp.length() > 0L);

diff --git a/ehri-core/pom.xml b/ehri-core/pom.xml
@@ -132,6 +132,18 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.neo4j.test</groupId>
+            <artifactId>neo4j-harness</artifactId>
+            <version>${neo4j.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-nop</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.neo4j.community</groupId>
             <artifactId>it-test-support</artifactId>

diff --git a/ehri-core/src/main/java/eu/ehri/project/core/impl/Neo4jGraphManager.java b/ehri-core/src/main/java/eu/ehri/project/core/impl/Neo4jGraphManager.java
@@ -31,8 +31,8 @@
 import eu.ehri.project.models.EntityClass;
 import eu.ehri.project.models.annotations.EntityType;
 import eu.ehri.project.models.utils.ClassUtils;
-import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Label;
+import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.schema.ConstraintDefinition;
 import org.neo4j.graphdb.schema.IndexDefinition;
 import org.neo4j.graphdb.schema.Schema;
@@ -101,7 +101,6 @@ public Vertex updateVertex(String id, EntityClass type,
 
     @Override
     public void initialize() {
-        createIndicesAndConstraints(graph.getBaseGraph().getRawGraph());
     }
 
     @Override
@@ -127,19 +126,27 @@ public Vertex setLabels(Vertex vertex) {
     }
 
     /**
-     * Create the graph schema
+     * Drop the existing schema.
      *
-     * @param graph the underlying graph service
+     * @param tx the underlying graph transactions
      */
-    public static void createIndicesAndConstraints(GraphDatabaseService graph) {
-        Schema schema = graph.schema();
+    public static void dropIndicesAndConstraints(Transaction tx) {
+        Schema schema = tx.schema();
         for (ConstraintDefinition constraintDefinition : schema.getConstraints()) {
             constraintDefinition.drop();
         }
         for (IndexDefinition indexDefinition : schema.getIndexes()) {
             indexDefinition.drop();
         }
+    }
 
+    /**
+     * Create the graph schema
+     *
+     * @param tx the underlying graph transaction
+     */
+    public static void createIndicesAndConstraints(Transaction tx) {
+        Schema schema = tx.schema();
         schema.constraintFor(Label.label(BASE_LABEL))
                 .assertPropertyIsUnique(EntityType.ID_KEY)
                 .create();

diff --git a/ehri-core/src/main/java/eu/ehri/project/core/impl/TxNeo4jGraph.java b/ehri-core/src/main/java/eu/ehri/project/core/impl/TxNeo4jGraph.java
@@ -22,6 +22,7 @@
 import eu.ehri.project.core.Tx;
 import eu.ehri.project.core.TxGraph;
 import eu.ehri.project.core.impl.neo4j.Neo4j2Graph;
+import org.neo4j.dbms.api.DatabaseManagementService;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Transaction;
 import org.slf4j.Logger;
@@ -41,20 +42,15 @@ public static class UnderlyingTxRemovedError extends RuntimeException {
 
     public static final Logger logger = LoggerFactory.getLogger(TxNeo4jGraph.class);
 
-    public TxNeo4jGraph(String directory) {
-        super(directory);
+    public TxNeo4jGraph(DatabaseManagementService service, GraphDatabaseService graph) {
+        super(service, graph);
     }
 
-    public TxNeo4jGraph(GraphDatabaseService rawGraph) {
-        super(rawGraph);
+    public TxNeo4jGraph(String directory) {
+        super(directory);
     }
 
-    private ThreadLocal<Neo4jTx> etx = new ThreadLocal<Neo4jTx>() {
-        @Override
-        public Neo4jTx initialValue() {
-            return null;
-        }
-    };
+    private final ThreadLocal<Neo4jTx> etx = ThreadLocal.withInitial(() -> null);
 
     public Tx beginTx() {
         logger.trace("Begin tx: {}", Thread.currentThread().getName());
@@ -89,7 +85,7 @@ public void commit() {
 
     @Override
     public void shutdown() {
-        getRawGraph().shutdown();
+        getManagementService().shutdown();
     }
 
     /**
@@ -102,18 +98,6 @@ public void autoStartTransaction(boolean forWrite) {
         // Not allowing auto-start TX
     }
 
-    /**
-     * Since we override autoStartTx to do nothing, we must also
-     * override this function (which loads key indices and sneakily
-     * commits them) when the graph is constructed. We do not use
-     * key indexable functionality so nothing is lost (unless the
-     * base class is changed, but, can't do much about that.)
-     */
-    @Override
-    public void init() {
-        // Don't load key indices
-    }
-
     /**
      * Checks if the graph is currently in a transaction.
      *
@@ -130,7 +114,7 @@ public class Neo4jTx implements Tx {
          *
          * @return a Neo4j transaction
          */
-        Transaction underlying() {
+        public Transaction underlying() {
             return tx.get();
         }
 
@@ -140,7 +124,7 @@ public void success() {
             if (transaction == null) {
                 throw new UnderlyingTxRemovedError("Underlying transaction removed!");
             }
-            transaction.success();
+            transaction.commit();
         }
 
         public void close() {
@@ -160,7 +144,7 @@ public void failure() {
             if (transaction == null) {
                 throw new UnderlyingTxRemovedError("Underlying transaction removed!");
             }
-            transaction.failure();
+            transaction.rollback();
         }
     }
-}
+}
diff --git a/ehri-core/src/main/java/eu/ehri/project/core/impl/neo4j/Neo4j2EdgeIterable.java b/ehri-core/src/main/java/eu/ehri/project/core/impl/neo4j/Neo4j2EdgeIterable.java
@@ -4,29 +4,29 @@
 import com.tinkerpop.blueprints.CloseableIterable;
 import com.tinkerpop.blueprints.Edge;
 import org.neo4j.graphdb.Relationship;
-import org.neo4j.graphdb.index.IndexHits;
+import org.neo4j.graphdb.ResourceIterable;
 
 import java.util.Iterator;
 
 
 public class Neo4j2EdgeIterable<T extends Edge> implements CloseableIterable<Neo4j2Edge> {
 
-    private final Iterable<Relationship> relationships;
+    private final ResourceIterable<Relationship> relationships;
     private final Neo4j2Graph graph;
 
     @Deprecated
-    public Neo4j2EdgeIterable(Iterable<Relationship> relationships, Neo4j2Graph graph, boolean checkTransaction) {
+    public Neo4j2EdgeIterable(ResourceIterable<Relationship> relationships, Neo4j2Graph graph, boolean checkTransaction) {
         this(relationships, graph);
     }
 
-    public Neo4j2EdgeIterable(Iterable<Relationship> relationships, Neo4j2Graph graph) {
+    public Neo4j2EdgeIterable(ResourceIterable<Relationship> relationships, Neo4j2Graph graph) {
         this.relationships = relationships;
         this.graph = graph;
     }
 
     public Iterator<Neo4j2Edge> iterator() {
         graph.autoStartTransaction(true);
-        return new Iterator<Neo4j2Edge>() {
+        return new Iterator<>() {
             private final Iterator<Relationship> itty = relationships.iterator();
 
             public void remove() {
@@ -47,8 +47,6 @@ public boolean hasNext() {
     }
 
     public void close() {
-        if (this.relationships instanceof IndexHits) {
-            ((IndexHits<?>) this.relationships).close();
-        }
+        this.relationships.iterator().close();
     }
 }
diff --git a/ehri-core/src/main/java/eu/ehri/project/core/impl/neo4j/Neo4j2Element.java b/ehri-core/src/main/java/eu/ehri/project/core/impl/neo4j/Neo4j2Element.java
@@ -5,22 +5,18 @@
 import com.tinkerpop.blueprints.Element;
 import com.tinkerpop.blueprints.Vertex;
 import com.tinkerpop.blueprints.util.ElementHelper;
+import org.neo4j.graphdb.Entity;
 import org.neo4j.graphdb.Node;
-import org.neo4j.graphdb.PropertyContainer;
 import org.neo4j.graphdb.Relationship;
 
 import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
+import java.util.*;
 
 
 abstract class Neo4j2Element implements Element {
 
     protected final Neo4j2Graph graph;
-    protected PropertyContainer rawElement;
+    protected Entity rawElement;
 
     public Neo4j2Element(Neo4j2Graph graph) {
         this.graph = graph;
@@ -63,7 +59,7 @@ public int hashCode() {
         return this.getId().hashCode();
     }
 
-    public PropertyContainer getRawElement() {
+    public Entity getRawElement() {
         return this.rawElement;
     }