Feature/neo4j 5x wip NOT FOR MERGING

.github/workflows/CI.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up JDK 1.8
+    - name: Set up JDK 17
       uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
-        java-version: '8'
+        java-version: '17'
         cache: 'maven'
 
     - name: Build with Maven

.travis.yml

@@ -1,7 +1,7 @@
 sudo: false
 
 language: java
-jdk: openjdk8
+jdk: openjdk11
 
 cache:
   directories:

Dockerfile

@@ -1,5 +1,5 @@
 # Docker file for EHRI backend web service
-FROM neo4j:3.5.28
+FROM neo4j:4.2.9
 
 # Set git commit build revision as a label which
 # we can inspect to figure out the image version.

build/pom.xml

@@ -50,6 +50,7 @@
                             <include>com.tinkerpop:*</include>
                             <include>com.tinkerpop.*:*</include>
                             <include>stax:stax-api</include>
+                            <include>com.carrotsearch:hppc</include>
                             <include>org.javassist:javassist</include>
                             <include>org.codehaus.jettison:jettison</include>
                             <include>org.neo4j:neo4j-graphviz</include>
@@ -66,6 +67,7 @@
                             <include>commons-cli:commons-cli</include>
                             <include>commons-codec:commons-codec</include>
                             <include>org.apache.jena:*</include>
+                            <include>org.apache.log4j:*</include>
                             <include>xerces:xercesImpl</include>
                             <include>xml-apis:xml-apis</include>
                             <include>com.typesafe:config</include>

ehri-cli/pom.xml

@@ -27,6 +27,16 @@
                     </archive>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.16</version>
+                <configuration>
+                    <argLine>
+                        --add-opens java.base/java.lang=ALL-UNNAMED
+                    </argLine>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
@@ -113,7 +123,7 @@
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
-            <version>1.2.10</version>
+            <version>1.5.6</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

ehri-cli/src/main/java/eu/ehri/project/commands/CmdEntryPoint.java

@@ -85,6 +85,7 @@ public CmdEntryPoint() {
         mmap.put(LoadFixtures.NAME, LoadFixtures.class);
         mmap.put(Initialize.NAME, Initialize.class);
         mmap.put(GenSchema.NAME, GenSchema.class);
+        mmap.put(SetLabels.NAME, SetLabels.class);
         mmap.put(UserAdd.NAME, UserAdd.class);
         mmap.put(UserMod.NAME, UserMod.class);
         mmap.put(EntityAdd.NAME, EntityAdd.class);

ehri-cli/src/main/java/eu/ehri/project/commands/GenSchema.java

@@ -24,6 +24,8 @@
 import eu.ehri.project.core.impl.Neo4jGraphManager;
 import eu.ehri.project.core.impl.neo4j.Neo4j2Graph;
 import org.apache.commons.cli.CommandLine;
+import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.graphdb.Transaction;
 
 /**
  * Command for generating the (Neo4j) graph schema.
@@ -47,8 +49,17 @@ public String getHelp() {
     public int execWithOptions(FramedGraph<?> graph, CommandLine cmdLine) throws Exception {
         Graph baseGraph = graph.getBaseGraph();
         if (baseGraph instanceof Neo4j2Graph) {
-            Neo4jGraphManager.createIndicesAndConstraints(
-                    ((Neo4j2Graph) baseGraph).getRawGraph());
+            GraphDatabaseService service = ((Neo4j2Graph) baseGraph).getRawGraph();
+            try (Transaction tx = service.beginTx()) {
+                System.err.println("INFO: Dropping all indices and constraints...");
+                Neo4jGraphManager.dropIndicesAndConstraints(tx);
+                tx.commit();
+            }
+            try (Transaction tx = service.beginTx()) {
+                System.err.println("INFO: Creating all indices and constraints...");
+                Neo4jGraphManager.createIndicesAndConstraints(tx);
+                tx.commit();
+            }
         } else {
             System.err.println("ERROR: Cannot generate schema on a non-Neo4j2 graph");
         }

ehri-cli/src/main/java/eu/ehri/project/commands/GraphSON.java

@@ -19,12 +19,10 @@
 
 package eu.ehri.project.commands;
 
-import com.tinkerpop.blueprints.Vertex;
 import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
 import com.tinkerpop.blueprints.util.io.graphson.GraphSONReader;
 import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
 import com.tinkerpop.frames.FramedGraph;
-import eu.ehri.project.core.impl.Neo4jGraphManager;
 import eu.ehri.project.core.impl.neo4j.Neo4j2Graph;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
@@ -148,19 +146,8 @@ private void loadDump(FramedGraph<?> graph,
             reader.inputGraph(readStream, bufferSize);
             if (!cmdLine.hasOption("skip-setting-labels")) {
                 if (graph.getBaseGraph() instanceof Neo4j2Graph) {
-                    // safe, due to the above instanceof
-                    @SuppressWarnings("unchecked")
-                    FramedGraph<Neo4j2Graph> neo4j2Graph = ((FramedGraph<Neo4j2Graph>) graph);
-                    Neo4jGraphManager<?> manager = new Neo4jGraphManager<>(neo4j2Graph);
-                    int i = 0;
-                    for (Vertex v : graph.getVertices()) {
-                        manager.setLabels(v);
-                        i++;
-                        if (i % 10000 == 0) {
-                            neo4j2Graph.getBaseGraph().commit();
-                        }
-                    }
-                    System.err.println("Labelled " + i + " vertices");
+                    SetLabels labeler = new SetLabels();
+                    labeler.execWithOptions(graph, cmdLine);
                 }
             }
         } finally {

ehri-cli/src/main/java/eu/ehri/project/commands/SetLabels.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2022 Data Archiving and Networked Services (an institute of
+ * Koninklijke Nederlandse Akademie van Wetenschappen), King's College London,
+ * Georg-August-Universitaet Goettingen Stiftung Oeffentlichen Rechts
+ *
+ * Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+ * the European Commission - subsequent versions of the EUPL (the "Licence");
+ * You may not use this work except in compliance with the Licence.
+ * You may obtain a copy of the Licence at:
+ *
+ * https://joinup.ec.europa.eu/software/page/eupl
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the Licence is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the Licence for the specific language governing
+ * permissions and limitations under the Licence.
+ */
+
+package eu.ehri.project.commands;
+
+import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.frames.FramedGraph;
+import eu.ehri.project.core.impl.neo4j.Neo4j2Graph;
+import eu.ehri.project.models.annotations.EntityType;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.neo4j.graphdb.*;
+
+import static eu.ehri.project.core.impl.Neo4jGraphManager.BASE_LABEL;
+
+/**
+ * Command for generating the (Neo4j) graph schema.
+ */
+public class SetLabels extends BaseCommand {
+
+    final static String NAME = "set-labels";
+
+    @Override
+    public String getUsage() {
+        return NAME;
+    }
+
+    @Override
+    public String getHelp() {
+        return "Set node labels to match types per the __type property.";
+    }
+
+    @Override
+    protected void setCustomOptions(Options options) {
+        options.addOption(Option.builder("b")
+                .hasArg().type(Integer.class)
+                .longOpt("buffer-size")
+                .desc("Transaction buffer size").build());
+    }
+
+    public static void labelNodes(GraphDatabaseService service, int batchSize, Label baseLabel) {
+        long processedCount = 0;
+        boolean hasMore = true;
+
+        while (hasMore) {
+            try (Transaction tx = service.beginTx()) {
+                // Query that uses SKIP/LIMIT for pagination
+                String query = "MATCH (n) RETURN n SKIP " + processedCount + " " + "LIMIT " + batchSize;
+                ResourceIterator<Node> nodes = tx.execute(query).columnAs("n");
+
+                int batchCount = 0;
+                while (nodes.hasNext()) {
+                    Node node = nodes.next();
+                    node.addLabel(baseLabel);
+                    node.addLabel(Label.label((String) node.getProperty(EntityType.TYPE_KEY)));
+                    batchCount++;
+                }
+
+                // If we processed fewer nodes than the batch size, we're done
+                hasMore = (batchCount == batchSize);
+                processedCount += batchCount;
+
+                tx.commit();
+                System.out.println("Processed " + processedCount + " nodes");
+            }
+        }
+    }
+
+    @Override
+    public int execWithOptions(FramedGraph<?> graph, CommandLine cmdLine) throws Exception {
+        int bufferSize = cmdLine.hasOption("buffer-size")
+                ? Integer.parseInt(cmdLine.getOptionValue("buffer-size"))
+                : 1000;
+        Graph baseGraph = graph.getBaseGraph();
+        if (baseGraph instanceof Neo4j2Graph) {
+            GraphDatabaseService service = ((Neo4j2Graph) baseGraph).getRawGraph();
+            Label baseLabel = Label.label(BASE_LABEL);
+            labelNodes(service, bufferSize, baseLabel);
+        } else {
+            System.err.println("ERROR: Cannot set labels on non Neo4j2-graph");
+        }
+        return 0;
+    }
+}

ehri-cli/src/test/java/eu/ehri/project/commands/GraphSONTest.java

@@ -66,6 +66,7 @@ public void testSaveDumpAndRead() throws Exception {
 
         assertEquals(0, graphSON.execWithOptions(graph1, outCmdLine));
         graph1.shutdown();
+        resetGraph();
 
         assertTrue(temp.exists());
         assertTrue(temp.length() > 0L);

ehri-core/pom.xml

@@ -72,7 +72,9 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.19.1</version>
                 <configuration>
-                    <argLine>-Xmx1024m</argLine>
+                    <argLine>
+                        -Xmx1024m --add-opens java.base/java.lang=ALL-UNNAMED
+                    </argLine>
                     <excludes>
                     </excludes>
                 </configuration>
@@ -127,6 +129,18 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.neo4j.test</groupId>
+            <artifactId>neo4j-harness</artifactId>
+            <version>${neo4j.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-nop</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.neo4j.community</groupId>
             <artifactId>it-test-support</artifactId>
@@ -190,7 +204,7 @@
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
-            <version>1.2.10</version>
+            <version>1.5.6</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

ehri-core/src/main/java/eu/ehri/project/core/impl/Neo4jGraphManager.java

@@ -31,8 +31,8 @@
 import eu.ehri.project.models.EntityClass;
 import eu.ehri.project.models.annotations.EntityType;
 import eu.ehri.project.models.utils.ClassUtils;
-import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Label;
+import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.schema.ConstraintDefinition;
 import org.neo4j.graphdb.schema.IndexDefinition;
 import org.neo4j.graphdb.schema.Schema;
@@ -101,7 +101,6 @@ public Vertex updateVertex(String id, EntityClass type,
 
     @Override
     public void initialize() {
-        createIndicesAndConstraints(graph.getBaseGraph().getRawGraph());
     }
 
     @Override
@@ -127,22 +126,32 @@ public Vertex setLabels(Vertex vertex) {
     }
 
     /**
-     * Create the graph schema
+     * Drop the existing schema.
      *
-     * @param graph the underlying graph service
+     * @param tx the underlying graph transactions
      */
-    public static void createIndicesAndConstraints(GraphDatabaseService graph) {
-        Schema schema = graph.schema();
+    public static void dropIndicesAndConstraints(Transaction tx) {
+        Schema schema = tx.schema();
         for (ConstraintDefinition constraintDefinition : schema.getConstraints()) {
             constraintDefinition.drop();
         }
         for (IndexDefinition indexDefinition : schema.getIndexes()) {
             indexDefinition.drop();
         }
+    }
 
+    /**
+     * Create the graph schema
+     *
+     * @param tx the underlying graph transaction
+     */
+    public static void createIndicesAndConstraints(Transaction tx) {
+        Schema schema = tx.schema();
+        logger.trace("Creating index on property: {} -> {}", BASE_LABEL, EntityType.ID_KEY);
         schema.constraintFor(Label.label(BASE_LABEL))
                 .assertPropertyIsUnique(EntityType.ID_KEY)
                 .create();
+        logger.trace("Creating index on property: {} -> {}", BASE_LABEL, EntityType.TYPE_KEY);
         schema.indexFor(Label.label(BASE_LABEL))
                 .on(EntityType.TYPE_KEY)
                 .create();
@@ -162,8 +171,7 @@ public static void createIndicesAndConstraints(GraphDatabaseService graph) {
 
             Collection<String> uniquePropertyKeys = ClassUtils.getUniquePropertyKeys(cls.getJavaClass());
             for (String unique : uniquePropertyKeys) {
-                logger.trace("Creating constraint on unique property: {} -> {}",
-                        cls.getName(), unique);
+                logger.trace("Creating constraint on unique property: {} -> {}", cls.getName(), unique);
                 schema.constraintFor(Label.label(cls.getName()))
                         .assertPropertyIsUnique(unique)
                         .create();