Add PID support, with lots of underlying changes

Dockerfile

@@ -1,5 +1,5 @@
 # Docker file for EHRI backend web service
-FROM docker.io/neo4j:3.5.28
+FROM docker.io/neo4j:3.5.35
 
 # Set git commit build revision as a label which
 # we can inspect to figure out the image version.

build/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>eu.ehri-project</groupId>
         <artifactId>ehri-data</artifactId>
-        <version>0.15.5</version>
+        <version>0.16.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>build</artifactId>
@@ -70,7 +70,7 @@
                             <include>xerces:xercesImpl</include>
                             <include>xml-apis:xml-apis</include>
                             <include>com.typesafe:config</include>
-                            <include>com.flipkart.zjsonpatch:zjsonpatch</include>
+                            <include>io.github.vishwakarma:zjsonpatch</include>
                             <include>org.apache.commons:commons-collections4</include>
 
                             <!-- GraphQL deps -->

ehri-cli/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>eu.ehri-project</groupId>
         <artifactId>ehri-data</artifactId>
-        <version>0.15.5</version>
+        <version>0.16.0-SNAPSHOT</version>
     </parent>
     <artifactId>ehri-cli</artifactId>
     <name>Command Line Tools</name>

ehri-cli/src/main/java/eu/ehri/project/commands/ImportCommand.java

@@ -22,16 +22,19 @@
 import com.google.common.base.Charsets;
 import com.google.common.collect.Lists;
 import com.tinkerpop.frames.FramedGraph;
+import eu.ehri.project.IdGeneratorFactory;
 import eu.ehri.project.acl.SystemScope;
 import eu.ehri.project.core.GraphManager;
 import eu.ehri.project.core.GraphManagerFactory;
 import eu.ehri.project.importers.ImportLog;
 import eu.ehri.project.importers.ImportOptions;
+import eu.ehri.project.importers.PreImportCallback;
 import eu.ehri.project.importers.base.ItemImporter;
 import eu.ehri.project.importers.base.SaxXmlHandler;
 import eu.ehri.project.importers.managers.SaxImportManager;
 import eu.ehri.project.models.UserProfile;
 import eu.ehri.project.models.base.PermissionScope;
+import eu.ehri.project.models.idgen.RandomIdGenerator;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
@@ -51,6 +54,8 @@ public abstract class ImportCommand extends BaseCommand {
     private final Class<? extends SaxXmlHandler> handler;
     private final Class<? extends ItemImporter<?, ?>> importer;
 
+    private static final RandomIdGenerator idGenerator = IdGeneratorFactory.getIdGenerator();
+
     public ImportCommand(Class<? extends SaxXmlHandler> handler, Class<? extends ItemImporter<?, ?>> importer) {
         this.handler = handler;
         this.importer = importer;
@@ -114,7 +119,7 @@ protected void setCustomOptions(Options options) {
 
     @Override
     public int execWithOptions(FramedGraph<?> graph,
-            CommandLine cmdLine) throws Exception {
+                               CommandLine cmdLine) throws Exception {
 
         GraphManager manager = GraphManagerFactory.getInstance(graph);
 
@@ -159,10 +164,11 @@ public int execWithOptions(FramedGraph<?> graph,
                     .withLang(lang);
 
             ImportLog log = SaxImportManager.create(graph, scope, user,
-                    importer,
-                    handler,
-                    options,
-                    Lists.newArrayList())
+                            importer,
+                            handler,
+                            options
+                    )
+                    .withPreCallback(PreImportCallback.generatePid(idGenerator))
                     .importFiles(filePaths, logMessage);
             System.out.println(log);
 

ehri-cli/src/test/java/eu/ehri/project/commands/EntityAddTest.java

@@ -34,7 +34,7 @@ public class EntityAddTest extends AbstractFixtureTest {
     @Test
     public void testEntityAdd() throws Exception {
         String[] args = new String[]{EntityClass.COUNTRY.getName(),
-                "-P", "name=Elbonia", "-P", "identifier=el",
+                "-P", "name=Elbonia", "-P", "identifier=el", "-P", "__pid=el-1234",
                 "--user", "mike"};
         EntityAdd ua = new EntityAdd();
         CommandLine cmdLine = ua.getCmdLine(args);
@@ -46,7 +46,7 @@ public void testEntityAdd() throws Exception {
     @Test
     public void testEntityAddWithBadType() throws Exception {
         String[] args = new String[]{EntityClass.ADDRESS.getName(),
-                "-P", "name=Elbonia", "-P", "identifier=el",
+                "-P", "name=Elbonia", "-P", "identifier=el", "-P", "__pid=el-1234",
                 "--user", "mike"};
         EntityAdd ua = new EntityAdd();
         CommandLine cmdLine = ua.getCmdLine(args);

ehri-core/README.md

@@ -1,4 +1,20 @@
 ehri-core
 =========
 
-Classes defining models and handling business logic such as persistence, access control, and permissions validation. 
+Classes defining models and handling business logic such as persistence, access control, and permissions validation. 
+
+There are some semantics for managing generic property-graph data that this package uses:
+
+There are two reserved properties for system use:
+
+ * `__id`
+ * `__type`
+
+Other properties beginning with two underscores (`__`) are "initialisation properties" and can 
+only be set via generic means when the vertex is first created. When a vertex is updated, all
+initialisation properties are ignored. When a vertex is serialized to a `Bundle` object, all
+initialisation properties are included in the data section (except reserved properties) but
+are ignored in object comparison.
+
+Properties beginning with a single underscore are considered metadata and are ignored in object
+comparison, but are otherwise handled like regular vertex properties.

ehri-core/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>eu.ehri-project</groupId>
         <artifactId>ehri-data</artifactId>
-        <version>0.15.5</version>
+        <version>0.16.0-SNAPSHOT</version>
     </parent>
     <artifactId>ehri-core</artifactId>
     <packaging>jar</packaging>
@@ -167,9 +167,9 @@
             <version>1.21</version>
         </dependency>
         <dependency>
-            <groupId>com.flipkart.zjsonpatch</groupId>
+            <groupId>io.github.vishwakarma</groupId>
             <artifactId>zjsonpatch</artifactId>
-            <version>0.4.11</version>
+            <version>0.5.0</version>
         </dependency>
         <dependency>
             <groupId>eu.ehri-project</groupId>

ehri-core/src/main/java/eu/ehri/project/acl/AclManager.java

@@ -719,7 +719,7 @@ private boolean isInScope(PermissionGrant grant) {
     /**
      * Get the content type with the given id.
      */
-    private ContentTypes getContentType(EntityClass type) {
+    public ContentTypes getContentType(EntityClass type) {
         try {
             return ContentTypes.withName(type.getName());
         } catch (NoSuchElementException e) {

ehri-core/src/main/java/eu/ehri/project/api/Api.java

@@ -69,6 +69,17 @@ interface BatchCallback {
         void onItemDeleted(int itemNumber, String id);
     }
 
+    /**
+     * Get any item by ID or PID.
+     *
+     * @param id the item's ID
+     * @param usePid if the ID is a PID
+     * @return an {@link Accessible} item
+     * @throws ItemNotFound if the item is not found
+     * @throws InvalidIdentifierError if the item is not a PID type
+     */
+    Accessible getAny(String id, boolean usePid) throws ItemNotFound, InvalidIdentifierError;
+
     /**
      * Fetch an item, as a user. This only provides access control.
      *
@@ -80,6 +91,17 @@ interface BatchCallback {
      */
     <E extends Accessible> E get(String id, Class<E> cls) throws ItemNotFound;
 
+    /**
+     * Fetch an item by PID, as a user. This only provides access control.
+     *
+     * @param pid the item persistent ID
+     * @param cls the item's class
+     * @param <E> the type of entity
+     * @return the given framed vertex
+     * @throws ItemNotFound if the item does not exist
+     */
+    <E extends Accessible> E getByPid(String pid, Class<E> cls) throws ItemNotFound, InvalidIdentifierError;
+
     /**
      * Update an object bundle, also updating dependent items.
      *
@@ -456,7 +478,7 @@ InheritedItemPermissionSet setItemPermissions(Accessible item, Accessor accessor
         /**
          * Set members of a given group, removing existing members not in this set.
          *
-         * @param group the group
+         * @param group         the group
          * @param usersOrGroups the users or groups to add
          * @throws PermissionDenied if the grant action is not available to the user
          */

ehri-core/src/main/java/eu/ehri/project/api/impl/ApiImpl.java

@@ -2,6 +2,7 @@
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
+import com.tinkerpop.blueprints.Vertex;
 import com.tinkerpop.frames.FramedGraph;
 import eu.ehri.project.acl.*;
 import eu.ehri.project.api.*;
@@ -134,16 +135,52 @@ public <E extends Accessible> E get(String id, Class<E> cls) throws ItemNotFound
             }
             return item;
         } catch (ItemNotFound e) {
-            // If the item has been deleted, augment the exception with a deletion time...
-            Optional<Version> versionOpt = versionManager.versionAtDeletion(id);
-            if (versionOpt.isPresent()) {
-                Version version = versionOpt.get();
-                SystemEvent event = version.getTriggeringEvent();
-                throw e.withDeletedAt(event.getTimestamp());
-            } else {
-                throw e;
+            throw augmentNotFoundInfo(id, false, e);
+        }
+    }
+
+    @Override
+    public <E extends Accessible> E getByPid(String pid, Class<E> cls) throws ItemNotFound, InvalidIdentifierError {
+        try {
+            Optional<Vertex> opt = manager.getVertex(Ontology.PID_KEY, pid);
+            if (!opt.isPresent()) {
+                throw new ItemNotFound(pid);
+            }
+            E item = graph.frame(opt.get(), cls);
+            if (!PersistentIdentifiable.class.isAssignableFrom(manager.getEntityClass(item).getJavaClass())) {
+                throw new InvalidIdentifierError(pid);
+            }
+            if (!aclManager.canAccess(item, accessor)) {
+                throw new ItemNotFound(pid);
+            }
+            return item;
+        } catch (ItemNotFound e) {
+            throw augmentNotFoundInfo(pid, true, e);
+        }
+    }
+
+    @Override
+    public Accessible getAny(String id, boolean usePid) throws ItemNotFound, InvalidIdentifierError {
+        try {
+            Accessible item = usePid ? getByPid(id, Accessible.class) : get(id, Accessible.class);
+            if (!Accessible.class.isAssignableFrom(manager.getEntityClass(item).getJavaClass())) {
+                throw new ItemNotFound(id);
+            } else if (!aclManager.getContentTypeFilterFunction().compute(item.asVertex())) {
+                throw new ItemNotFound(id);
+            } else if (!aclManager.canAccess(item, accessor)) {
+                throw new ItemNotFound(id);
             }
+            return item;
+        } catch (ItemNotFound e) {
+            throw augmentNotFoundInfo(id, usePid, e);
         }
+
+    }
+
+    private ItemNotFound augmentNotFoundInfo(String id, boolean usePid, ItemNotFound e) {
+        // If the item has been deleted, augment the exception with a deletion time...
+        Optional<Version> versionOpt = versionManager.versionAtDeletion(id, usePid);
+        return versionOpt.map(v -> addTimestamp(e, v)).orElse(e);
     }
 
     @Override
@@ -511,7 +548,7 @@ public List<String> deleteChildren(String parentId, boolean all, boolean version
         Iterable<Accessible> children = all
                 ? scope.getAllContainedItems()
                 : scope.getContainedItems();
-        List<String> ids = StreamSupport
+        final List<String> ids = StreamSupport
                 .stream(children.spliterator(), false)
                 .map(Entity::getId)
                 .sorted()
@@ -662,4 +699,9 @@ private void ensureCanModifyGroupMembership(Group group, Accessor user, Accessor
             helper.checkEntityPermission(group, grantee, PermissionType.UPDATE);
         }
     }
+
+    private ItemNotFound addTimestamp(ItemNotFound e, Version version) {
+        SystemEvent event = version.getTriggeringEvent();
+        return e.withDeletedAt(event.getTimestamp());
+    }
 }

ehri-core/src/main/java/eu/ehri/project/core/GraphManager.java

@@ -28,6 +28,7 @@
 import eu.ehri.project.models.base.Entity;
 
 import java.util.Map;
+import java.util.Optional;
 
 /**
  * An abstraction over the basic {@link FramedGraph} class
@@ -100,6 +101,15 @@ public interface GraphManager {
      */
     Vertex getVertex(String id) throws ItemNotFound;
 
+    /**
+     * Get a node with the given key/value pair.
+     *
+     * @param key   The vertex's property
+     * @param value The vertex's property value
+     * @return The vertex, if found
+     */
+    Optional<Vertex> getVertex(String key, Object value);
+
     /**
      * Get a node with the given ID, and frame it with the given interface
      * class.
@@ -154,6 +164,17 @@ public interface GraphManager {
      */
     CloseableIterable<Vertex> getVertices(Iterable<String> ids);
 
+    /**
+     * Get a CloseableIterable of vertices where the given property has the
+     * provided values. If a vertex is not found with an input value the
+     * result in the returned iterable will be null.
+     *
+     * @param key    the property key
+     * @param values property values
+     * @return an iterable of vertices with the given values for the provided property
+     */
+    CloseableIterable<Vertex> getVertices(String key, Iterable<?> values);
+
     /**
      * Get a CloseableIterable of vertices with the given type, and the given
      * key/value property.

ehri-core/src/main/java/eu/ehri/project/core/impl/BlueprintsGraphManager.java

@@ -39,6 +39,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Optional;
 
 /**
  * Implementation of GraphManager that uses a single index to manage all nodes.
@@ -145,6 +146,16 @@ public Vertex getVertex(String id) throws ItemNotFound {
         }
     }
 
+    @Override
+    public Optional<Vertex> getVertex(String key, Object value) {
+        Preconditions.checkNotNull(key, "attempt to fetch vertex with a null property key");
+        try {
+            return Optional.of(graph.getVertices(key, value).iterator().next());
+        } catch (NoSuchElementException e) {
+            return Optional.empty();
+        }
+    }
+
     @Override
     public CloseableIterable<Vertex> getVertices(EntityClass type) {
         return new WrappingCloseableIterable<>(
@@ -163,6 +174,13 @@ public CloseableIterable<Vertex> getVertices(Iterable<String> ids) {
         return new WrappingCloseableIterable<>(verts);
     }
 
+    @Override
+    public CloseableIterable<Vertex> getVertices(String key, Iterable<?> values) {
+        Iterable<Vertex> verts = Iterables
+                .transform(values, value -> getVertex(key, value).orElse(null));
+        return new WrappingCloseableIterable<>(verts);
+    }
+
     @Override
     public CloseableIterable<Vertex> getVertices(String key, Object value, EntityClass type) {
         // NB: This is rather annoying.
@@ -178,10 +196,9 @@ public CloseableIterable<Vertex> getVertices(String key, Object value, EntityCla
     @Override
     public Vertex createVertex(String id, EntityClass type,
             Map<String, ?> data) throws IntegrityError {
-        Preconditions
-                .checkNotNull(id, "null vertex ID given for item creation");
+        Preconditions.checkNotNull(id, "null vertex ID given for item creation");
         Map<String, ?> indexData = getVertexData(id, type, data);
-        checkExists(id);
+        assertDoesNotExist(id);
         Vertex node = graph.addVertex(null);
         for (Map.Entry<String, ?> entry : indexData.entrySet()) {
             if (entry.getValue() == null)
@@ -262,9 +279,9 @@ private <E extends Element> void addProperties(E item, Map<String, ?> data) {
         }
     }
 
-    private void checkExists(String id) throws IntegrityError {
+    private void assertDoesNotExist(String id) throws IntegrityError {
         if (exists(id)) {
-            throw new IntegrityError(id);
+            throw new IntegrityError("Integrity error for id value: " + id);
         }
     }