DCM: persist per dataverse available file upload mechanisms IQSS#3145

src/main/java/edu/harvard/iq/dataverse/Dataset.java

@@ -40,6 +40,23 @@
         uniqueConstraints = @UniqueConstraint(columnNames = {"authority,protocol,identifier,doiseparator"}))
 public class Dataset extends DvObjectContainer {
 
+    public enum FileUploadMechanism {
+        /**
+         * Files are uploaded through the GUI or SWORD.
+         *
+         * @todo Instead of "STANDARD" should we split out "GUI" and "SWORD" as
+         * separate mechanisms? What if we add a non-SWORD API endpoint for
+         * uploads ( https://github.com/IQSS/dataverse/issues/1612 )some day?
+         */
+        STANDARD,
+        /**
+         * Files are uploaded via rsync only and upload via any other mechanism
+         * is not allowed. This option requires setup of the Data Capture
+         * Module.
+         */
+        RSYNC
+    };
+
 //    public static final String REDIRECT_URL = "/dataset.xhtml?persistentId=";
     public static final String TARGET_URL = "/citation?persistentId=";
     private static final long serialVersionUID = 1L;

src/main/java/edu/harvard/iq/dataverse/Dataverse.java

@@ -715,4 +715,33 @@ public void setCitationRedirectURL(String citationRedirectURL) {
         this.citationRedirectURL = citationRedirectURL;
     }    
 
+    /**
+     * Persisted as a colon delimited set of Strings based on the
+     * Dataset.FileUploadMechanism (i.e. "RSYNC:STANDARD" using TreeSet to keep
+     * the list sorted. "STANDARD" is returned if null. No inheritance.
+     *
+     * @todo Is there a smarter or better way to persist this this information?
+     *
+     * @todo Make this field non-nullable if we ever populate the existing
+     * dataverses with "STANDARD" rather than null.
+     */
+    @Column(nullable = true)
+    String fileUploadMechanisms;
+
+    public String getFileUploadMechanisms() {
+        /**
+         * @todo Remove this is we end up populating all the existing dataverses
+         * with "STANDARD" rather than null at which point we should make the
+         * field non-nullable.
+         */
+        if (fileUploadMechanisms == null) {
+            return Dataset.FileUploadMechanism.STANDARD.toString();
+        }
+        return fileUploadMechanisms;
+    }
+
+    public void setFileUploadMechanisms(String fileUploadMechanisms) {
+        this.fileUploadMechanisms = fileUploadMechanisms;
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -35,6 +35,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
 import edu.harvard.iq.dataverse.export.DDIExportServiceBean;
 import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil;
+import edu.harvard.iq.dataverse.util.EjbUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*;
@@ -48,6 +49,7 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.ejb.EJB;
+import javax.ejb.EJBException;
 import javax.json.Json;
 import javax.json.JsonArrayBuilder;
 import javax.json.JsonNumber;
@@ -593,6 +595,16 @@ public Response getRsync(@PathParam("identifier") String id) {
             }
         } catch (WrappedResponse ex) {
             return ex.getResponse();
+        } catch (EJBException ex) {
+            /**
+             * @todo Ask Michael if we can simply have `execCommand` (and the
+             * GUI equivalent, which is `commandEngine.submit` catch a
+             * EJBException and/or RuntimeException instead of having this log
+             * here. Note how DatasetPage, for example, has to catch
+             * EJBException when issuing CreateDatasetCommand. The engine should
+             * probably be doing more error handling.
+             */
+            return errorResponse(Response.Status.INTERNAL_SERVER_ERROR, "Unable to get an rsync script: " + EjbUtil.ejbExceptionToString(ex));
         }
         return errorResponse(Response.Status.NOT_FOUND, "An rsync script was not found for dataset id " + id);
     }

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java

@@ -86,6 +86,12 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException {
         }
         String message = response.getBody().getObject().getString("status");
         logger.info("Message from Data Caputure Module upload request endpoint: " + message);
+        /**
+         * @todo Should we persist to the database the fact that we have
+         * requested a script? That way we could avoid hitting ur.py (upload
+         * request) over and over since it is preferred that we only hit it
+         * once.
+         */
         /**
          * @todo Don't expect to get the script from ur.py (upload request). Go
          * fetch it from sr.py (script request) after a minute or so. (Cron runs

src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java

@@ -0,0 +1,19 @@
+package edu.harvard.iq.dataverse.util;
+
+import javax.ejb.EJBException;
+
+public class EjbUtil {
+
+    /**
+     * @param ex An EJBException.
+     * @return The message from the root cause of the EJBException as a String.
+     */
+    public static String ejbExceptionToString(EJBException ex) {
+        Throwable cause = ex;
+        // An EJBException always has a cause. It won't be null.
+        while (cause.getCause() != null) {
+            cause = cause.getCause();
+        }
+        return cause.getLocalizedMessage();
+    }
+}

src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java

@@ -29,6 +29,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
+import java.util.TreeSet;
 import javax.json.Json;
 import javax.json.JsonArray;
 import javax.json.JsonObject;
@@ -83,7 +84,18 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
             }
             dv.setDataverseContacts(dvContactList);
         }
-
+        if (jobj.containsKey("fileUploadMechanismsEnabled")) {
+            JsonArray mechs = jobj.getJsonArray("fileUploadMechanismsEnabled");
+            Set<String> mechsToPersist = new TreeSet<>();
+            for (JsonValue mech : mechs) {
+                JsonString jsonString = (JsonString) mech;
+                mechsToPersist.add(jsonString.getString());
+            }
+            if (!mechs.isEmpty()) {
+                dv.setFileUploadMechanisms(String.join(":", mechsToPersist));
+            }
+        }
+
         /*  We decided that subject is not user set, but gotten from the subject of the dataverse's
             datasets - leavig this code in for now, in case we need to go back to it at some point
         

src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java

@@ -37,7 +37,6 @@
 import java.util.TreeSet;
 
 import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
-import java.math.BigDecimal;
 import java.util.Collection;
 import java.util.Deque;
 import java.util.LinkedList;
@@ -158,13 +157,23 @@ public static JsonObjectBuilder json( DataverseRole role ) {
 	}
 
 	public static JsonObjectBuilder json( Dataverse dv ) {
+		/** @todo refactor this fileUploadMechanisms stuff into its own method */
+		JsonArrayBuilder fileUploadMechanismsEnabledArray = Json.createArrayBuilder();
+                /** @todo Each element in the array should be an object with a description taken from the bundle. */
+                String fileUploadMechanismsEnabledString = dv.getFileUploadMechanisms();
+                if (fileUploadMechanismsEnabledString != null) {
+                    for (String mech : fileUploadMechanismsEnabledString.split(":")) {
+                        fileUploadMechanismsEnabledArray.add(mech);
+                    }
+                }
 		JsonObjectBuilder bld = jsonObjectBuilder()
 						.add("id", dv.getId() )
 						.add("alias", dv.getAlias()) 
 						.add("name", dv.getName())
                                                 .add("affiliation", dv.getAffiliation())
                                                 .add("dataverseContacts", json(dv.getDataverseContacts()))
 						.add("permissionRoot", dv.isPermissionRoot())
+						.add("fileUploadMechanismsEnabled", fileUploadMechanismsEnabledArray)
 						.add("description", dv.getDescription());
 		if ( dv.getOwner() != null ) {
 			bld.add("ownerId", dv.getOwner().getId());

src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java

@@ -10,6 +10,9 @@
 import com.jayway.restassured.http.ContentType;
 import static junit.framework.Assert.assertEquals;
 import com.jayway.restassured.path.json.JsonPath;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import javax.json.Json;
@@ -18,6 +21,7 @@
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assert.fail;
 
 public class DatasetsIT {
 
@@ -43,6 +47,10 @@ public void testCreateDataset() {
         Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken1);
         createDataverse1Response.prettyPrint();
         dataverseAlias1 = UtilIT.getAliasFromResponse(createDataverse1Response);
+        createDataverse1Response.then().assertThat()
+                .statusCode(201)
+                .body("data.alias", equalTo(dataverseAlias1))
+                .body("data.fileUploadMechanismsEnabled[0]", equalTo(Dataset.FileUploadMechanism.STANDARD.toString()));
 
         Response createDataset1Response = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias1, apiToken1);
         createDataset1Response.prettyPrint();
@@ -66,10 +74,24 @@ public void testCreateDatasetWithDcmDependency() {
         String username = UtilIT.getUsernameFromResponse(createUser);
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
         long userId = JsonPath.from(createUser.body().asString()).getLong("data.authenticatedUser.id");
+        Response urlConfigured = given()
+                .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/admin/settings/" + SettingsServiceBean.Key.DataCaptureModuleUrl.toString());
+        if (urlConfigured.getStatusCode() != 200) {
+            fail(SettingsServiceBean.Key.DataCaptureModuleUrl + " has not been not configured. This test cannot run without it.");
+        }
 
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        /**
+         * @todo Query system to see which file upload mechanisms are available.
+         */
+        String dataverseAlias = UtilIT.getRandomIdentifier();
+        List<String> fileUploadMechanismsEnabled = Arrays.asList(Dataset.FileUploadMechanism.RSYNC.toString());
+        Response createDataverseResponse = UtilIT.createDataverse(dataverseAlias, fileUploadMechanismsEnabled, apiToken);
         createDataverseResponse.prettyPrint();
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+        createDataverseResponse.then().assertThat()
+                .statusCode(201)
+                .body("data.alias", equalTo(dataverseAlias))
+                .body("data.fileUploadMechanismsEnabled[0]", equalTo(Dataset.FileUploadMechanism.RSYNC.toString()));
 
         /**
          * @todo Make this configurable at runtime similar to

src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java

@@ -20,6 +20,7 @@
 import static com.jayway.restassured.RestAssured.given;
 import static com.jayway.restassured.path.xml.XmlPath.from;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import java.util.ArrayList;
 import java.util.List;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -85,7 +86,7 @@ private static String getRandomUsername() {
         return "user" + getRandomIdentifier().substring(0, 8);
     }
 
-    private static String getRandomIdentifier() {
+    static String getRandomIdentifier() {
         return UUID.randomUUID().toString().substring(0, 8);
     }
 
@@ -133,23 +134,33 @@ public static Response getServiceDocument(String apiToken) {
         return response;
     }
 
-    static Response createDataverse(String alias, String apiToken) {
+    static Response createDataverse(String alias, List<String> fileUploadMechanismsEnabled, String apiToken) {
         JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder();
         contactArrayBuilder.add(Json.createObjectBuilder().add("contactEmail", getEmailFromUserName(getRandomIdentifier())));
         JsonArrayBuilder subjectArrayBuilder = Json.createArrayBuilder();
         subjectArrayBuilder.add("Other");
+        JsonArrayBuilder fileUploadMechanismEnabled = Json.createArrayBuilder();
+        fileUploadMechanismsEnabled.stream().forEach((mechanism) -> {
+            fileUploadMechanismEnabled.add(mechanism);
+        });
         JsonObject dvData = Json.createObjectBuilder()
                 .add("alias", alias)
                 .add("name", alias)
                 .add("dataverseContacts", contactArrayBuilder)
                 .add("dataverseSubjects", subjectArrayBuilder)
+                .add("fileUploadMechanismsEnabled", fileUploadMechanismEnabled)
                 .build();
         Response createDataverseResponse = given()
                 .body(dvData.toString()).contentType(ContentType.JSON)
                 .when().post("/api/dataverses/:root?key=" + apiToken);
         return createDataverseResponse;
     }
 
+    static Response createDataverse(String alias, String apiToken) {
+        List<String> fileUploadMechanismsEnabled = new ArrayList<>();
+        return createDataverse(alias, fileUploadMechanismsEnabled, apiToken);
+    }
+
     static Response createRandomDataverse(String apiToken) {
         String alias = getRandomIdentifier();
         return createDataverse(alias, apiToken);