From 0074ee0e70871c0886edaab7b0dc4ae8e1347617 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 23 Jun 2016 12:09:19 -0400 Subject: [PATCH] DCM: persist per dataverse available file upload mechanisms #3145 --- .../edu/harvard/iq/dataverse/Dataset.java | 17 +++++++++++ .../edu/harvard/iq/dataverse/Dataverse.java | 29 +++++++++++++++++++ .../harvard/iq/dataverse/api/Datasets.java | 12 ++++++++ .../impl/RequestRsyncScriptCommand.java | 6 ++++ .../harvard/iq/dataverse/util/EjbUtil.java | 19 ++++++++++++ .../iq/dataverse/util/json/JsonParser.java | 14 ++++++++- .../iq/dataverse/util/json/JsonPrinter.java | 11 ++++++- .../harvard/iq/dataverse/api/DatasetsIT.java | 26 +++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 15 ++++++++-- 9 files changed, 143 insertions(+), 6 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 2196784a756..817b05bf22d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -40,6 +40,23 @@ uniqueConstraints = @UniqueConstraint(columnNames = {"authority,protocol,identifier,doiseparator"})) public class Dataset extends DvObjectContainer { + public enum FileUploadMechanism { + /** + * Files are uploaded through the GUI or SWORD. + * + * @todo Instead of "STANDARD" should we split out "GUI" and "SWORD" as + * separate mechanisms? What if we add a non-SWORD API endpoint for + * uploads ( https://github.com/IQSS/dataverse/issues/1612 )some day? + */ + STANDARD, + /** + * Files are uploaded via rsync only and upload via any other mechanism + * is not allowed. This option requires setup of the Data Capture + * Module. + */ + RSYNC + }; + // public static final String REDIRECT_URL = "/dataset.xhtml?persistentId="; public static final String TARGET_URL = "/citation?persistentId="; private static final long serialVersionUID = 1L; diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 729681a8ad0..39652f67ee1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -715,4 +715,33 @@ public void setCitationRedirectURL(String citationRedirectURL) { this.citationRedirectURL = citationRedirectURL; } + /** + * Persisted as a colon delimited set of Strings based on the + * Dataset.FileUploadMechanism (i.e. "RSYNC:STANDARD" using TreeSet to keep + * the list sorted. "STANDARD" is returned if null. No inheritance. + * + * @todo Is there a smarter or better way to persist this this information? + * + * @todo Make this field non-nullable if we ever populate the existing + * dataverses with "STANDARD" rather than null. + */ + @Column(nullable = true) + String fileUploadMechanisms; + + public String getFileUploadMechanisms() { + /** + * @todo Remove this is we end up populating all the existing dataverses + * with "STANDARD" rather than null at which point we should make the + * field non-nullable. + */ + if (fileUploadMechanisms == null) { + return Dataset.FileUploadMechanism.STANDARD.toString(); + } + return fileUploadMechanisms; + } + + public void setFileUploadMechanisms(String fileUploadMechanisms) { + this.fileUploadMechanisms = fileUploadMechanisms; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index ee7125ec40f..d9ab4c32c2c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -35,6 +35,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.DDIExportServiceBean; import edu.harvard.iq.dataverse.export.ddi.DdiExportUtil; +import edu.harvard.iq.dataverse.util.EjbUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParseException; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; @@ -48,6 +49,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.EJB; +import javax.ejb.EJBException; import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.json.JsonNumber; @@ -593,6 +595,16 @@ public Response getRsync(@PathParam("identifier") String id) { } } catch (WrappedResponse ex) { return ex.getResponse(); + } catch (EJBException ex) { + /** + * @todo Ask Michael if we can simply have `execCommand` (and the + * GUI equivalent, which is `commandEngine.submit` catch a + * EJBException and/or RuntimeException instead of having this log + * here. Note how DatasetPage, for example, has to catch + * EJBException when issuing CreateDatasetCommand. The engine should + * probably be doing more error handling. + */ + return errorResponse(Response.Status.INTERNAL_SERVER_ERROR, "Unable to get an rsync script: " + EjbUtil.ejbExceptionToString(ex)); } return errorResponse(Response.Status.NOT_FOUND, "An rsync script was not found for dataset id " + id); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java index 65133844625..088c2348531 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestRsyncScriptCommand.java @@ -86,6 +86,12 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { } String message = response.getBody().getObject().getString("status"); logger.info("Message from Data Caputure Module upload request endpoint: " + message); + /** + * @todo Should we persist to the database the fact that we have + * requested a script? That way we could avoid hitting ur.py (upload + * request) over and over since it is preferred that we only hit it + * once. + */ /** * @todo Don't expect to get the script from ur.py (upload request). Go * fetch it from sr.py (script request) after a minute or so. (Cron runs diff --git a/src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java new file mode 100644 index 00000000000..cf337b0a020 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/EjbUtil.java @@ -0,0 +1,19 @@ +package edu.harvard.iq.dataverse.util; + +import javax.ejb.EJBException; + +public class EjbUtil { + + /** + * @param ex An EJBException. + * @return The message from the root cause of the EJBException as a String. + */ + public static String ejbExceptionToString(EJBException ex) { + Throwable cause = ex; + // An EJBException always has a cause. It won't be null. + while (cause.getCause() != null) { + cause = cause.getCause(); + } + return cause.getLocalizedMessage(); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index ebbe1117c3b..b201cd87dc9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -29,6 +29,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.TreeSet; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObject; @@ -83,7 +84,18 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException { } dv.setDataverseContacts(dvContactList); } - + if (jobj.containsKey("fileUploadMechanismsEnabled")) { + JsonArray mechs = jobj.getJsonArray("fileUploadMechanismsEnabled"); + Set mechsToPersist = new TreeSet<>(); + for (JsonValue mech : mechs) { + JsonString jsonString = (JsonString) mech; + mechsToPersist.add(jsonString.getString()); + } + if (!mechs.isEmpty()) { + dv.setFileUploadMechanisms(String.join(":", mechsToPersist)); + } + } + /* We decided that subject is not user set, but gotten from the subject of the dataverse's datasets - leavig this code in for now, in case we need to go back to it at some point diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index dd9957297e7..daaea3d6175 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -37,7 +37,6 @@ import java.util.TreeSet; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; -import java.math.BigDecimal; import java.util.Collection; import java.util.Deque; import java.util.LinkedList; @@ -158,6 +157,15 @@ public static JsonObjectBuilder json( DataverseRole role ) { } public static JsonObjectBuilder json( Dataverse dv ) { + /** @todo refactor this fileUploadMechanisms stuff into its own method */ + JsonArrayBuilder fileUploadMechanismsEnabledArray = Json.createArrayBuilder(); + /** @todo Each element in the array should be an object with a description taken from the bundle. */ + String fileUploadMechanismsEnabledString = dv.getFileUploadMechanisms(); + if (fileUploadMechanismsEnabledString != null) { + for (String mech : fileUploadMechanismsEnabledString.split(":")) { + fileUploadMechanismsEnabledArray.add(mech); + } + } JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dv.getId() ) .add("alias", dv.getAlias()) @@ -165,6 +173,7 @@ public static JsonObjectBuilder json( Dataverse dv ) { .add("affiliation", dv.getAffiliation()) .add("dataverseContacts", json(dv.getDataverseContacts())) .add("permissionRoot", dv.isPermissionRoot()) + .add("fileUploadMechanismsEnabled", fileUploadMechanismsEnabledArray) .add("description", dv.getDescription()); if ( dv.getOwner() != null ) { bld.add("ownerId", dv.getOwner().getId()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index fa0eb5c7961..15f09ed24f7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -10,6 +10,9 @@ import com.jayway.restassured.http.ContentType; import static junit.framework.Assert.assertEquals; import com.jayway.restassured.path.json.JsonPath; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.util.Arrays; import java.util.List; import java.util.Map; import javax.json.Json; @@ -18,6 +21,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.fail; public class DatasetsIT { @@ -43,6 +47,10 @@ public void testCreateDataset() { Response createDataverse1Response = UtilIT.createRandomDataverse(apiToken1); createDataverse1Response.prettyPrint(); dataverseAlias1 = UtilIT.getAliasFromResponse(createDataverse1Response); + createDataverse1Response.then().assertThat() + .statusCode(201) + .body("data.alias", equalTo(dataverseAlias1)) + .body("data.fileUploadMechanismsEnabled[0]", equalTo(Dataset.FileUploadMechanism.STANDARD.toString())); Response createDataset1Response = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias1, apiToken1); createDataset1Response.prettyPrint(); @@ -66,10 +74,24 @@ public void testCreateDatasetWithDcmDependency() { String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); long userId = JsonPath.from(createUser.body().asString()).getLong("data.authenticatedUser.id"); + Response urlConfigured = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/admin/settings/" + SettingsServiceBean.Key.DataCaptureModuleUrl.toString()); + if (urlConfigured.getStatusCode() != 200) { + fail(SettingsServiceBean.Key.DataCaptureModuleUrl + " has not been not configured. This test cannot run without it."); + } - Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + /** + * @todo Query system to see which file upload mechanisms are available. + */ + String dataverseAlias = UtilIT.getRandomIdentifier(); + List fileUploadMechanismsEnabled = Arrays.asList(Dataset.FileUploadMechanism.RSYNC.toString()); + Response createDataverseResponse = UtilIT.createDataverse(dataverseAlias, fileUploadMechanismsEnabled, apiToken); createDataverseResponse.prettyPrint(); - String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + createDataverseResponse.then().assertThat() + .statusCode(201) + .body("data.alias", equalTo(dataverseAlias)) + .body("data.fileUploadMechanismsEnabled[0]", equalTo(Dataset.FileUploadMechanism.RSYNC.toString())); /** * @todo Make this configurable at runtime similar to diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index f6ea51ac41c..c2d481b9165 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -20,6 +20,7 @@ import static com.jayway.restassured.RestAssured.given; import static com.jayway.restassured.path.xml.XmlPath.from; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.util.ArrayList; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -85,7 +86,7 @@ private static String getRandomUsername() { return "user" + getRandomIdentifier().substring(0, 8); } - private static String getRandomIdentifier() { + static String getRandomIdentifier() { return UUID.randomUUID().toString().substring(0, 8); } @@ -133,16 +134,21 @@ public static Response getServiceDocument(String apiToken) { return response; } - static Response createDataverse(String alias, String apiToken) { + static Response createDataverse(String alias, List fileUploadMechanismsEnabled, String apiToken) { JsonArrayBuilder contactArrayBuilder = Json.createArrayBuilder(); contactArrayBuilder.add(Json.createObjectBuilder().add("contactEmail", getEmailFromUserName(getRandomIdentifier()))); JsonArrayBuilder subjectArrayBuilder = Json.createArrayBuilder(); subjectArrayBuilder.add("Other"); + JsonArrayBuilder fileUploadMechanismEnabled = Json.createArrayBuilder(); + fileUploadMechanismsEnabled.stream().forEach((mechanism) -> { + fileUploadMechanismEnabled.add(mechanism); + }); JsonObject dvData = Json.createObjectBuilder() .add("alias", alias) .add("name", alias) .add("dataverseContacts", contactArrayBuilder) .add("dataverseSubjects", subjectArrayBuilder) + .add("fileUploadMechanismsEnabled", fileUploadMechanismEnabled) .build(); Response createDataverseResponse = given() .body(dvData.toString()).contentType(ContentType.JSON) @@ -150,6 +156,11 @@ static Response createDataverse(String alias, String apiToken) { return createDataverseResponse; } + static Response createDataverse(String alias, String apiToken) { + List fileUploadMechanismsEnabled = new ArrayList<>(); + return createDataverse(alias, fileUploadMechanismsEnabled, apiToken); + } + static Response createRandomDataverse(String apiToken) { String alias = getRandomIdentifier(); return createDataverse(alias, apiToken);