From 07b34b0b2edcbaec727310ff3c49e09430a8c06a Mon Sep 17 00:00:00 2001 From: Robert Treacy Date: Thu, 29 Jul 2021 17:35:26 -0400 Subject: [PATCH 001/232] initial commit impements POST-redirect-GET for DP Creator tool POST is currently done on server, gets a redirect response, and GETs the new location in the browser Need to change the way the base context is gotten for POST, as in the GET code, it always uses the extenal tool url as provided in the configuration - the redirect use be a different context than the configured tool url. --- .../externaltools/ExternalToolHandler.java | 153 +++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index a4a51666cc5..ff616d08a4f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -8,14 +8,26 @@ import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.io.IOException; import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObject; import javax.json.JsonReader; +import javax.ws.rs.HttpMethod; /** * Handles an operation on a specific file. Requires a file id in order to be @@ -33,6 +45,8 @@ public class ExternalToolHandler { private ApiToken apiToken; private String localeCode; + private String requestMethod; + private String toolContext; /** * File level tool @@ -44,6 +58,7 @@ public class ExternalToolHandler { */ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToken apiToken, FileMetadata fileMetadata, String localeCode) { this.externalTool = externalTool; + toolContext = externalTool.getToolUrl(); if (dataFile == null) { String error = "A DataFile is required."; logger.warning("Error in ExternalToolHandler constructor: " + error); @@ -106,6 +121,16 @@ public String getQueryParametersForUrl() { // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. public String getQueryParametersForUrl(boolean preview) { + requestMethod = requestMethod(); + if (requestMethod().equals(HttpMethod.POST)){ + try { + return getFormData(); + } catch (IOException ex) { + Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); + } catch (InterruptedException ex) { + Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); + } + } String toolParameters = externalTool.getToolParameters(); JsonReader jsonReader = Json.createReader(new StringReader(toolParameters)); JsonObject obj = jsonReader.readObject(); @@ -183,9 +208,135 @@ private String getQueryParam(String key, String value) { } return null; } + + private String getFormDataValue(String key, String value) { + ReservedWord reservedWord = ReservedWord.fromString(value); + switch (reservedWord) { + case FILE_ID: + // getDataFile is never null for file tools because of the constructor + return ""+getDataFile().getId(); + case FILE_PID: + GlobalId filePid = getDataFile().getGlobalId(); + if (filePid != null) { + return ""+getDataFile().getGlobalId(); + } + break; + case SITE_URL: + return ""+SystemConfig.getDataverseSiteUrlStatic(); + case API_TOKEN: + String apiTokenString = null; + ApiToken theApiToken = getApiToken(); + if (theApiToken != null) { + apiTokenString = theApiToken.getTokenString(); + return "" + apiTokenString; + } + break; + case DATASET_ID: + return "" + dataset.getId(); + case DATASET_PID: + return "" + dataset.getGlobalId().asString(); + case DATASET_VERSION: + String versionString = null; + if(fileMetadata!=null) { //true for file case + versionString = fileMetadata.getDatasetVersion().getFriendlyVersionNumber(); + } else { //Dataset case - return the latest visible version (unless/until the dataset case allows specifying a version) + if (getApiToken() != null) { + versionString = dataset.getLatestVersion().getFriendlyVersionNumber(); + } else { + versionString = dataset.getLatestVersionForCopy().getFriendlyVersionNumber(); + } + } + if (("DRAFT").equals(versionString)) { + versionString = ":draft"; // send the token needed in api calls that can be substituted for a numeric + // version. + } + return "" + versionString; + case FILE_METADATA_ID: + if(fileMetadata!=null) { //true for file case + return "" + fileMetadata.getId(); + } + case LOCALE_CODE: + return "" + getLocaleCode(); + default: + break; + } + return null; + } + + private String getFormData() throws IOException, InterruptedException{ + String url = ""; + String toolParameters = externalTool.getToolParameters(); + JsonReader jsonReader = Json.createReader(new StringReader(toolParameters)); + JsonObject obj = jsonReader.readObject(); + JsonArray queryParams = obj.getJsonArray("queryParameters"); + if (queryParams == null || queryParams.isEmpty()) { + return ""; + } + Map data = new HashMap<>(); + queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> { + queryParam.keySet().forEach((key) -> { + String value = queryParam.getString(key); + String param = getFormDataValue(key, value); + if (param != null && !param.isEmpty()) { + data.put(key,param); + } + }); + }); + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder().POST(ofFormData(data)).uri(URI.create(externalTool.getToolUrl())) + .header("Content-Type", "application/x-www-form-urlencoded") + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + boolean redirect=false; + int status = response.statusCode(); + if (status != HttpURLConnection.HTTP_OK) { + if (status == HttpURLConnection.HTTP_MOVED_TEMP + || status == HttpURLConnection.HTTP_MOVED_PERM + || status == HttpURLConnection.HTTP_SEE_OTHER) { + redirect = true; + } + } + if (redirect=true){ + String newUrl = response.headers().firstValue("location").get(); + System.out.println(newUrl); + toolContext = "http://" + response.uri().getAuthority(); + + url = newUrl; + } + + System.out.println(response.statusCode()); + System.out.println(response.body()); + + return url; + + } + + public static HttpRequest.BodyPublisher ofFormData(Map data) { + var builder = new StringBuilder(); + data.entrySet().stream().map((var entry) -> { + if (builder.length() > 0) { + builder.append("&"); + } + StringBuilder append = builder.append(URLEncoder.encode(entry.getKey().toString(), StandardCharsets.UTF_8)); + return entry; + }).forEachOrdered(entry -> { + builder.append("="); + builder.append(URLEncoder.encode(entry.getValue().toString(), StandardCharsets.UTF_8)); + }); + return HttpRequest.BodyPublishers.ofString(builder.toString()); + } + + // placeholder for a way to use the POST method instead of the GET method + public String requestMethod(){ + if (externalTool.getDisplayName().startsWith("DP")) + return HttpMethod.POST; + return HttpMethod.GET; + } public String getToolUrlWithQueryParams() { - return externalTool.getToolUrl() + getQueryParametersForUrl(); + String params = getQueryParametersForUrl(); + return toolContext + params; } public String getToolUrlForPreviewMode() { From f74e0c2c855d7c5de2dc5233bcb4d5a34c159629 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 9 Sep 2021 17:47:56 -0400 Subject: [PATCH 002/232] add bonding box indexing --- conf/solr/8.8.1/schema.xml | 9 ++++ .../iq/dataverse/search/IndexServiceBean.java | 42 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/conf/solr/8.8.1/schema.xml b/conf/solr/8.8.1/schema.xml index c6f6cd37cd6..622c4661f6c 100644 --- a/conf/solr/8.8.1/schema.xml +++ b/conf/solr/8.8.1/schema.xml @@ -450,6 +450,9 @@ + + + + + @@ -909,6 +915,9 @@ --> + + diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index d72e2a7f642..b718a63ed95 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.DataFileTag; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; @@ -883,6 +884,47 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d } } } + + //ToDo - define a geom/bbox type solr field and find those instead of just this one + if(dsfType.getName().equals(DatasetFieldConstant.geographicBoundingBox)) { + for (DatasetFieldCompoundValue compoundValue : dsf.getDatasetFieldCompoundValues()) { + String westLon=null; + String eastLon=null; + String northLat=null; + String southLat=null; + for(DatasetField childDsf: compoundValue.getChildDatasetFields()) { + switch (childDsf.getDatasetFieldType().getName()) { + case DatasetFieldConstant.westLongitude: + westLon = childDsf.getRawValue(); + break; + case DatasetFieldConstant.eastLongitude: + eastLon = childDsf.getRawValue(); + break; + case DatasetFieldConstant.northLatitude: + northLat = childDsf.getRawValue(); + break; + case DatasetFieldConstant.southLatitude: + southLat = childDsf.getRawValue(); + break; + } + } + if ((eastLon != null || westLon != null) && (northLat != null || southLat != null)) { + // we have a point or a box, so proceed + if (eastLon == null) { + eastLon = westLon; + } else if (westLon == null) { + westLon = eastLon; + } + if (northLat == null) { + northLat = southLat; + } else if (southLat == null) { + southLat = northLat; + } + //W, E, N, S + solrInputDocument.addField("solr_srpt", "ENVELOPE(" + westLon + "," + eastLon + "," + northLat + "," + southLat + ")"); + } + } + } } } From 4c0fce0dc7ee63e246b80097232608bf72ff3f28 Mon Sep 17 00:00:00 2001 From: roberttreacy Date: Wed, 16 Mar 2022 18:21:59 -0400 Subject: [PATCH 003/232] rename getQueryParametersForUrl to handleRequest remove some experimental code --- .../dataverse/externaltools/ExternalToolHandler.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index ff616d08a4f..84d5b75e34c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -115,12 +115,12 @@ public String getLocaleCode() { } // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. - public String getQueryParametersForUrl() { - return getQueryParametersForUrl(false); + public String handleRequest() { + return handleRequest(false); } // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. - public String getQueryParametersForUrl(boolean preview) { + public String handleRequest(boolean preview) { requestMethod = requestMethod(); if (requestMethod().equals(HttpMethod.POST)){ try { @@ -335,12 +335,12 @@ public String requestMethod(){ return HttpMethod.GET; } public String getToolUrlWithQueryParams() { - String params = getQueryParametersForUrl(); + String params = ExternalToolHandler.this.handleRequest(); return toolContext + params; } public String getToolUrlForPreviewMode() { - return externalTool.getToolUrl() + getQueryParametersForUrl(true); + return externalTool.getToolUrl() + handleRequest(true); } public ExternalTool getExternalTool() { From 36fb9854d8f8a731092d7db9313fa91f5709b20e Mon Sep 17 00:00:00 2001 From: roberttreacy Date: Wed, 16 Mar 2022 18:22:44 -0400 Subject: [PATCH 004/232] rename getQueryParametersForUrl to handleRequest --- .../externaltools/ExternalToolHandlerTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index c900c7e2523..8e70934b4ad 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -111,7 +111,7 @@ public void testGetToolUrlWithOptionalQueryParameters() { ApiToken apiToken = new ApiToken(); apiToken.setTokenString("7196b5ce-f200-4286-8809-03ffdbc255d7"); ExternalToolHandler externalToolHandler3 = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd, nullLocaleCode); - String result3 = externalToolHandler3.getQueryParametersForUrl(); + String result3 = externalToolHandler3.handleRequest(); System.out.println("result3: " + result3); assertEquals("?key1=42&key2=7196b5ce-f200-4286-8809-03ffdbc255d7", result3); @@ -131,7 +131,7 @@ public void testGetToolUrlWithOptionalQueryParameters() { ) .build().toString()); ExternalToolHandler externalToolHandler6 = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd, nullLocaleCode); - String result6 = externalToolHandler6.getQueryParametersForUrl(); + String result6 = externalToolHandler6.handleRequest(); System.out.println("result6: " + result6); assertEquals("?key1=42&key2=7196b5ce-f200-4286-8809-03ffdbc255d7&key3=2", result6); @@ -147,7 +147,7 @@ public void testGetToolUrlWithOptionalQueryParameters() { ) .build().toString()); ExternalToolHandler externalToolHandler4 = new ExternalToolHandler(externalTool, dataFile, nullApiToken, fmd, nullLocaleCode); - String result4 = externalToolHandler4.getQueryParametersForUrl(); + String result4 = externalToolHandler4.handleRequest(); System.out.println("result4: " + result4); assertEquals("?key1=42", result4); @@ -169,7 +169,7 @@ public void testGetToolUrlWithOptionalQueryParameters() { ) .build().toString()); ExternalToolHandler externalToolHandler7 = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd, "en"); - String result7 = externalToolHandler7.getQueryParametersForUrl(); + String result7 = externalToolHandler7.handleRequest(); System.out.println("result7: " + result7); assertEquals("?key1=42&key2=7196b5ce-f200-4286-8809-03ffdbc255d7&key3=2&key4=en", result7); @@ -187,7 +187,7 @@ public void testGetToolUrlWithOptionalQueryParameters() { Exception expectedException = null; try { ExternalToolHandler externalToolHandler5 = new ExternalToolHandler(externalTool, dataFile, nullApiToken, fmd, nullLocaleCode); - String result5 = externalToolHandler5.getQueryParametersForUrl(); + String result5 = externalToolHandler5.handleRequest(); System.out.println("result5: " + result5); } catch (Exception ex) { System.out.println("Exception caught: " + ex); From b90216f28491634029a37490966f1b97f59d0cdb Mon Sep 17 00:00:00 2001 From: Robert Treacy Date: Wed, 16 Mar 2022 18:36:40 -0400 Subject: [PATCH 005/232] add UrlSignerUtil.java --- .../iq/dataverse/util/UrlSignerUtil.java | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java new file mode 100644 index 00000000000..1da1797a8ae --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -0,0 +1,150 @@ +package edu.harvard.iq.dataverse.util; + +import java.net.URL; +import java.nio.charset.Charset; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.joda.time.LocalDateTime; + +/** + * Simple class to sign/validate URLs. + * + */ +public class UrlSignerUtil { + + private static final Logger logger = Logger.getLogger(UrlSignerUtil.class.getName()); + + /** + * + * @param baseUrl - the URL to sign - cannot contain query params + * "until","user", "method", or "token" + * @param timeout - how many minutes to make the URL valid for (note - time skew + * between the creator and receiver could affect the validation + * @param user - a string representing the user - should be understood by the + * creator/receiver + * @param method - one of the HTTP methods + * @param key - a secret key shared by the creator/receiver. In Dataverse + * this could be an APIKey (when sending URL to a tool that will + * use it to retrieve info from Dataverse) + * @return - the signed URL + */ + public static String signUrl(String baseUrl, Integer timeout, String user, String method, String key) { + StringBuilder signedUrl = new StringBuilder(baseUrl); + + boolean firstParam = true; + if (baseUrl.contains("?")) { + firstParam = false; + } + if (timeout != null) { + LocalDateTime validTime = LocalDateTime.now(); + validTime = validTime.plusMinutes(timeout); + validTime.toString(); + signedUrl.append(firstParam ? "?" : "&").append("until=").append(validTime); + firstParam=false; + } + if (user != null) { + signedUrl.append(firstParam ? "?" : "&").append("user=").append(user); + firstParam=false; + } + if (method != null) { + signedUrl.append(firstParam ? "?" : "&").append("method=").append(method); + } + signedUrl.append("&token="); + logger.fine("String to sign: " + signedUrl.toString() + ""); + signedUrl.append(DigestUtils.sha512Hex(signedUrl.toString() + key)); + logger.fine("Generated Signed URL: " + signedUrl.toString()); + if (logger.isLoggable(Level.FINE)) { + logger.fine( + "URL signature is " + (isValidUrl(signedUrl.toString(), method, user, key) ? "valid" : "invalid")); + } + return signedUrl.toString(); + } + + /** + * This method will only return true if the URL and parameters except the + * "token" are unchanged from the original/match the values sent to this method, + * and the "token" parameter matches what this method recalculates using the + * shared key THe method also assures that the "until" timestamp is after the + * current time. + * + * @param signedUrl - the signed URL as received from Dataverse + * @param method - an HTTP method. If provided, the method in the URL must + * match + * @param user - a string representing the user, if provided the value must + * match the one in the url + * @param key - the shared secret key to be used in validation + * @return - true if valid, false if not: e.g. the key is not the same as the + * one used to generate the "token" any part of the URL preceding the + * "token" has been altered the method doesn't match (e.g. the server + * has received a POST request and the URL only allows GET) the user + * string doesn't match (e.g. the server knows user A is logged in, but + * the URL is only for user B) the url has expired (was used after the + * until timestamp) + */ + public static boolean isValidUrl(String signedUrl, String method, String user, String key) { + boolean valid = true; + try { + URL url = new URL(signedUrl); + List params = URLEncodedUtils.parse(url.getQuery(), Charset.forName("UTF-8")); + String hash = null; + String dateString = null; + String allowedMethod = null; + String allowedUser = null; + for (NameValuePair nvp : params) { + if (nvp.getName().equals("token")) { + hash = nvp.getValue(); + logger.fine("Hash: " + hash); + } + if (nvp.getName().equals("until")) { + dateString = nvp.getValue(); + logger.fine("Until: " + dateString); + } + if (nvp.getName().equals("method")) { + allowedMethod = nvp.getValue(); + logger.fine("Method: " + allowedMethod); + } + if (nvp.getName().equals("user")) { + allowedUser = nvp.getValue(); + logger.fine("User: " + allowedUser); + } + } + + int index = signedUrl.indexOf("&token="); + // Assuming the token is last - doesn't have to be, but no reason for the URL + // params to be rearranged either, and this should only cause false negatives if + // it does happen + String urlToHash = signedUrl.substring(0, index + 7); + logger.fine("String to hash: " + urlToHash + ""); + String newHash = DigestUtils.sha512Hex(urlToHash + key); + logger.fine("Calculated Hash: " + newHash); + if (!hash.equals(newHash)) { + logger.fine("Hash doesn't match"); + valid = false; + } + if (dateString != null && LocalDateTime.parse(dateString).isBefore(LocalDateTime.now())) { + logger.fine("Url is expired"); + valid = false; + } + if (method != null && !method.equals(allowedMethod)) { + logger.fine("Method doesn't match"); + valid = false; + } + if (user != null && !user.equals(allowedUser)) { + logger.fine("User doesn't match"); + valid = false; + } + } catch (Throwable t) { + // Want to catch anything like null pointers, etc. to force valid=false upon any + // error + logger.warning("Bad URL: " + signedUrl + " : " + t.getMessage()); + valid = false; + } + return valid; + } + +} \ No newline at end of file From ac234374cf02f712c2c24da4fc13e1a39a80172b Mon Sep 17 00:00:00 2001 From: Robert Treacy Date: Mon, 2 May 2022 15:19:54 -0400 Subject: [PATCH 006/232] add signed Url to header and use POST for external tools, in particular DPCreator WIP - still need to handle use of signed Url to access resource on dataverse --- .../iq/dataverse/ConfigureFragmentBean.java | 1 + .../externaltools/ExternalToolHandler.java | 150 ++++-------------- 2 files changed, 31 insertions(+), 120 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java b/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java index d51a73fd2dc..58752af8520 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java @@ -106,6 +106,7 @@ public void generateApiToken() { ApiToken apiToken = new ApiToken(); User user = session.getUser(); if (user instanceof AuthenticatedUser) { + toolHandler.setUser(((AuthenticatedUser) user).getUserIdentifier()); apiToken = authService.findApiTokenByUser((AuthenticatedUser) user); if (apiToken == null) { //No un-expired token diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 84d5b75e34c..baa386485d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -2,31 +2,28 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.UrlSignerUtil; import java.io.IOException; import java.io.StringReader; import java.net.HttpURLConnection; import java.net.URI; -import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObject; import javax.json.JsonReader; +import javax.json.JsonString; import javax.ws.rs.HttpMethod; /** @@ -36,6 +33,13 @@ */ public class ExternalToolHandler { + /** + * @param user the user to set + */ + public void setUser(String user) { + this.user = user; + } + private static final Logger logger = Logger.getLogger(ExternalToolHandler.class.getCanonicalName()); private final ExternalTool externalTool; @@ -47,7 +51,9 @@ public class ExternalToolHandler { private String localeCode; private String requestMethod; private String toolContext; - + private String user; + private String siteUrl; + /** * File level tool * @@ -121,19 +127,11 @@ public String handleRequest() { // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. public String handleRequest(boolean preview) { - requestMethod = requestMethod(); - if (requestMethod().equals(HttpMethod.POST)){ - try { - return getFormData(); - } catch (IOException ex) { - Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); - } catch (InterruptedException ex) { - Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); - } - } String toolParameters = externalTool.getToolParameters(); JsonReader jsonReader = Json.createReader(new StringReader(toolParameters)); JsonObject obj = jsonReader.readObject(); + JsonString method = obj.getJsonString("httpMethod"); + requestMethod = method!=null?method.getString():HttpMethod.GET; JsonArray queryParams = obj.getJsonArray("queryParameters"); if (queryParams == null || queryParams.isEmpty()) { return ""; @@ -147,7 +145,14 @@ public String handleRequest(boolean preview) { params.add(param); } }); - }); + }); + if (requestMethod.equals(HttpMethod.POST)){ + try { + return postFormData(obj.getJsonNumber("timeOut").intValue(), params); + } catch (IOException | InterruptedException ex) { + Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); + } + } if (!preview) { return "?" + String.join("&", params); } else { @@ -168,7 +173,8 @@ private String getQueryParam(String key, String value) { } break; case SITE_URL: - return key + "=" + SystemConfig.getDataverseSiteUrlStatic(); + siteUrl = SystemConfig.getDataverseSiteUrlStatic(); + return key + "=" + siteUrl; case API_TOKEN: String apiTokenString = null; ApiToken theApiToken = getApiToken(); @@ -209,85 +215,16 @@ private String getQueryParam(String key, String value) { return null; } - private String getFormDataValue(String key, String value) { - ReservedWord reservedWord = ReservedWord.fromString(value); - switch (reservedWord) { - case FILE_ID: - // getDataFile is never null for file tools because of the constructor - return ""+getDataFile().getId(); - case FILE_PID: - GlobalId filePid = getDataFile().getGlobalId(); - if (filePid != null) { - return ""+getDataFile().getGlobalId(); - } - break; - case SITE_URL: - return ""+SystemConfig.getDataverseSiteUrlStatic(); - case API_TOKEN: - String apiTokenString = null; - ApiToken theApiToken = getApiToken(); - if (theApiToken != null) { - apiTokenString = theApiToken.getTokenString(); - return "" + apiTokenString; - } - break; - case DATASET_ID: - return "" + dataset.getId(); - case DATASET_PID: - return "" + dataset.getGlobalId().asString(); - case DATASET_VERSION: - String versionString = null; - if(fileMetadata!=null) { //true for file case - versionString = fileMetadata.getDatasetVersion().getFriendlyVersionNumber(); - } else { //Dataset case - return the latest visible version (unless/until the dataset case allows specifying a version) - if (getApiToken() != null) { - versionString = dataset.getLatestVersion().getFriendlyVersionNumber(); - } else { - versionString = dataset.getLatestVersionForCopy().getFriendlyVersionNumber(); - } - } - if (("DRAFT").equals(versionString)) { - versionString = ":draft"; // send the token needed in api calls that can be substituted for a numeric - // version. - } - return "" + versionString; - case FILE_METADATA_ID: - if(fileMetadata!=null) { //true for file case - return "" + fileMetadata.getId(); - } - case LOCALE_CODE: - return "" + getLocaleCode(); - default: - break; - } - return null; - } - - private String getFormData() throws IOException, InterruptedException{ + private String postFormData(Integer timeout,List params ) throws IOException, InterruptedException{ String url = ""; - String toolParameters = externalTool.getToolParameters(); - JsonReader jsonReader = Json.createReader(new StringReader(toolParameters)); - JsonObject obj = jsonReader.readObject(); - JsonArray queryParams = obj.getJsonArray("queryParameters"); - if (queryParams == null || queryParams.isEmpty()) { - return ""; - } - Map data = new HashMap<>(); - queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> { - queryParam.keySet().forEach((key) -> { - String value = queryParam.getString(key); - String param = getFormDataValue(key, value); - if (param != null && !param.isEmpty()) { - data.put(key,param); - } - }); - }); +// Integer timeout = obj.getJsonNumber("timeOut").intValue(); + url = UrlSignerUtil.signUrl(siteUrl, timeout, user, HttpMethod.POST, getApiToken().getTokenString()); HttpClient client = HttpClient.newHttpClient(); - HttpRequest request = HttpRequest.newBuilder().POST(ofFormData(data)).uri(URI.create(externalTool.getToolUrl())) + HttpRequest request = HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(String.join("&", params))).uri(URI.create(externalTool.getToolUrl())) .header("Content-Type", "application/x-www-form-urlencoded") - .build(); - + .header("signedUrl", url) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); boolean redirect=false; int status = response.statusCode(); @@ -300,40 +237,13 @@ private String getFormData() throws IOException, InterruptedException{ } if (redirect=true){ String newUrl = response.headers().firstValue("location").get(); - System.out.println(newUrl); toolContext = "http://" + response.uri().getAuthority(); url = newUrl; } - - System.out.println(response.statusCode()); - System.out.println(response.body()); - return url; - } - public static HttpRequest.BodyPublisher ofFormData(Map data) { - var builder = new StringBuilder(); - data.entrySet().stream().map((var entry) -> { - if (builder.length() > 0) { - builder.append("&"); - } - StringBuilder append = builder.append(URLEncoder.encode(entry.getKey().toString(), StandardCharsets.UTF_8)); - return entry; - }).forEachOrdered(entry -> { - builder.append("="); - builder.append(URLEncoder.encode(entry.getValue().toString(), StandardCharsets.UTF_8)); - }); - return HttpRequest.BodyPublishers.ofString(builder.toString()); - } - - // placeholder for a way to use the POST method instead of the GET method - public String requestMethod(){ - if (externalTool.getDisplayName().startsWith("DP")) - return HttpMethod.POST; - return HttpMethod.GET; - } public String getToolUrlWithQueryParams() { String params = ExternalToolHandler.this.handleRequest(); return toolContext + params; From d295d868d57aa41b458c0b5803990bb62f6cc558 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 4 May 2022 17:23:33 +0200 Subject: [PATCH 007/232] sorting of licenses with the new sort order column --- .../harvard/iq/dataverse/api/Licenses.java | 31 +++++++++++++++++++ .../harvard/iq/dataverse/license/License.java | 26 +++++++++++++--- .../dataverse/license/LicenseServiceBean.java | 12 +++++++ .../iq/dataverse/util/json/JsonPrinter.java | 3 +- .../V5.10.1.1__8671-sorting_licenses.sql | 9 ++++++ .../iq/dataverse/DatasetVersionTest.java | 2 +- .../harvard/iq/dataverse/api/LicensesIT.java | 14 +++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 9 +++++- .../export/SchemaDotOrgExporterTest.java | 2 +- .../iq/dataverse/util/FileUtilTest.java | 4 +-- 10 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 src/main/resources/db/migration/V5.10.1.1__8671-sorting_licenses.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Licenses.java b/src/main/java/edu/harvard/iq/dataverse/api/Licenses.java index 58e1f8cc2c5..1fdf7818cfb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Licenses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Licenses.java @@ -146,6 +146,37 @@ public Response setActiveState(@PathParam("id") long id, @PathParam("activeState } } + @PUT + @Path("/{id}/:sortOrder/{sortOrder}") + public Response setSortOrder(@PathParam("id") long id, @PathParam("sortOrder") long sortOrder) { + User authenticatedUser; + try { + authenticatedUser = findAuthenticatedUserOrDie(); + if (!authenticatedUser.isSuperuser()) { + return error(Status.FORBIDDEN, "must be superuser"); + } + } catch (WrappedResponse e) { + return error(Status.UNAUTHORIZED, "api key required"); + } + try { + if (licenseSvc.setSortOrder(id, sortOrder) == 0) { + return error(Response.Status.NOT_FOUND, "License with ID " + id + " not found"); + } + License license = licenseSvc.getById(id); + actionLogSvc + .log(new ActionLogRecord(ActionLogRecord.ActionType.Admin, "sortOrderLicenseChanged") + .setInfo("License " + license.getName() + "(" + license.getUri() + ") as id: " + id + + "has now sort order " + sortOrder + ".") + .setUserIdentifier(authenticatedUser.getIdentifier())); + return ok("License ID " + id + " sort order set to " + sortOrder); + } catch (WrappedResponse e) { + if (e.getCause() instanceof IllegalArgumentException) { + return badRequest(e.getCause().getMessage()); + } + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + @DELETE @Path("/{id}") public Response deleteLicenseById(@PathParam("id") long id) { diff --git a/src/main/java/edu/harvard/iq/dataverse/license/License.java b/src/main/java/edu/harvard/iq/dataverse/license/License.java index 96baacc6731..4f99470d7b4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/license/License.java +++ b/src/main/java/edu/harvard/iq/dataverse/license/License.java @@ -23,9 +23,9 @@ */ @NamedQueries({ @NamedQuery( name="License.findAll", - query="SELECT l FROM License l ORDER BY (case when l.isDefault then 0 else 1 end), l.id asc"), + query="SELECT l FROM License l ORDER BY (case when l.isDefault then 0 else 1 end), l.sortOrder, l.name asc"), @NamedQuery( name="License.findAllActive", - query="SELECT l FROM License l WHERE l.active='true' ORDER BY (case when l.isDefault then 0 else 1 end), l.id asc"), + query="SELECT l FROM License l WHERE l.active='true' ORDER BY (case when l.isDefault then 0 else 1 end), l.sortOrder, l.name asc"), @NamedQuery( name="License.findById", query = "SELECT l FROM License l WHERE l.id=:id"), @NamedQuery( name="License.findDefault", @@ -42,6 +42,8 @@ query = "UPDATE License l SET l.isDefault='false'"), @NamedQuery( name="License.setActiveState", query = "UPDATE License l SET l.active=:state WHERE l.id=:id"), + @NamedQuery( name="License.setSortOrder", + query = "UPDATE License l SET l.sortOrder=:sortOrder WHERE l.id=:id"), }) @Entity @@ -73,6 +75,9 @@ public class License { @Column(nullable = false) private boolean isDefault; + + @Column(nullable = false) + private Long sortOrder; @OneToMany(mappedBy="license") private List termsOfUseAndAccess; @@ -80,7 +85,7 @@ public class License { public License() { } - public License(String name, String shortDescription, URI uri, URI iconUrl, boolean active) { + public License(String name, String shortDescription, URI uri, URI iconUrl, boolean active, Long sortOrder) { this.name = name; this.shortDescription = shortDescription; this.uri = uri.toASCIIString(); @@ -91,6 +96,7 @@ public License(String name, String shortDescription, URI uri, URI iconUrl, boole } this.active = active; isDefault = false; + this.sortOrder = sortOrder; } public Long getId() { @@ -172,17 +178,26 @@ public void setTermsOfUseAndAccess(List termsOfUseAndAccess this.termsOfUseAndAccess = termsOfUseAndAccess; } + public Long getSortOrder() { + return sortOrder; + } + + public void setSortOrder(Long sortOrder) { + this.sortOrder = sortOrder; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; License license = (License) o; - return active == license.active && id.equals(license.id) && name.equals(license.name) && shortDescription.equals(license.shortDescription) && uri.equals(license.uri) && Objects.equals(iconUrl, license.iconUrl); + return active == license.active && id.equals(license.id) && name.equals(license.name) && shortDescription.equals(license.shortDescription) && uri.equals(license.uri) && Objects.equals(iconUrl, license.iconUrl) + && Objects.equals(sortOrder, license.sortOrder); } @Override public int hashCode() { - return Objects.hash(id, name, shortDescription, uri, iconUrl, active); + return Objects.hash(id, name, shortDescription, uri, iconUrl, active, sortOrder); } @Override @@ -195,6 +210,7 @@ public String toString() { ", iconUrl=" + iconUrl + ", active=" + active + ", isDefault=" + isDefault + + ", sortOrder=" + sortOrder + '}'; } diff --git a/src/main/java/edu/harvard/iq/dataverse/license/LicenseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/license/LicenseServiceBean.java index c18e168685a..b554fecd437 100644 --- a/src/main/java/edu/harvard/iq/dataverse/license/LicenseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/license/LicenseServiceBean.java @@ -93,11 +93,23 @@ public int setActive(Long id, boolean state) throws WrappedResponse { new IllegalArgumentException("License already " + (state ? "active" : "inactive")), null); } } + + public int setSortOrder(Long id, Long sortOrder) throws WrappedResponse { + License candidate = getById(id); + if (candidate == null) + return 0; + + return em.createNamedQuery("License.setSortOrder").setParameter("id", id).setParameter("sortOrder", sortOrder) + .executeUpdate(); + } public License save(License license) throws WrappedResponse { if (license.getId() != null) { throw new WrappedResponse(new IllegalArgumentException("There shouldn't be an ID in the request body"), null); } + if (license.getSortOrder() == null) { + throw new WrappedResponse(new IllegalArgumentException("There should be a sort order value in the request body"), null); + } try { em.persist(license); em.flush(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index ed3460b6759..e4f15e8992b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -800,7 +800,8 @@ public static JsonObjectBuilder json(License license) { .add("uri", license.getUri().toString()) .add("iconUrl", license.getIconUrl() == null ? null : license.getIconUrl().toString()) .add("active", license.isActive()) - .add("isDefault", license.isDefault()); + .add("isDefault", license.isDefault()) + .add("sortOrder", license.getSortOrder()); } public static Collector stringsToJsonArray() { diff --git a/src/main/resources/db/migration/V5.10.1.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.10.1.1__8671-sorting_licenses.sql new file mode 100644 index 00000000000..5bc18e69df0 --- /dev/null +++ b/src/main/resources/db/migration/V5.10.1.1__8671-sorting_licenses.sql @@ -0,0 +1,9 @@ +ALTER TABLE license +ADD COLUMN IF NOT EXISTS sortorder BIGINT; + +UPDATE license +SET sortorder = id +WHERE sortorder IS NULL; + +CREATE INDEX IF NOT EXISTS license_sortorder_id +ON license (sortorder, id); \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionTest.java index 884a2fd6244..a8e011d0036 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionTest.java @@ -92,7 +92,7 @@ public void testIsInReview() { @Test public void testGetJsonLd() throws ParseException { Dataset dataset = new Dataset(); - License license = new License("CC0 1.0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true); + License license = new License("CC0 1.0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true, 1l); license.setDefault(true); dataset.setProtocol("doi"); dataset.setAuthority("10.5072/FK2"); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java index 09443732f09..e189336b61e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java @@ -144,6 +144,20 @@ public void testLicenses(){ status = JsonPath.from(body).getString("status"); assertEquals("OK", status); + //Fail trying to set null sort order + Response setSortOrderErrorResponse = UtilIT.setLicenseSortOrderById(activeLicenseId, null, adminApiToken); + setSortOrderErrorResponse.prettyPrint(); + body = setSortOrderErrorResponse.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + //Succeed in setting sort order + Response setSortOrderResponse = UtilIT.setLicenseSortOrderById(activeLicenseId, 2l, adminApiToken); + setSortOrderResponse.prettyPrint(); + body = setSortOrderResponse.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("OK", status); + //Succeed in deleting our test license Response deleteLicenseByIdResponse = UtilIT.deleteLicenseById(licenseId, adminApiToken); deleteLicenseByIdResponse.prettyPrint(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 7b9b5f3b129..f9bdabe367b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2808,7 +2808,14 @@ static Response setLicenseActiveById(Long id, boolean state, String apiToken) { .put("/api/licenses/"+id.toString() + "/:active/" + state); return activateLicenseResponse; } - + + static Response setLicenseSortOrderById(Long id, Long sortOrder, String apiToken) { + Response setSortOrderLicenseResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .urlEncodingEnabled(false) + .put("/api/licenses/"+id.toString() + "/:sortOrder/" + sortOrder); + return setSortOrderLicenseResponse; + } static Response updateDatasetJsonLDMetadata(Integer datasetId, String apiToken, String jsonLDBody, boolean replace) { Response response = given() diff --git a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java index b5453e75fe5..641eaf68a3e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java @@ -67,7 +67,7 @@ public static void tearDownClass() { public void testExportDataset() throws Exception { File datasetVersionJson = new File("src/test/resources/json/dataset-finch2.json"); String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); - License license = new License("CC0 1.0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0/"), URI.create("/resources/images/cc0.png"), true); + License license = new License("CC0 1.0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0/"), URI.create("/resources/images/cc0.png"), true, 1l); license.setDefault(true); JsonReader jsonReader1 = Json.createReader(new StringReader(datasetVersionAsJson)); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index 141e97b9b9b..7b5a5ef9d78 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -138,7 +138,7 @@ public void testIsDownloadPopupRequiredLicenseCC0() { DatasetVersion dsv1 = new DatasetVersion(); dsv1.setVersionState(DatasetVersion.VersionState.RELEASED); TermsOfUseAndAccess termsOfUseAndAccess = new TermsOfUseAndAccess(); - License license = new License("CC0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true); + License license = new License("CC0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true, 1l); license.setDefault(true); termsOfUseAndAccess.setLicense(license); dsv1.setTermsOfUseAndAccess(termsOfUseAndAccess); @@ -155,7 +155,7 @@ public void testIsDownloadPopupRequiredHasTermsOfUseAndCc0License() { * the popup when the are Terms of Use. This feels like a bug since the * Terms of Use should probably be shown. */ - License license = new License("CC0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true); + License license = new License("CC0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true, 2l); license.setDefault(true); termsOfUseAndAccess.setLicense(license); termsOfUseAndAccess.setTermsOfUse("be excellent to each other"); From c9ff44b09130222a9f60c203d199b06f21f01ed2 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 9 May 2022 12:25:43 +0200 Subject: [PATCH 008/232] license sorting documentation --- doc/release-notes/8671-sorting-licenses.md | 3 +++ doc/sphinx-guides/source/api/native-api.rst | 7 ++++++ .../source/installation/config.rst | 24 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 doc/release-notes/8671-sorting-licenses.md diff --git a/doc/release-notes/8671-sorting-licenses.md b/doc/release-notes/8671-sorting-licenses.md new file mode 100644 index 00000000000..34ad697d5a7 --- /dev/null +++ b/doc/release-notes/8671-sorting-licenses.md @@ -0,0 +1,3 @@ +## License sorting + +Licenses as shown in the dropdown in UI can be now sorted by the superusers. See [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide for reference. \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 5c56166dd6a..cb387dbbef2 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3806,3 +3806,10 @@ Superusers can delete a license that is not in use by the license ``$ID``: .. code-block:: bash curl -X DELETE -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/$ID + +Superusers can change the sorting order of a license specified by the license ``$ID``: + +.. code-block:: bash + + export SORT_ORDER=100 + curl -X PUT -H 'Content-Type: application/json' -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/$ID/:sortOrder/$SORT_ORDER \ No newline at end of file diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 55d96335a68..d0a7cff1ea3 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -997,6 +997,30 @@ Disabling Custom Dataset Terms See :ref:`:AllowCustomTermsOfUse` for how to disable the "Custom Dataset Terms" option. +.. _ChangeLicenseSortOrder: + +Sorting licenses +---------------- + +The default order of licenses in the dropdown in the user interface is as follows: + +* The default license is shown first +* Followed by the remaining installed licenses in the order of installation +* The custom license is at the end + +Only the order of the installed licenses can be changed with the API calls. The default license always remains first and the custom license last. + +The order of licenses can be changed by setting the ``sortOrder`` property of a license. For the purpose of making sorting easier and to allow grouping of the licenses, ``sortOrder`` property does not have to be unique. Licenses with the same ``sortOrder`` are sorted by their name alfabetically. Nevertheless, you can set a unique ``sortOrder`` for every license in order to sort them fully manually. + +The ``sortOrder`` is an whole number and is used to sort licenses in ascending fashion. All licenses must have a sort order and initially it is set to installation order (``id`` property). + +Changing the sorting order of a license specified by the license ``$ID`` is done by superusers using the following API call: + +.. code-block:: bash + + export SORT_ORDER=100 + curl -X PUT -H 'Content-Type: application/json' -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/$ID/:sortOrder/$SORT_ORDER + .. _BagIt Export: BagIt Export From 7aeaa72b9583ddbc3e9585f28ef6d0572a81e0ee Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 10 May 2022 16:50:36 +0200 Subject: [PATCH 009/232] renamed flyway script to unique version --- ...-sorting_licenses.sql => V5.10.1.2__8671-sorting_licenses.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.10.1.1__8671-sorting_licenses.sql => V5.10.1.2__8671-sorting_licenses.sql} (100%) diff --git a/src/main/resources/db/migration/V5.10.1.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql similarity index 100% rename from src/main/resources/db/migration/V5.10.1.1__8671-sorting_licenses.sql rename to src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql From 7f1561d239031beba167c024d432a88ce7813e33 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 16 May 2022 12:52:01 +0200 Subject: [PATCH 010/232] licenses are now sorted first by sortOrder then by ID --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- src/main/java/edu/harvard/iq/dataverse/license/License.java | 6 +++--- .../db/migration/V5.10.1.2__8671-sorting_licenses.sql | 4 ---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index b99ee2bca83..8bc1e063075 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1051,9 +1051,9 @@ The default order of licenses in the dropdown in the user interface is as follow Only the order of the installed licenses can be changed with the API calls. The default license always remains first and the custom license last. -The order of licenses can be changed by setting the ``sortOrder`` property of a license. For the purpose of making sorting easier and to allow grouping of the licenses, ``sortOrder`` property does not have to be unique. Licenses with the same ``sortOrder`` are sorted by their name alfabetically. Nevertheless, you can set a unique ``sortOrder`` for every license in order to sort them fully manually. +The order of licenses can be changed by setting the ``sortOrder`` property of a license. For the purpose of making sorting easier and to allow grouping of the licenses, ``sortOrder`` property does not have to be unique. Licenses with the same ``sortOrder`` are sorted by their ID, i.e., first by the sortOrder, then by the ID. Nevertheless, you can set a unique ``sortOrder`` for every license in order to sort them fully manually. -The ``sortOrder`` is an whole number and is used to sort licenses in ascending fashion. All licenses must have a sort order and initially it is set to installation order (``id`` property). +The ``sortOrder`` is an whole number and is used to sort licenses in ascending fashion. Changing the sorting order of a license specified by the license ``$ID`` is done by superusers using the following API call: diff --git a/src/main/java/edu/harvard/iq/dataverse/license/License.java b/src/main/java/edu/harvard/iq/dataverse/license/License.java index 4f99470d7b4..0c8465e88e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/license/License.java +++ b/src/main/java/edu/harvard/iq/dataverse/license/License.java @@ -23,9 +23,9 @@ */ @NamedQueries({ @NamedQuery( name="License.findAll", - query="SELECT l FROM License l ORDER BY (case when l.isDefault then 0 else 1 end), l.sortOrder, l.name asc"), + query="SELECT l FROM License l ORDER BY (case when l.isDefault then 0 else 1 end), l.sortOrder, l.id asc"), @NamedQuery( name="License.findAllActive", - query="SELECT l FROM License l WHERE l.active='true' ORDER BY (case when l.isDefault then 0 else 1 end), l.sortOrder, l.name asc"), + query="SELECT l FROM License l WHERE l.active='true' ORDER BY (case when l.isDefault then 0 else 1 end), l.sortOrder, l.id asc"), @NamedQuery( name="License.findById", query = "SELECT l FROM License l WHERE l.id=:id"), @NamedQuery( name="License.findDefault", @@ -76,7 +76,7 @@ public class License { @Column(nullable = false) private boolean isDefault; - @Column(nullable = false) + @Column(nullable = true) private Long sortOrder; @OneToMany(mappedBy="license") diff --git a/src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql index 5bc18e69df0..43631ebd165 100644 --- a/src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql +++ b/src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql @@ -1,9 +1,5 @@ ALTER TABLE license ADD COLUMN IF NOT EXISTS sortorder BIGINT; -UPDATE license -SET sortorder = id -WHERE sortorder IS NULL; - CREATE INDEX IF NOT EXISTS license_sortorder_id ON license (sortorder, id); \ No newline at end of file From 46212be671fbc42a6d56f4069baafac9a29521ee Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 31 May 2022 12:51:53 +0200 Subject: [PATCH 011/232] updated documentation and example with mandatory sort order in licenses --- doc/release-notes/5.10-release-notes.md | 2 +- doc/sphinx-guides/source/_static/api/add-license.json | 3 ++- doc/sphinx-guides/source/api/native-api.rst | 2 +- doc/sphinx-guides/source/api/sword.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 3 +-- scripts/api/data/licenses/licenseCC-BY-4.0.json | 3 ++- scripts/api/data/licenses/licenseCC-BY-NC-4.0.json | 3 ++- scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json | 3 ++- scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json | 3 ++- scripts/api/data/licenses/licenseCC-BY-ND-4.0.json | 3 ++- scripts/api/data/licenses/licenseCC-BY-SA-4.0.json | 3 ++- scripts/api/data/licenses/licenseCC0-1.0.json | 3 ++- ...rting_licenses.sql => V5.10.1.3__8671-sorting_licenses.sql} | 0 13 files changed, 20 insertions(+), 13 deletions(-) rename src/main/resources/db/migration/{V5.10.1.2__8671-sorting_licenses.sql => V5.10.1.3__8671-sorting_licenses.sql} (100%) diff --git a/doc/release-notes/5.10-release-notes.md b/doc/release-notes/5.10-release-notes.md index 0da42a7b527..4e9e5e0ef94 100644 --- a/doc/release-notes/5.10-release-notes.md +++ b/doc/release-notes/5.10-release-notes.md @@ -6,7 +6,7 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S ### Multiple License Support -Users can now select from a set of configured licenses in addition to or instead of the previous Creative Commons CC0 choice or provide custom terms of use (if configured) for their datasets. Administrators can configure their Dataverse instance via API to allow any desired license as a choice and can enable or disable the option to allow custom terms. Administrators can also mark licenses as "inactive" to disallow future use while keeping that license for existing datasets. For upgrades, only the CC0 license will be preinstalled. New installations will have both CC0 and CC BY preinstalled. The [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide shows how to add or remove licenses. +Users can now select from a set of configured licenses in addition to or instead of the previous Creative Commons CC0 choice or provide custom terms of use (if configured) for their datasets. Administrators can configure their Dataverse instance via API to allow any desired license as a choice and can enable or disable the option to allow custom terms. Administrators can also mark licenses as "inactive" to disallow future use while keeping that license for existing datasets. For upgrades, only the CC0 license will be preinstalled. New installations will have both CC0 1.0 preinstalled. The [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide shows how to add or remove licenses. **Note: Datasets in existing installations will automatically be updated to conform to new requirements that custom terms cannot be used with a standard license and that custom terms cannot be empty. Administrators may wish to manually update datasets with these conditions if they do not like the automated migration choices. See the "Notes for Dataverse Installation Administrators" section below for details.** diff --git a/doc/sphinx-guides/source/_static/api/add-license.json b/doc/sphinx-guides/source/_static/api/add-license.json index 969d6d58dab..a9d5dd34093 100644 --- a/doc/sphinx-guides/source/_static/api/add-license.json +++ b/doc/sphinx-guides/source/_static/api/add-license.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/licenses/by/4.0", "shortDescription": "Creative Commons Attribution 4.0 International License.", "iconUrl": "https://i.creativecommons.org/l/by/4.0/88x31.png", - "active": true + "active": true, + "sortOrder": 2 } diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 249d1812507..da82be9ad7b 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3858,7 +3858,7 @@ View the details of the standard license with the database ID specified in ``$ID curl $SERVER_URL/api/licenses/$ID -Superusers can add a new license by posting a JSON file adapted from this example :download:`add-license.json <../_static/api/add-license.json>`. The ``name`` and ``uri`` of the new license must be unique. If you are interested in adding a Creative Commons license, you are encouarged to use the JSON files under :ref:`adding-creative-commons-licenses`: +Superusers can add a new license by posting a JSON file adapted from this example :download:`add-license.json <../_static/api/add-license.json>`. The ``name`` and ``uri`` of the new license must be unique. Sort order field is mandatory. If you are interested in adding a Creative Commons license, you are encouarged to use the JSON files under :ref:`adding-creative-commons-licenses`: .. code-block:: bash diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index 11b43e98774..8041dff4891 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -82,7 +82,7 @@ New features as of v1.1 - "Contributor" can now be populated and the "Type" (Editor, Funder, Researcher, etc.) can be specified with an XML attribute. For example: ``CaffeineForAll`` -- "License" can now be set with ``dcterms:license`` and the possible values determined by the installation ("CC0 1.0" and "CC BY 4.0" by default). "License" interacts with "Terms of Use" (``dcterms:rights``) in that if you include ``dcterms:rights`` in the XML and don't include ``dcterms:license``, the license will be "Custom Dataset Terms" and "Terms of Use" will be populated. If you don't include ``dcterms:rights``, the default license will be used. It is invalid to specify a license and also include ``dcterms:rights``; an error will be returned. For backwards compatibility, ``dcterms:rights`` is allowed to be blank (i.e. ````) but blank values will not be persisted to the database and the license will be set to "Custom Dataset Terms". Note that if admins of an installation have disabled "Custom Dataset Terms" you will get an error if you try to pass ``dcterms:rights``. +- "License" can now be set with ``dcterms:license`` and the possible values determined by the installation ("CC0 1.0" by default). "License" interacts with "Terms of Use" (``dcterms:rights``) in that if you include ``dcterms:rights`` in the XML and don't include ``dcterms:license``, the license will be "Custom Dataset Terms" and "Terms of Use" will be populated. If you don't include ``dcterms:rights``, the default license will be used. It is invalid to specify a license and also include ``dcterms:rights``; an error will be returned. For backwards compatibility, ``dcterms:rights`` is allowed to be blank (i.e. ````) but blank values will not be persisted to the database and the license will be set to "Custom Dataset Terms". Note that if admins of an installation have disabled "Custom Dataset Terms" you will get an error if you try to pass ``dcterms:rights``. - "Contact E-mail" is automatically populated from dataset owner's email. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 99ed622c911..61e13ad10c8 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -991,7 +991,6 @@ Configuring Licenses Out of the box, users select from the following licenses or terms: - CC0 1.0 (default) -- CC BY 4.0 - Custom Dataset Terms You have a lot of control over which licenses and terms are available. You can remove licenses and add new ones. You can decide which license is the default. You can remove "Custom Dataset Terms" as a option. You can remove all licenses and make "Custom Dataset Terms" the only option. @@ -1015,7 +1014,7 @@ Licenses are added with curl using JSON file as explained in the API Guide under Adding Creative Common Licenses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -JSON files for `Creative Commons licenses `_ are provided below. Note that a new installation of Dataverse already includes CC0 and CC BY. +JSON files for `Creative Commons licenses `_ are provided below. Note that a new installation of Dataverse already includes CC0. - :download:`licenseCC0-1.0.json <../../../../scripts/api/data/licenses/licenseCC0-1.0.json>` - :download:`licenseCC-BY-4.0.json <../../../../scripts/api/data/licenses/licenseCC-BY-4.0.json>` diff --git a/scripts/api/data/licenses/licenseCC-BY-4.0.json b/scripts/api/data/licenses/licenseCC-BY-4.0.json index 5596e65e947..59201b8d08e 100644 --- a/scripts/api/data/licenses/licenseCC-BY-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-4.0.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/licenses/by/4.0", "shortDescription": "Creative Commons Attribution 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by/4.0/88x31.png", - "active": true + "active": true, + "sortOrder": 2 } diff --git a/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json b/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json index 8154c9ec5df..c19087664db 100644 --- a/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/licenses/by-nc/4.0", "shortDescription": "Creative Commons Attribution-NonCommercial 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nc/4.0/88x31.png", - "active": true + "active": true, + "sortOrder": 4 } diff --git a/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json b/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json index 247ce52f6ea..2e374917d28 100644 --- a/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/licenses/by-nc-nd/4.0", "shortDescription": "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nc-nd/4.0/88x31.png", - "active": true + "active": true, + "sortOrder": 7 } diff --git a/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json b/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json index e9726fb6374..5018884f65e 100644 --- a/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/licenses/by-nc-sa/4.0", "shortDescription": "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nc-sa/4.0/88x31.png", - "active": true + "active": true, + "sortOrder": 3 } diff --git a/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json b/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json index 7ae81bacc10..317d459a7ae 100644 --- a/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/licenses/by-nd/4.0", "shortDescription": "Creative Commons Attribution-NoDerivatives 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nd/4.0/88x31.png", - "active": true + "active": true, + "sortOrder": 6 } diff --git a/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json b/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json index e9a02880885..0d28c9423aa 100644 --- a/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/licenses/by-sa/4.0", "shortDescription": "Creative Commons Attribution-ShareAlike 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-sa/4.0/88x31.png", - "active": true + "active": true, + "sortOrder": 5 } diff --git a/scripts/api/data/licenses/licenseCC0-1.0.json b/scripts/api/data/licenses/licenseCC0-1.0.json index 396ba133327..216260a5de8 100644 --- a/scripts/api/data/licenses/licenseCC0-1.0.json +++ b/scripts/api/data/licenses/licenseCC0-1.0.json @@ -3,5 +3,6 @@ "uri": "http://creativecommons.org/publicdomain/zero/1.0", "shortDescription": "Creative Commons CC0 1.0 Universal Public Domain Dedication.", "iconUrl": "https://licensebuttons.net/p/zero/1.0/88x31.png", - "active": true + "active": true, + "sortOrder": 1 } diff --git a/src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.10.1.3__8671-sorting_licenses.sql similarity index 100% rename from src/main/resources/db/migration/V5.10.1.2__8671-sorting_licenses.sql rename to src/main/resources/db/migration/V5.10.1.3__8671-sorting_licenses.sql From adc8c18e8481426a5614efbccdc94e3be5d9c051 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Thu, 2 Jun 2022 17:41:36 +0200 Subject: [PATCH 012/232] revert of removing cc by from documentation --- doc/release-notes/5.10-release-notes.md | 2 +- doc/sphinx-guides/source/api/sword.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/5.10-release-notes.md b/doc/release-notes/5.10-release-notes.md index 4e9e5e0ef94..0da42a7b527 100644 --- a/doc/release-notes/5.10-release-notes.md +++ b/doc/release-notes/5.10-release-notes.md @@ -6,7 +6,7 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S ### Multiple License Support -Users can now select from a set of configured licenses in addition to or instead of the previous Creative Commons CC0 choice or provide custom terms of use (if configured) for their datasets. Administrators can configure their Dataverse instance via API to allow any desired license as a choice and can enable or disable the option to allow custom terms. Administrators can also mark licenses as "inactive" to disallow future use while keeping that license for existing datasets. For upgrades, only the CC0 license will be preinstalled. New installations will have both CC0 1.0 preinstalled. The [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide shows how to add or remove licenses. +Users can now select from a set of configured licenses in addition to or instead of the previous Creative Commons CC0 choice or provide custom terms of use (if configured) for their datasets. Administrators can configure their Dataverse instance via API to allow any desired license as a choice and can enable or disable the option to allow custom terms. Administrators can also mark licenses as "inactive" to disallow future use while keeping that license for existing datasets. For upgrades, only the CC0 license will be preinstalled. New installations will have both CC0 and CC BY preinstalled. The [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide shows how to add or remove licenses. **Note: Datasets in existing installations will automatically be updated to conform to new requirements that custom terms cannot be used with a standard license and that custom terms cannot be empty. Administrators may wish to manually update datasets with these conditions if they do not like the automated migration choices. See the "Notes for Dataverse Installation Administrators" section below for details.** diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index 8041dff4891..11b43e98774 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -82,7 +82,7 @@ New features as of v1.1 - "Contributor" can now be populated and the "Type" (Editor, Funder, Researcher, etc.) can be specified with an XML attribute. For example: ``CaffeineForAll`` -- "License" can now be set with ``dcterms:license`` and the possible values determined by the installation ("CC0 1.0" by default). "License" interacts with "Terms of Use" (``dcterms:rights``) in that if you include ``dcterms:rights`` in the XML and don't include ``dcterms:license``, the license will be "Custom Dataset Terms" and "Terms of Use" will be populated. If you don't include ``dcterms:rights``, the default license will be used. It is invalid to specify a license and also include ``dcterms:rights``; an error will be returned. For backwards compatibility, ``dcterms:rights`` is allowed to be blank (i.e. ````) but blank values will not be persisted to the database and the license will be set to "Custom Dataset Terms". Note that if admins of an installation have disabled "Custom Dataset Terms" you will get an error if you try to pass ``dcterms:rights``. +- "License" can now be set with ``dcterms:license`` and the possible values determined by the installation ("CC0 1.0" and "CC BY 4.0" by default). "License" interacts with "Terms of Use" (``dcterms:rights``) in that if you include ``dcterms:rights`` in the XML and don't include ``dcterms:license``, the license will be "Custom Dataset Terms" and "Terms of Use" will be populated. If you don't include ``dcterms:rights``, the default license will be used. It is invalid to specify a license and also include ``dcterms:rights``; an error will be returned. For backwards compatibility, ``dcterms:rights`` is allowed to be blank (i.e. ````) but blank values will not be persisted to the database and the license will be set to "Custom Dataset Terms". Note that if admins of an installation have disabled "Custom Dataset Terms" you will get an error if you try to pass ``dcterms:rights``. - "Contact E-mail" is automatically populated from dataset owner's email. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 61e13ad10c8..99ed622c911 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -991,6 +991,7 @@ Configuring Licenses Out of the box, users select from the following licenses or terms: - CC0 1.0 (default) +- CC BY 4.0 - Custom Dataset Terms You have a lot of control over which licenses and terms are available. You can remove licenses and add new ones. You can decide which license is the default. You can remove "Custom Dataset Terms" as a option. You can remove all licenses and make "Custom Dataset Terms" the only option. @@ -1014,7 +1015,7 @@ Licenses are added with curl using JSON file as explained in the API Guide under Adding Creative Common Licenses ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -JSON files for `Creative Commons licenses `_ are provided below. Note that a new installation of Dataverse already includes CC0. +JSON files for `Creative Commons licenses `_ are provided below. Note that a new installation of Dataverse already includes CC0 and CC BY. - :download:`licenseCC0-1.0.json <../../../../scripts/api/data/licenses/licenseCC0-1.0.json>` - :download:`licenseCC-BY-4.0.json <../../../../scripts/api/data/licenses/licenseCC-BY-4.0.json>` From 7e82009436f8b1f17fdc0caed8a0f65c3f0da500 Mon Sep 17 00:00:00 2001 From: Robert Treacy Date: Wed, 8 Jun 2022 14:58:48 -0400 Subject: [PATCH 013/232] use signedUrl for getting authenticated user. add allowedUrls field to ExtrenalTool --- .../iq/dataverse/api/AbstractApiBean.java | 19 +++++++++ .../dataverse/externaltools/ExternalTool.java | 40 ++++++++++++++++++- .../externaltools/ExternalToolHandler.java | 17 ++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index d2c3f68dba2..24994497267 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -49,6 +49,7 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.UrlSignerUtil; import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean; @@ -419,10 +420,28 @@ private AuthenticatedUser findAuthenticatedUserOrDie( String key, String wfid ) } else { throw new WrappedResponse(badWFKey(wfid)); } + } else { + AuthenticatedUser authUser = getAuthenticatedUserFromSignedUrl(); + if (authUser != null) { + return authUser; + } } //Just send info about the apiKey - workflow users will learn about invocationId elsewhere throw new WrappedResponse(badApiKey(null)); } + + private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { + AuthenticatedUser authUser = null; + String signedUrl = httpRequest.getRequestURL().toString(); + String user = httpRequest.getParameter("user"); + String method = httpRequest.getMethod(); + String key = httpRequest.getParameter("token"); + boolean validated = UrlSignerUtil.isValidUrl(signedUrl, method, user, key); + if (validated){ + authUser = authSvc.getAuthenticatedUser(user); + } + return authUser; + } protected Dataverse findDataverseOrDie( String dvIdtf ) throws WrappedResponse { Dataverse dv = findDataverse(dvIdtf); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index c996e332bdb..b393ee7c747 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -41,6 +41,7 @@ public class ExternalTool implements Serializable { public static final String TOOL_PARAMETERS = "toolParameters"; public static final String CONTENT_TYPE = "contentType"; public static final String TOOL_NAME = "toolName"; + public static final String ALLOWED_URLS = "allowedUrls"; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -97,6 +98,13 @@ public class ExternalTool implements Serializable { @Column(nullable = true, columnDefinition = "TEXT") private String contentType; + /** + * Path for retrieving data through the REST api. Used to build signedUrls + * for POST headers, as in DPCreator + */ + @Column(nullable = true, columnDefinition = "TEXT") + private String allowedUrls; + /** * This default constructor is only here to prevent this error at * deployment: @@ -122,6 +130,18 @@ public ExternalTool(String displayName, String toolName, String description, Lis this.contentType = contentType; } + public ExternalTool(String displayName, String toolName, String description, List externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType, String allowedUrls) { + this.displayName = displayName; + this.toolName = toolName; + this.description = description; + this.externalToolTypes = externalToolTypes; + this.scope = scope; + this.toolUrl = toolUrl; + this.toolParameters = toolParameters; + this.contentType = contentType; + this.allowedUrls = allowedUrls; + } + public enum Type { EXPLORE("explore"), @@ -273,6 +293,9 @@ public JsonObjectBuilder toJson() { if (getContentType() != null) { jab.add(CONTENT_TYPE, getContentType()); } + if (getAllowedUrls()!= null) { + jab.add(ALLOWED_URLS,getAllowedUrls()); + } return jab; } @@ -292,7 +315,8 @@ public enum ReservedWord { DATASET_PID("datasetPid"), DATASET_VERSION("datasetVersion"), FILE_METADATA_ID("fileMetadataId"), - LOCALE_CODE("localeCode"); + LOCALE_CODE("localeCode"), + ALLOWED_URLS("allowedUrls"); private final String text; private final String START = "{"; @@ -355,5 +379,19 @@ public String getDisplayNameLang() { return displayName; } + /** + * @return the allowedUrls + */ + public String getAllowedUrls() { + return allowedUrls; + } + + /** + * @param allowedUrls the allowedUrls to set + */ + public void setAllowedUrls(String allowedUrls) { + this.allowedUrls = allowedUrls; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index baa386485d3..8061303b434 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -33,6 +33,20 @@ */ public class ExternalToolHandler { + /** + * @return the allowedUrls + */ + public String getAllowedUrls() { + return allowedUrls; + } + + /** + * @param allowedUrls the allowedUrls to set + */ + public void setAllowedUrls(String allowedUrls) { + this.allowedUrls = allowedUrls; + } + /** * @param user the user to set */ @@ -53,6 +67,7 @@ public void setUser(String user) { private String toolContext; private String user; private String siteUrl; + private String allowedUrls; /** * File level tool @@ -209,6 +224,8 @@ private String getQueryParam(String key, String value) { } case LOCALE_CODE: return key + "=" + getLocaleCode(); + case ALLOWED_URLS: + return key + "=" + getAllowedUrls(); default: break; } From 7c9fa06aa274331aedec4d5ffa2889e37c55389e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 9 Jun 2022 11:09:21 -0400 Subject: [PATCH 014/232] fix for validation method/comments --- .../iq/dataverse/api/AbstractApiBean.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 24994497267..402908c57e3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -432,14 +432,22 @@ private AuthenticatedUser findAuthenticatedUserOrDie( String key, String wfid ) private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { AuthenticatedUser authUser = null; - String signedUrl = httpRequest.getRequestURL().toString(); + // The signUrl contains a param telling which user this is supposed to be for. + // We don't trust this. So we lookup that user, and get their API key, and use + // that as a secret in validation the signedURL. If the signature can't be + // validating with their key, the user (or their API key) has been changed and + // we reject the request. + //ToDo - add null checks/ verify that calling methods catch things. String user = httpRequest.getParameter("user"); + AuthenticatedUser targetUser = authSvc.getAuthenticatedUser(user); + String key = authSvc.findApiTokenByUser(targetUser).getTokenString(); + String signedUrl = httpRequest.getRequestURL().toString(); String method = httpRequest.getMethod(); - String key = httpRequest.getParameter("token"); + boolean validated = UrlSignerUtil.isValidUrl(signedUrl, method, user, key); if (validated){ - authUser = authSvc.getAuthenticatedUser(user); - } + authUser = targetUser; + } return authUser; } From 39180ccbba124894e5d5f7a999e07ed11b36fb46 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 9 Jun 2022 11:09:40 -0400 Subject: [PATCH 015/232] JSON API call to request signedUrl --- .../edu/harvard/iq/dataverse/api/Admin.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 4085b504578..4ab542b469c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -31,6 +31,7 @@ import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProvider; import edu.harvard.iq.dataverse.authorization.providers.shib.ShibServiceBean; import edu.harvard.iq.dataverse.authorization.providers.shib.ShibUtil; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailData; import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailException; @@ -44,6 +45,7 @@ import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.json.JsonObjectBuilder; +import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -94,6 +96,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.UrlSignerUtil; import java.io.IOException; import java.io.OutputStream; @@ -2061,4 +2064,43 @@ public Response getBannerMessages(@PathParam("id") Long id) throws WrappedRespon .collect(toJsonArray())); } + + @POST + @Consumes("application/json") + @Path("/requestSignedUrl") + public Response getSignedUrl(JsonObject urlInfo) throws WrappedResponse { + AuthenticatedUser superuser = authSvc.getAdminUser(); + + if (superuser == null) { + return error(Response.Status.FORBIDDEN, "Requesting signed URLs is restricted to superusers."); + } + + String userId = urlInfo.getString("user"); + String key=null; + if(userId!=null) { + AuthenticatedUser user = authSvc.getAuthenticatedUser(userId); + if(user!=null) { + ApiToken apiToken = authSvc.findApiTokenByUser(user); + if(apiToken!=null && !apiToken.isExpired() && ! apiToken.isDisabled()) { + key = apiToken.getTokenString(); + } + } else { + userId=superuser.getIdentifier(); + //We ~know this exists - the superuser just used it and it was unexpired/not disabled. (ToDo - if we want this to work with workflow tokens (or as a signed URL, we should do more checking as for the user above)) + } + key = authSvc.findApiTokenByUser(superuser).getTokenString(); + } + if(key==null) { + return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); + } + + String baseUrl = urlInfo.getString("url"); + int timeout = urlInfo.getInt("timeout", 10); + String method = urlInfo.getString("method", "GET"); + + String signedUrl = UrlSignerUtil.signUrl(baseUrl, timeout, userId, method, key); + + return ok(Json.createObjectBuilder().add("signedUrl", signedUrl)); + } + } From 55fafa573fba5ab9ed89ab0c706be14357410aaa Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 9 Jun 2022 12:04:56 -0400 Subject: [PATCH 016/232] json read object/array from string methods from other branches --- .../harvard/iq/dataverse/util/json/JsonUtil.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java index ae6935945e8..ef506990f69 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java @@ -3,6 +3,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; + +import java.io.StringReader; import java.io.StringWriter; import java.util.HashMap; import java.util.Map; @@ -55,5 +57,16 @@ public static String prettyPrint(javax.json.JsonObject jsonObject) { } return stringWriter.toString(); } + + public static javax.json.JsonObject getJsonObject(String serializedJson) { + try (StringReader rdr = new StringReader(serializedJson)) { + return Json.createReader(rdr).readObject(); + } + } + public static JsonArray getJsonArray(String serializedJson) { + try (StringReader rdr = new StringReader(serializedJson)) { + return Json.createReader(rdr).readArray(); + } + } } From 881e3db2564eab58dc91b76162b9465cc1d5f2b1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 14 Jun 2022 12:52:32 -0400 Subject: [PATCH 017/232] define/use an additional secret key --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 2 +- .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 402908c57e3..4adac3feace 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -440,7 +440,7 @@ private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { //ToDo - add null checks/ verify that calling methods catch things. String user = httpRequest.getParameter("user"); AuthenticatedUser targetUser = authSvc.getAuthenticatedUser(user); - String key = authSvc.findApiTokenByUser(targetUser).getTokenString(); + String key = System.getProperty(SystemConfig.API_SIGNING_SECRET,"") + authSvc.findApiTokenByUser(targetUser).getTokenString(); String signedUrl = httpRequest.getRequestURL().toString(); String method = httpRequest.getMethod(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 4ab542b469c..f0546aaca30 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2088,7 +2088,7 @@ public Response getSignedUrl(JsonObject urlInfo) throws WrappedResponse { userId=superuser.getIdentifier(); //We ~know this exists - the superuser just used it and it was unexpired/not disabled. (ToDo - if we want this to work with workflow tokens (or as a signed URL, we should do more checking as for the user above)) } - key = authSvc.findApiTokenByUser(superuser).getTokenString(); + key = System.getProperty(SystemConfig.API_SIGNING_SECRET,"") + authSvc.findApiTokenByUser(superuser).getTokenString(); } if(key==null) { return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 6ea63e2b51f..3c7f05bec1e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -124,6 +124,11 @@ public class SystemConfig { public final static String DEFAULTCURATIONLABELSET = "DEFAULT"; public final static String CURATIONLABELSDISABLED = "DISABLED"; + // A secret used in signing URLs - individual urls are signed using this and the + // intended user's apiKey, creating an aggregate key that is unique to the user + // but not known to the user (as their apiKey is) + public final static String API_SIGNING_SECRET = "dataverse.api-signing-secret;"; + public String getVersion() { return getVersion(false); } From 208ab95a947d66f17559853bedd4f870f76504a4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 21 Jun 2022 11:24:05 -0400 Subject: [PATCH 018/232] refactor to allow URL token substitution outside tools framework --- .../dataverse/externaltools/ExternalTool.java | 58 ----- .../externaltools/ExternalToolHandler.java | 116 +-------- .../ExternalToolServiceBean.java | 3 +- .../iq/dataverse/util/URLTokenUtil.java | 231 ++++++++++++++++++ .../iq/dataverse/util/UrlTokenUtilTest.java | 50 ++++ 5 files changed, 289 insertions(+), 169 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index b393ee7c747..476181af852 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -299,64 +299,6 @@ public JsonObjectBuilder toJson() { return jab; } - public enum ReservedWord { - - // TODO: Research if a format like "{reservedWord}" is easily parse-able or if another format would be - // better. The choice of curly braces is somewhat arbitrary, but has been observed in documenation for - // various REST APIs. For example, "Variable substitutions will be made when a variable is named in {brackets}." - // from https://swagger.io/specification/#fixed-fields-29 but that's for URLs. - FILE_ID("fileId"), - FILE_PID("filePid"), - SITE_URL("siteUrl"), - API_TOKEN("apiToken"), - // datasetId is the database id - DATASET_ID("datasetId"), - // datasetPid is the DOI or Handle - DATASET_PID("datasetPid"), - DATASET_VERSION("datasetVersion"), - FILE_METADATA_ID("fileMetadataId"), - LOCALE_CODE("localeCode"), - ALLOWED_URLS("allowedUrls"); - - private final String text; - private final String START = "{"; - private final String END = "}"; - - private ReservedWord(final String text) { - this.text = START + text + END; - } - - /** - * This is a centralized method that enforces that only reserved words - * are allowed to be used by external tools. External tool authors - * cannot pass their own query parameters through Dataverse such as - * "mode=mode1". - * - * @throws IllegalArgumentException - */ - public static ReservedWord fromString(String text) throws IllegalArgumentException { - if (text != null) { - for (ReservedWord reservedWord : ReservedWord.values()) { - if (text.equals(reservedWord.text)) { - return reservedWord; - } - } - } - // TODO: Consider switching to a more informative message that enumerates the valid reserved words. - boolean moreInformativeMessage = false; - if (moreInformativeMessage) { - throw new IllegalArgumentException("Unknown reserved word: " + text + ". A reserved word must be one of these values: " + Arrays.asList(ReservedWord.values()) + "."); - } else { - throw new IllegalArgumentException("Unknown reserved word: " + text); - } - } - - @Override - public String toString() { - return text; - } - } - public String getDescriptionLang() { String description = ""; if (this.toolName != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 8061303b434..8a1e9661e3a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -3,10 +3,9 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.users.ApiToken; -import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord; -import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.URLTokenUtil; + import edu.harvard.iq.dataverse.util.UrlSignerUtil; import java.io.IOException; import java.io.StringReader; @@ -19,6 +18,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; + import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonObject; @@ -31,8 +31,7 @@ * instantiated. Applies logic based on an {@link ExternalTool} specification, * such as constructing a URL to access that file. */ -public class ExternalToolHandler { - +public class ExternalToolHandler extends URLTokenUtil { /** * @return the allowedUrls */ @@ -54,15 +53,8 @@ public void setUser(String user) { this.user = user; } - private static final Logger logger = Logger.getLogger(ExternalToolHandler.class.getCanonicalName()); - private final ExternalTool externalTool; - private final DataFile dataFile; - private final Dataset dataset; - private final FileMetadata fileMetadata; - private ApiToken apiToken; - private String localeCode; private String requestMethod; private String toolContext; private String user; @@ -78,23 +70,9 @@ public void setUser(String user) { * used anonymously. */ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToken apiToken, FileMetadata fileMetadata, String localeCode) { + super(dataFile, apiToken, fileMetadata, localeCode); this.externalTool = externalTool; toolContext = externalTool.getToolUrl(); - if (dataFile == null) { - String error = "A DataFile is required."; - logger.warning("Error in ExternalToolHandler constructor: " + error); - throw new IllegalArgumentException(error); - } - if (fileMetadata == null) { - String error = "A FileMetadata is required."; - logger.warning("Error in ExternalToolHandler constructor: " + error); - throw new IllegalArgumentException(error); - } - this.dataFile = dataFile; - this.apiToken = apiToken; - this.fileMetadata = fileMetadata; - dataset = fileMetadata.getDatasetVersion().getDataset(); - this.localeCode = localeCode; } /** @@ -106,33 +84,8 @@ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToke * used anonymously. */ public ExternalToolHandler(ExternalTool externalTool, Dataset dataset, ApiToken apiToken, String localeCode) { + super(dataset, apiToken, localeCode); this.externalTool = externalTool; - if (dataset == null) { - String error = "A Dataset is required."; - logger.warning("Error in ExternalToolHandler constructor: " + error); - throw new IllegalArgumentException(error); - } - this.dataset = dataset; - this.apiToken = apiToken; - this.dataFile = null; - this.fileMetadata = null; - this.localeCode = localeCode; - } - - public DataFile getDataFile() { - return dataFile; - } - - public FileMetadata getFileMetadata() { - return fileMetadata; - } - - public ApiToken getApiToken() { - return apiToken; - } - - public String getLocaleCode() { - return localeCode; } // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. @@ -175,63 +128,6 @@ public String handleRequest(boolean preview) { } } - private String getQueryParam(String key, String value) { - ReservedWord reservedWord = ReservedWord.fromString(value); - switch (reservedWord) { - case FILE_ID: - // getDataFile is never null for file tools because of the constructor - return key + "=" + getDataFile().getId(); - case FILE_PID: - GlobalId filePid = getDataFile().getGlobalId(); - if (filePid != null) { - return key + "=" + getDataFile().getGlobalId(); - } - break; - case SITE_URL: - siteUrl = SystemConfig.getDataverseSiteUrlStatic(); - return key + "=" + siteUrl; - case API_TOKEN: - String apiTokenString = null; - ApiToken theApiToken = getApiToken(); - if (theApiToken != null) { - apiTokenString = theApiToken.getTokenString(); - return key + "=" + apiTokenString; - } - break; - case DATASET_ID: - return key + "=" + dataset.getId(); - case DATASET_PID: - return key + "=" + dataset.getGlobalId().asString(); - case DATASET_VERSION: - String versionString = null; - if(fileMetadata!=null) { //true for file case - versionString = fileMetadata.getDatasetVersion().getFriendlyVersionNumber(); - } else { //Dataset case - return the latest visible version (unless/until the dataset case allows specifying a version) - if (getApiToken() != null) { - versionString = dataset.getLatestVersion().getFriendlyVersionNumber(); - } else { - versionString = dataset.getLatestVersionForCopy().getFriendlyVersionNumber(); - } - } - if (("DRAFT").equals(versionString)) { - versionString = ":draft"; // send the token needed in api calls that can be substituted for a numeric - // version. - } - return key + "=" + versionString; - case FILE_METADATA_ID: - if(fileMetadata!=null) { //true for file case - return key + "=" + fileMetadata.getId(); - } - case LOCALE_CODE: - return key + "=" + getLocaleCode(); - case ALLOWED_URLS: - return key + "=" + getAllowedUrls(); - default: - break; - } - return null; - } - private String postFormData(Integer timeout,List params ) throws IOException, InterruptedException{ String url = ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index 95fd900e4d2..d49d66c26f7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -3,8 +3,9 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.authorization.users.ApiToken; -import edu.harvard.iq.dataverse.externaltools.ExternalTool.ReservedWord; import edu.harvard.iq.dataverse.externaltools.ExternalTool.Type; +import edu.harvard.iq.dataverse.util.URLTokenUtil; +import edu.harvard.iq.dataverse.util.URLTokenUtil.ReservedWord; import edu.harvard.iq.dataverse.externaltools.ExternalTool.Scope; import java.io.StringReader; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java new file mode 100644 index 00000000000..78280cd0f0f --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java @@ -0,0 +1,231 @@ +package edu.harvard.iq.dataverse.util; + +import java.util.Arrays; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; + +public class URLTokenUtil { + + protected static final Logger logger = Logger.getLogger(URLTokenUtil.class.getCanonicalName()); + protected final DataFile dataFile; + protected final Dataset dataset; + protected final FileMetadata fileMetadata; + protected ApiToken apiToken; + protected String localeCode; + + /** + * File level + * + * @param dataFile Required. + * @param apiToken The apiToken can be null + * @param fileMetadata Required. + * @param localeCode optional. + * + */ + public URLTokenUtil(DataFile dataFile, ApiToken apiToken, FileMetadata fileMetadata, String localeCode) + throws IllegalArgumentException { + if (dataFile == null) { + String error = "A DataFile is required."; + logger.warning("Error in URLTokenUtil constructor: " + error); + throw new IllegalArgumentException(error); + } + if (fileMetadata == null) { + String error = "A FileMetadata is required."; + logger.warning("Error in URLTokenUtil constructor: " + error); + throw new IllegalArgumentException(error); + } + this.dataFile = dataFile; + this.dataset = fileMetadata.getDatasetVersion().getDataset(); + this.fileMetadata = fileMetadata; + this.apiToken = apiToken; + this.localeCode = localeCode; + } + + /** + * Dataset level + * + * @param dataset Required. + * @param apiToken The apiToken can be null + */ + public URLTokenUtil(Dataset dataset, ApiToken apiToken, String localeCode) { + if (dataset == null) { + String error = "A Dataset is required."; + logger.warning("Error in URLTokenUtil constructor: " + error); + throw new IllegalArgumentException(error); + } + this.dataset = dataset; + this.dataFile = null; + this.fileMetadata = null; + this.apiToken = apiToken; + this.localeCode = localeCode; + } + + public DataFile getDataFile() { + return dataFile; + } + + public FileMetadata getFileMetadata() { + return fileMetadata; + } + + public ApiToken getApiToken() { + return apiToken; + } + + public String getLocaleCode() { + return localeCode; + } + + public String getQueryParam(String key, String value) { + String tokenValue = null; + tokenValue = getTokenValue(value); + if (tokenValue != null) { + return key + '=' + tokenValue; + } else { + return null; + } + } + + /** + * Tries to replace all occurrences of {} with the value for the + * corresponding ReservedWord + * + * @param url - the input string in which to replace tokens, normally a url + * @throws IllegalArgumentException if there is no matching ReservedWord or if + * the configuation of this instance doesn't + * have values for this ReservedWord (e.g. + * asking for FILE_PID when using the dataset + * constructor, etc.) + */ + public String replaceTokensWithValues(String url) { + String newUrl = url; + Pattern pattern = Pattern.compile("(\\{.*?\\})"); + Matcher matcher = pattern.matcher(url); + while(matcher.find()) { + String token = matcher.group(1); + ReservedWord reservedWord = ReservedWord.fromString(token); + String tValue = getTokenValue(token); + logger.info("Replacing " + reservedWord.toString() + " with " + tValue + " in " + newUrl); + newUrl = newUrl.replace(reservedWord.toString(), tValue); + } + return newUrl; + } + + private String getTokenValue(String value) { + ReservedWord reservedWord = ReservedWord.fromString(value); + switch (reservedWord) { + case FILE_ID: + // getDataFile is never null for file tools because of the constructor + return getDataFile().getId().toString(); + case FILE_PID: + GlobalId filePid = getDataFile().getGlobalId(); + if (filePid != null) { + return getDataFile().getGlobalId().asString(); + } + break; + case SITE_URL: + return SystemConfig.getDataverseSiteUrlStatic(); + case API_TOKEN: + String apiTokenString = null; + ApiToken theApiToken = getApiToken(); + if (theApiToken != null) { + apiTokenString = theApiToken.getTokenString(); + } + return apiTokenString; + case DATASET_ID: + return dataset.getId().toString(); + case DATASET_PID: + return dataset.getGlobalId().asString(); + case DATASET_VERSION: + String versionString = null; + if (fileMetadata != null) { // true for file case + versionString = fileMetadata.getDatasetVersion().getFriendlyVersionNumber(); + } else { // Dataset case - return the latest visible version (unless/until the dataset + // case allows specifying a version) + if (getApiToken() != null) { + versionString = dataset.getLatestVersion().getFriendlyVersionNumber(); + } else { + versionString = dataset.getLatestVersionForCopy().getFriendlyVersionNumber(); + } + } + if (("DRAFT").equals(versionString)) { + versionString = ":draft"; // send the token needed in api calls that can be substituted for a numeric + // version. + } + return versionString; + case FILE_METADATA_ID: + if (fileMetadata != null) { // true for file case + return fileMetadata.getId().toString(); + } + case LOCALE_CODE: + return getLocaleCode(); + default: + break; + } + throw new IllegalArgumentException("Cannot replace reserved word: " + value); + } + + public enum ReservedWord { + + // TODO: Research if a format like "{reservedWord}" is easily parse-able or if + // another format would be + // better. The choice of curly braces is somewhat arbitrary, but has been + // observed in documentation for + // various REST APIs. For example, "Variable substitutions will be made when a + // variable is named in {brackets}." + // from https://swagger.io/specification/#fixed-fields-29 but that's for URLs. + FILE_ID("fileId"), FILE_PID("filePid"), SITE_URL("siteUrl"), API_TOKEN("apiToken"), + // datasetId is the database id + DATASET_ID("datasetId"), + // datasetPid is the DOI or Handle + DATASET_PID("datasetPid"), DATASET_VERSION("datasetVersion"), FILE_METADATA_ID("fileMetadataId"), + LOCALE_CODE("localeCode"); + + private final String text; + private final String START = "{"; + private final String END = "}"; + + private ReservedWord(final String text) { + this.text = START + text + END; + } + + /** + * This is a centralized method that enforces that only reserved words are + * allowed to be used by external tools. External tool authors cannot pass their + * own query parameters through Dataverse such as "mode=mode1". + * + * @throws IllegalArgumentException + */ + public static ReservedWord fromString(String text) throws IllegalArgumentException { + if (text != null) { + for (ReservedWord reservedWord : ReservedWord.values()) { + if (text.equals(reservedWord.text)) { + return reservedWord; + } + } + } + // TODO: Consider switching to a more informative message that enumerates the + // valid reserved words. + boolean moreInformativeMessage = false; + if (moreInformativeMessage) { + throw new IllegalArgumentException( + "Unknown reserved word: " + text + ". A reserved word must be one of these values: " + + Arrays.asList(ReservedWord.values()) + "."); + } else { + throw new IllegalArgumentException("Unknown reserved word: " + text); + } + } + + @Override + public String toString() { + return text; + } + } +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java new file mode 100644 index 00000000000..ffc6b813045 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java @@ -0,0 +1,50 @@ +package edu.harvard.iq.dataverse.util; + +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +public class UrlTokenUtilTest { + + @Test + public void testGetToolUrlWithOptionalQueryParameters() { + + DataFile dataFile = new DataFile(); + dataFile.setId(42l); + FileMetadata fmd = new FileMetadata(); + DatasetVersion dv = new DatasetVersion(); + Dataset ds = new Dataset(); + ds.setId(50L); + ds.setGlobalId(new GlobalId("doi:10.5072/FK2ABCDEF")); + dv.setDataset(ds); + fmd.setDatasetVersion(dv); + List fmdl = new ArrayList(); + fmdl.add(fmd); + dataFile.setFileMetadatas(fmdl); + ApiToken apiToken = new ApiToken(); + apiToken.setTokenString("7196b5ce-f200-4286-8809-03ffdbc255d7"); + URLTokenUtil urlTokenUtil = new URLTokenUtil(dataFile, apiToken, fmd, "en"); + assertEquals("en", urlTokenUtil.replaceTokensWithValues("{localeCode}")); + assertEquals("42 test en", urlTokenUtil.replaceTokensWithValues("{fileId} test {localeCode}")); + assertEquals("42 test en", urlTokenUtil.replaceTokensWithValues("{fileId} test {localeCode}")); + + assertEquals("https://librascholar.org/api/files/42/metadata?key=" + apiToken.getTokenString(), urlTokenUtil.replaceTokensWithValues("{siteUrl}/api/files/{fileId}/metadata?key={apiToken}")); + + URLTokenUtil urlTokenUtil2 = new URLTokenUtil(ds, apiToken, "en"); + assertEquals("https://librascholar.org/api/datasets/50?key=" + apiToken.getTokenString(), urlTokenUtil2.replaceTokensWithValues("{siteUrl}/api/datasets/{datasetId}?key={apiToken}")); + assertEquals("https://librascholar.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2ABCDEF&key=" + apiToken.getTokenString(), urlTokenUtil2.replaceTokensWithValues("{siteUrl}/api/datasets/:persistentId/?persistentId={datasetPid}&key={apiToken}")); + } +} From 49286e361964161cc8a02d092037fd3919d6c81d Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Thu, 23 Jun 2022 12:18:56 +0200 Subject: [PATCH 019/232] license sorting: renamed sql script --- ...-sorting_licenses.sql => V5.11.0.1__8671-sorting_licenses.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.10.1.3__8671-sorting_licenses.sql => V5.11.0.1__8671-sorting_licenses.sql} (100%) diff --git a/src/main/resources/db/migration/V5.10.1.3__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.11.0.1__8671-sorting_licenses.sql similarity index 100% rename from src/main/resources/db/migration/V5.10.1.3__8671-sorting_licenses.sql rename to src/main/resources/db/migration/V5.11.0.1__8671-sorting_licenses.sql From 0c22b1839f6977f4ae0253905aa740cdac27976c Mon Sep 17 00:00:00 2001 From: Robert Treacy Date: Fri, 15 Jul 2022 17:45:05 -0400 Subject: [PATCH 020/232] sending a list of allowed api calls to DPCreator --- .../dataverse/externaltools/ExternalTool.java | 24 +++---- .../externaltools/ExternalToolHandler.java | 66 ++++++++++++++----- .../ExternalToolServiceBean.java | 6 +- .../iq/dataverse/util/URLTokenUtil.java | 6 +- 4 files changed, 70 insertions(+), 32 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 476181af852..79c0e3dd8f1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -41,7 +41,7 @@ public class ExternalTool implements Serializable { public static final String TOOL_PARAMETERS = "toolParameters"; public static final String CONTENT_TYPE = "contentType"; public static final String TOOL_NAME = "toolName"; - public static final String ALLOWED_URLS = "allowedUrls"; + public static final String ALLOWED_API_CALLS = "allowedApiCalls"; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -103,7 +103,7 @@ public class ExternalTool implements Serializable { * for POST headers, as in DPCreator */ @Column(nullable = true, columnDefinition = "TEXT") - private String allowedUrls; + private String allowedApiCalls; /** * This default constructor is only here to prevent this error at @@ -130,7 +130,7 @@ public ExternalTool(String displayName, String toolName, String description, Lis this.contentType = contentType; } - public ExternalTool(String displayName, String toolName, String description, List externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType, String allowedUrls) { + public ExternalTool(String displayName, String toolName, String description, List externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType, String allowedApiCalls) { this.displayName = displayName; this.toolName = toolName; this.description = description; @@ -139,7 +139,7 @@ public ExternalTool(String displayName, String toolName, String description, Lis this.toolUrl = toolUrl; this.toolParameters = toolParameters; this.contentType = contentType; - this.allowedUrls = allowedUrls; + this.allowedApiCalls = allowedApiCalls; } public enum Type { @@ -293,8 +293,8 @@ public JsonObjectBuilder toJson() { if (getContentType() != null) { jab.add(CONTENT_TYPE, getContentType()); } - if (getAllowedUrls()!= null) { - jab.add(ALLOWED_URLS,getAllowedUrls()); + if (getAllowedApiCalls()!= null) { + jab.add(ALLOWED_API_CALLS,getAllowedApiCalls()); } return jab; } @@ -322,17 +322,17 @@ public String getDisplayNameLang() { } /** - * @return the allowedUrls + * @return the allowedApiCalls */ - public String getAllowedUrls() { - return allowedUrls; + public String getAllowedApiCalls() { + return allowedApiCalls; } /** - * @param allowedUrls the allowedUrls to set + * @param allowedApiCalls the allowedApiCalls to set */ - public void setAllowedUrls(String allowedUrls) { - this.allowedUrls = allowedUrls; + public void setAllowedApiCalls(String allowedApiCalls) { + this.allowedApiCalls = allowedApiCalls; } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 8a1e9661e3a..83440608350 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.util.UrlSignerUtil; import java.io.IOException; import java.io.StringReader; +import java.io.StringWriter; import java.net.HttpURLConnection; import java.net.URI; import java.net.http.HttpClient; @@ -21,9 +22,11 @@ import javax.json.Json; import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonReader; import javax.json.JsonString; +import javax.json.JsonWriter; import javax.ws.rs.HttpMethod; /** @@ -33,17 +36,17 @@ */ public class ExternalToolHandler extends URLTokenUtil { /** - * @return the allowedUrls + * @return the allowedApiCalls */ - public String getAllowedUrls() { - return allowedUrls; + public String getAllowedApiCalls() { + return allowedApiCalls; } /** - * @param allowedUrls the allowedUrls to set + * @param allowedApiCalls the allowedApiCalls to set */ - public void setAllowedUrls(String allowedUrls) { - this.allowedUrls = allowedUrls; + public void setAllowedApiCalls(String allowedApiCalls) { + this.allowedApiCalls = allowedApiCalls; } /** @@ -59,7 +62,7 @@ public void setUser(String user) { private String toolContext; private String user; private String siteUrl; - private String allowedUrls; + private String allowedApiCalls; /** * File level tool @@ -113,14 +116,44 @@ public String handleRequest(boolean preview) { params.add(param); } }); - }); - if (requestMethod.equals(HttpMethod.POST)){ + }); + + StringWriter allowedApiCallsStringWriter = new StringWriter(); + String allowedApis; + try (JsonWriter jsonWriter = Json.createWriter(allowedApiCallsStringWriter)) { + JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + allowedApiCalls = externalTool.getAllowedApiCalls(); + JsonReader jsonReaderApis = Json.createReader(new StringReader(allowedApiCalls)); + JsonObject objApis = jsonReaderApis.readObject(); + JsonArray apis = objApis.getJsonArray("apis"); + apis.getValuesAs(JsonObject.class).forEach(((apiObj) -> { + String name = apiObj.getJsonString("name").toString(); + String httpmethod = apiObj.getJsonString("method").toString(); + int timeout = apiObj.getInt("timeOut"); + String apiPath = replaceTokensWithValues(apiObj.getJsonString("urlTemplate").toString()); + String url = UrlSignerUtil.signUrl(apiPath, timeout, user,httpmethod, getApiToken().getTokenString()); + jsonArrayBuilder.add( + Json.createObjectBuilder().add("name", name) + .add("httpMethod", httpmethod) + .add("signedUrl", url) + .add("timeOut", timeout)); + })); + JsonArray allowedApiCallsArray = jsonArrayBuilder.build(); + jsonWriter.writeArray(allowedApiCallsArray); + allowedApis = allowedApiCallsStringWriter.toString(); try { - return postFormData(obj.getJsonNumber("timeOut").intValue(), params); - } catch (IOException | InterruptedException ex) { + allowedApiCallsStringWriter.close(); + } catch (IOException ex) { Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); } } + if (requestMethod.equals(HttpMethod.POST)){ + try { + return postFormData(allowedApis); + } catch (IOException | InterruptedException ex) { + Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); + } + } if (!preview) { return "?" + String.join("&", params); } else { @@ -129,14 +162,11 @@ public String handleRequest(boolean preview) { } - private String postFormData(Integer timeout,List params ) throws IOException, InterruptedException{ - String url = ""; -// Integer timeout = obj.getJsonNumber("timeOut").intValue(); - url = UrlSignerUtil.signUrl(siteUrl, timeout, user, HttpMethod.POST, getApiToken().getTokenString()); + private String postFormData(String allowedApis ) throws IOException, InterruptedException{ + String url = null; HttpClient client = HttpClient.newHttpClient(); - HttpRequest request = HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(String.join("&", params))).uri(URI.create(externalTool.getToolUrl())) - .header("Content-Type", "application/x-www-form-urlencoded") - .header("signedUrl", url) + HttpRequest request = HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(allowedApis)).uri(URI.create(externalTool.getToolUrl())) + .header("Content-Type", "application/json") .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); boolean redirect=false; diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index d49d66c26f7..ad7cee14e4a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -169,6 +169,8 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { String toolUrl = getRequiredTopLevelField(jsonObject, TOOL_URL); JsonObject toolParametersObj = jsonObject.getJsonObject(TOOL_PARAMETERS); JsonArray queryParams = toolParametersObj.getJsonArray("queryParameters"); + JsonObject allowedApiCallsObj = jsonObject.getJsonObject(ALLOWED_API_CALLS); + JsonArray apis = allowedApiCallsObj.getJsonArray("apis"); boolean allRequiredReservedWordsFound = false; if (scope.equals(Scope.FILE)) { List requiredReservedWordCandidates = new ArrayList<>(); @@ -221,8 +223,10 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { } String toolParameters = toolParametersObj.toString(); + String allowedApiCalls = allowedApiCallsObj.toString(); - return new ExternalTool(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType); +// return new ExternalTool(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType); + return new ExternalTool(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType, allowedApiCalls); } private static String getRequiredTopLevelField(JsonObject jsonObject, String key) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java index 78280cd0f0f..1a1e92a2802 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java @@ -166,6 +166,8 @@ private String getTokenValue(String value) { } case LOCALE_CODE: return getLocaleCode(); + case ALLOWED_API_CALLS: + default: break; } @@ -186,7 +188,9 @@ public enum ReservedWord { DATASET_ID("datasetId"), // datasetPid is the DOI or Handle DATASET_PID("datasetPid"), DATASET_VERSION("datasetVersion"), FILE_METADATA_ID("fileMetadataId"), - LOCALE_CODE("localeCode"); + LOCALE_CODE("localeCode"), + ALLOWED_API_CALLS ("allowedApiCalls"); + private final String text; private final String START = "{"; From 1b31e6c2c8b3eae21eb85818bf025dcd11d17f24 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 20 Jul 2022 17:24:37 -0400 Subject: [PATCH 021/232] tweak json read/write, getString, cleanup, logging --- .../dataverse/externaltools/ExternalTool.java | 8 +- .../externaltools/ExternalToolHandler.java | 80 +++++++------------ .../ExternalToolServiceBean.java | 4 +- 3 files changed, 36 insertions(+), 56 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 79c0e3dd8f1..bda9ebad063 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -20,7 +20,6 @@ import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.OneToMany; -import javax.persistence.Transient; /** * A specification or definition for how an external tool is intended to @@ -30,8 +29,6 @@ @Entity public class ExternalTool implements Serializable { - private static final Logger logger = Logger.getLogger(ExternalToolServiceBean.class.getCanonicalName()); - public static final String DISPLAY_NAME = "displayName"; public static final String DESCRIPTION = "description"; public static final String LEGACY_SINGLE_TYPE = "type"; @@ -99,8 +96,9 @@ public class ExternalTool implements Serializable { private String contentType; /** - * Path for retrieving data through the REST api. Used to build signedUrls - * for POST headers, as in DPCreator + * Set of API calls the tool would like to be able to use (e,.g. for retrieving + * data through the Dataverse REST api). Used to build signedUrls for POST + * headers, as in DPCreator */ @Column(nullable = true, columnDefinition = "TEXT") private String allowedApiCalls; diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 83440608350..54489953606 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -7,6 +7,8 @@ import edu.harvard.iq.dataverse.util.URLTokenUtil; import edu.harvard.iq.dataverse.util.UrlSignerUtil; +import edu.harvard.iq.dataverse.util.json.JsonUtil; + import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; @@ -35,19 +37,6 @@ * such as constructing a URL to access that file. */ public class ExternalToolHandler extends URLTokenUtil { - /** - * @return the allowedApiCalls - */ - public String getAllowedApiCalls() { - return allowedApiCalls; - } - - /** - * @param allowedApiCalls the allowedApiCalls to set - */ - public void setAllowedApiCalls(String allowedApiCalls) { - this.allowedApiCalls = allowedApiCalls; - } /** * @param user the user to set @@ -61,8 +50,7 @@ public void setUser(String user) { private String requestMethod; private String toolContext; private String user; - private String siteUrl; - private String allowedApiCalls; + /** * File level tool @@ -98,9 +86,7 @@ public String handleRequest() { // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. public String handleRequest(boolean preview) { - String toolParameters = externalTool.getToolParameters(); - JsonReader jsonReader = Json.createReader(new StringReader(toolParameters)); - JsonObject obj = jsonReader.readObject(); + JsonObject obj = JsonUtil.getJsonObject(externalTool.getToolParameters()); JsonString method = obj.getJsonString("httpMethod"); requestMethod = method!=null?method.getString():HttpMethod.GET; JsonArray queryParams = obj.getJsonArray("queryParameters"); @@ -118,36 +104,32 @@ public String handleRequest(boolean preview) { }); }); - StringWriter allowedApiCallsStringWriter = new StringWriter(); String allowedApis; - try (JsonWriter jsonWriter = Json.createWriter(allowedApiCallsStringWriter)) { - JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); - allowedApiCalls = externalTool.getAllowedApiCalls(); - JsonReader jsonReaderApis = Json.createReader(new StringReader(allowedApiCalls)); - JsonObject objApis = jsonReaderApis.readObject(); - JsonArray apis = objApis.getJsonArray("apis"); - apis.getValuesAs(JsonObject.class).forEach(((apiObj) -> { - String name = apiObj.getJsonString("name").toString(); - String httpmethod = apiObj.getJsonString("method").toString(); - int timeout = apiObj.getInt("timeOut"); - String apiPath = replaceTokensWithValues(apiObj.getJsonString("urlTemplate").toString()); - String url = UrlSignerUtil.signUrl(apiPath, timeout, user,httpmethod, getApiToken().getTokenString()); - jsonArrayBuilder.add( - Json.createObjectBuilder().add("name", name) - .add("httpMethod", httpmethod) - .add("signedUrl", url) - .add("timeOut", timeout)); - })); - JsonArray allowedApiCallsArray = jsonArrayBuilder.build(); - jsonWriter.writeArray(allowedApiCallsArray); - allowedApis = allowedApiCallsStringWriter.toString(); - try { - allowedApiCallsStringWriter.close(); - } catch (IOException ex) { - Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); - } - } - if (requestMethod.equals(HttpMethod.POST)){ + JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + + JsonObject objApis = JsonUtil.getJsonObject(externalTool.getAllowedApiCalls()); + + JsonArray apis = objApis.getJsonArray("apis"); + apis.getValuesAs(JsonObject.class).forEach(((apiObj) -> { + String name = apiObj.getJsonString("name").getString(); + String httpmethod = apiObj.getJsonString("method").getString(); + int timeout = apiObj.getInt("timeOut"); + String urlTemplate = apiObj.getJsonString("urlTemplate").getString(); + logger.fine("URL Template: " + urlTemplate); + String apiPath = replaceTokensWithValues(urlTemplate); + logger.fine("URL WithTokens: " + apiPath); + String url = UrlSignerUtil.signUrl(apiPath, timeout, user, httpmethod, getApiToken().getTokenString()); + logger.fine("Signed URL: " + url); + jsonArrayBuilder.add(Json.createObjectBuilder().add("name", name).add("httpMethod", httpmethod) + .add("signedUrl", url).add("timeOut", timeout)); + })); + JsonArray allowedApiCallsArray = jsonArrayBuilder.build(); + allowedApis = JsonUtil.prettyPrint(allowedApiCallsArray); + logger.fine("Sending these signed URLS: " + allowedApis); + + //ToDo - if the allowedApiCalls() are defined, could/should we send them to tools using GET as well? + + if (requestMethod.equals(HttpMethod.POST)) { try { return postFormData(allowedApis); } catch (IOException | InterruptedException ex) { @@ -167,7 +149,7 @@ private String postFormData(String allowedApis ) throws IOException, Interrupted HttpClient client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(allowedApis)).uri(URI.create(externalTool.getToolUrl())) .header("Content-Type", "application/json") - .build(); + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); boolean redirect=false; int status = response.statusCode(); @@ -178,7 +160,7 @@ private String postFormData(String allowedApis ) throws IOException, Interrupted redirect = true; } } - if (redirect=true){ + if (redirect==true){ String newUrl = response.headers().firstValue("location").get(); toolContext = "http://" + response.uri().getAuthority(); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index ad7cee14e4a..432aa26714d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -6,6 +6,7 @@ import edu.harvard.iq.dataverse.externaltools.ExternalTool.Type; import edu.harvard.iq.dataverse.util.URLTokenUtil; import edu.harvard.iq.dataverse.util.URLTokenUtil.ReservedWord; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.externaltools.ExternalTool.Scope; import java.io.StringReader; @@ -151,8 +152,7 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { if (manifest == null || manifest.isEmpty()) { throw new IllegalArgumentException("External tool manifest was null or empty!"); } - JsonReader jsonReader = Json.createReader(new StringReader(manifest)); - JsonObject jsonObject = jsonReader.readObject(); + JsonObject jsonObject = JsonUtil.getJsonObject(manifest); //Note: ExternalToolServiceBeanTest tests are dependent on the order of these retrievals String displayName = getRequiredTopLevelField(jsonObject, DISPLAY_NAME); String toolName = getOptionalTopLevelField(jsonObject, TOOL_NAME); From d4189f37aad75f41f6a4ef3908aed761dec81061 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 4 Aug 2022 18:20:37 -0400 Subject: [PATCH 022/232] add signer tests, flip param order so sign/validate match, fix val bug --- .../iq/dataverse/api/AbstractApiBean.java | 2 +- .../iq/dataverse/util/UrlSignerUtil.java | 253 +++++++++--------- .../iq/dataverse/util/UrlSignerUtilTest.java | 50 ++++ 3 files changed, 178 insertions(+), 127 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 4adac3feace..7ddde7064fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -444,7 +444,7 @@ private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { String signedUrl = httpRequest.getRequestURL().toString(); String method = httpRequest.getMethod(); - boolean validated = UrlSignerUtil.isValidUrl(signedUrl, method, user, key); + boolean validated = UrlSignerUtil.isValidUrl(signedUrl, user, method, key); if (validated){ authUser = targetUser; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java index 1da1797a8ae..b11334520e6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -17,134 +17,135 @@ */ public class UrlSignerUtil { - private static final Logger logger = Logger.getLogger(UrlSignerUtil.class.getName()); + private static final Logger logger = Logger.getLogger(UrlSignerUtil.class.getName()); - /** - * - * @param baseUrl - the URL to sign - cannot contain query params - * "until","user", "method", or "token" - * @param timeout - how many minutes to make the URL valid for (note - time skew - * between the creator and receiver could affect the validation - * @param user - a string representing the user - should be understood by the - * creator/receiver - * @param method - one of the HTTP methods - * @param key - a secret key shared by the creator/receiver. In Dataverse - * this could be an APIKey (when sending URL to a tool that will - * use it to retrieve info from Dataverse) - * @return - the signed URL - */ - public static String signUrl(String baseUrl, Integer timeout, String user, String method, String key) { - StringBuilder signedUrl = new StringBuilder(baseUrl); + /** + * + * @param baseUrl - the URL to sign - cannot contain query params + * "until","user", "method", or "token" + * @param timeout - how many minutes to make the URL valid for (note - time skew + * between the creator and receiver could affect the validation + * @param user - a string representing the user - should be understood by the + * creator/receiver + * @param method - one of the HTTP methods + * @param key - a secret key shared by the creator/receiver. In Dataverse + * this could be an APIKey (when sending URL to a tool that will + * use it to retrieve info from Dataverse) + * @return - the signed URL + */ + public static String signUrl(String baseUrl, Integer timeout, String user, String method, String key) { + StringBuilder signedUrl = new StringBuilder(baseUrl); - boolean firstParam = true; - if (baseUrl.contains("?")) { - firstParam = false; - } - if (timeout != null) { - LocalDateTime validTime = LocalDateTime.now(); - validTime = validTime.plusMinutes(timeout); - validTime.toString(); - signedUrl.append(firstParam ? "?" : "&").append("until=").append(validTime); - firstParam=false; - } - if (user != null) { - signedUrl.append(firstParam ? "?" : "&").append("user=").append(user); - firstParam=false; - } - if (method != null) { - signedUrl.append(firstParam ? "?" : "&").append("method=").append(method); - } - signedUrl.append("&token="); - logger.fine("String to sign: " + signedUrl.toString() + ""); - signedUrl.append(DigestUtils.sha512Hex(signedUrl.toString() + key)); - logger.fine("Generated Signed URL: " + signedUrl.toString()); - if (logger.isLoggable(Level.FINE)) { - logger.fine( - "URL signature is " + (isValidUrl(signedUrl.toString(), method, user, key) ? "valid" : "invalid")); - } - return signedUrl.toString(); - } + boolean firstParam = true; + if (baseUrl.contains("?")) { + firstParam = false; + } + if (timeout != null) { + LocalDateTime validTime = LocalDateTime.now(); + validTime = validTime.plusMinutes(timeout); + validTime.toString(); + signedUrl.append(firstParam ? "?" : "&").append("until=").append(validTime); + firstParam = false; + } + if (user != null) { + signedUrl.append(firstParam ? "?" : "&").append("user=").append(user); + firstParam = false; + } + if (method != null) { + signedUrl.append(firstParam ? "?" : "&").append("method=").append(method); + firstParam=false; + } + signedUrl.append(firstParam ? "?" : "&").append("token="); + logger.fine("String to sign: " + signedUrl.toString() + ""); + signedUrl.append(DigestUtils.sha512Hex(signedUrl.toString() + key)); + logger.fine("Generated Signed URL: " + signedUrl.toString()); + if (logger.isLoggable(Level.FINE)) { + logger.fine( + "URL signature is " + (isValidUrl(signedUrl.toString(), user, method, key) ? "valid" : "invalid")); + } + return signedUrl.toString(); + } - /** - * This method will only return true if the URL and parameters except the - * "token" are unchanged from the original/match the values sent to this method, - * and the "token" parameter matches what this method recalculates using the - * shared key THe method also assures that the "until" timestamp is after the - * current time. - * - * @param signedUrl - the signed URL as received from Dataverse - * @param method - an HTTP method. If provided, the method in the URL must - * match - * @param user - a string representing the user, if provided the value must - * match the one in the url - * @param key - the shared secret key to be used in validation - * @return - true if valid, false if not: e.g. the key is not the same as the - * one used to generate the "token" any part of the URL preceding the - * "token" has been altered the method doesn't match (e.g. the server - * has received a POST request and the URL only allows GET) the user - * string doesn't match (e.g. the server knows user A is logged in, but - * the URL is only for user B) the url has expired (was used after the - * until timestamp) - */ - public static boolean isValidUrl(String signedUrl, String method, String user, String key) { - boolean valid = true; - try { - URL url = new URL(signedUrl); - List params = URLEncodedUtils.parse(url.getQuery(), Charset.forName("UTF-8")); - String hash = null; - String dateString = null; - String allowedMethod = null; - String allowedUser = null; - for (NameValuePair nvp : params) { - if (nvp.getName().equals("token")) { - hash = nvp.getValue(); - logger.fine("Hash: " + hash); - } - if (nvp.getName().equals("until")) { - dateString = nvp.getValue(); - logger.fine("Until: " + dateString); - } - if (nvp.getName().equals("method")) { - allowedMethod = nvp.getValue(); - logger.fine("Method: " + allowedMethod); - } - if (nvp.getName().equals("user")) { - allowedUser = nvp.getValue(); - logger.fine("User: " + allowedUser); - } - } + /** + * This method will only return true if the URL and parameters except the + * "token" are unchanged from the original/match the values sent to this method, + * and the "token" parameter matches what this method recalculates using the + * shared key THe method also assures that the "until" timestamp is after the + * current time. + * + * @param signedUrl - the signed URL as received from Dataverse + * @param method - an HTTP method. If provided, the method in the URL must + * match + * @param user - a string representing the user, if provided the value must + * match the one in the url + * @param key - the shared secret key to be used in validation + * @return - true if valid, false if not: e.g. the key is not the same as the + * one used to generate the "token" any part of the URL preceding the + * "token" has been altered the method doesn't match (e.g. the server + * has received a POST request and the URL only allows GET) the user + * string doesn't match (e.g. the server knows user A is logged in, but + * the URL is only for user B) the url has expired (was used after the + * until timestamp) + */ + public static boolean isValidUrl(String signedUrl, String user, String method, String key) { + boolean valid = true; + try { + URL url = new URL(signedUrl); + List params = URLEncodedUtils.parse(url.getQuery(), Charset.forName("UTF-8")); + String hash = null; + String dateString = null; + String allowedMethod = null; + String allowedUser = null; + for (NameValuePair nvp : params) { + if (nvp.getName().equals("token")) { + hash = nvp.getValue(); + logger.fine("Hash: " + hash); + } + if (nvp.getName().equals("until")) { + dateString = nvp.getValue(); + logger.fine("Until: " + dateString); + } + if (nvp.getName().equals("method")) { + allowedMethod = nvp.getValue(); + logger.fine("Method: " + allowedMethod); + } + if (nvp.getName().equals("user")) { + allowedUser = nvp.getValue(); + logger.fine("User: " + allowedUser); + } + } - int index = signedUrl.indexOf("&token="); - // Assuming the token is last - doesn't have to be, but no reason for the URL - // params to be rearranged either, and this should only cause false negatives if - // it does happen - String urlToHash = signedUrl.substring(0, index + 7); - logger.fine("String to hash: " + urlToHash + ""); - String newHash = DigestUtils.sha512Hex(urlToHash + key); - logger.fine("Calculated Hash: " + newHash); - if (!hash.equals(newHash)) { - logger.fine("Hash doesn't match"); - valid = false; - } - if (dateString != null && LocalDateTime.parse(dateString).isBefore(LocalDateTime.now())) { - logger.fine("Url is expired"); - valid = false; - } - if (method != null && !method.equals(allowedMethod)) { - logger.fine("Method doesn't match"); - valid = false; - } - if (user != null && !user.equals(allowedUser)) { - logger.fine("User doesn't match"); - valid = false; - } - } catch (Throwable t) { - // Want to catch anything like null pointers, etc. to force valid=false upon any - // error - logger.warning("Bad URL: " + signedUrl + " : " + t.getMessage()); - valid = false; - } - return valid; - } + int index = signedUrl.indexOf(((dateString==null && allowedMethod==null && allowedUser==null) ? "?":"&") + "token="); + // Assuming the token is last - doesn't have to be, but no reason for the URL + // params to be rearranged either, and this should only cause false negatives if + // it does happen + String urlToHash = signedUrl.substring(0, index + 7); + logger.fine("String to hash: " + urlToHash + ""); + String newHash = DigestUtils.sha512Hex(urlToHash + key); + logger.fine("Calculated Hash: " + newHash); + if (!hash.equals(newHash)) { + logger.fine("Hash doesn't match"); + valid = false; + } + if (dateString != null && LocalDateTime.parse(dateString).isBefore(LocalDateTime.now())) { + logger.fine("Url is expired"); + valid = false; + } + if (method != null && !method.equals(allowedMethod)) { + logger.fine("Method doesn't match"); + valid = false; + } + if (user != null && !user.equals(allowedUser)) { + logger.fine("User doesn't match"); + valid = false; + } + } catch (Throwable t) { + // Want to catch anything like null pointers, etc. to force valid=false upon any + // error + logger.warning("Bad URL: " + signedUrl + " : " + t.getMessage()); + valid = false; + } + return valid; + } -} \ No newline at end of file +} diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java new file mode 100644 index 00000000000..2b9d507758f --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlSignerUtilTest.java @@ -0,0 +1,50 @@ +package edu.harvard.iq.dataverse.util; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.junit.Test; + +public class UrlSignerUtilTest { + + @Test + public void testSignAndValidate() { + + final String url = "http://localhost:8080/api/test1"; + final String get = "GET"; + final String post = "POST"; + + final String user1 = "Alice"; + final String user2 = "Bob"; + final int tooQuickTimeout = -1; + final int longTimeout = 1000; + final String key = "abracadabara open sesame"; + final String badkey = "abracadabara open says me"; + + Logger.getLogger(UrlSignerUtil.class.getName()).setLevel(Level.FINE); + + String signedUrl1 = UrlSignerUtil.signUrl(url, longTimeout, user1, get, key); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl1, user1, get, key)); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl1, user1, null, key)); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl1, null, get, key)); + + assertFalse(UrlSignerUtil.isValidUrl(signedUrl1, null, get, badkey)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl1, user2, get, key)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl1, user1, post, key)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl1.replace(user1, user2), user1, get, key)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl1.replace(user1, user2), user2, get, key)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl1.replace(user1, user2), null, get, key)); + + String signedUrl2 = UrlSignerUtil.signUrl(url, null, null, null, key); + assertTrue(UrlSignerUtil.isValidUrl(signedUrl2, null, null, key)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl2, null, post, key)); + assertFalse(UrlSignerUtil.isValidUrl(signedUrl2, user1, null, key)); + + String signedUrl3 = UrlSignerUtil.signUrl(url, tooQuickTimeout, user1, get, key); + + assertFalse(UrlSignerUtil.isValidUrl(signedUrl3, user1, get, key)); + } +} From 0124da0e8263698be7e04ee96e4f5b61f93ea08e Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 8 Aug 2022 11:58:25 +0200 Subject: [PATCH 023/232] added sortOrder column in the license test file --- src/test/resources/json/license.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/resources/json/license.json b/src/test/resources/json/license.json index dd251322110..00502ded9a6 100644 --- a/src/test/resources/json/license.json +++ b/src/test/resources/json/license.json @@ -3,5 +3,6 @@ "uri": "http://dataverse..org/licenses/test/1.0", "iconUrl": "http://dataverse.org/licenses/test/1.0/icon.png", "shortDescription": "Dataverse Test License v1.0.", - "active": false + "active": false, + "sortOrder": 1 } From 5c7674c3c2f61de4e65b90b8d8e20db382127dc5 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 8 Aug 2022 12:10:32 +0200 Subject: [PATCH 024/232] added icompatibilities mention for the license sorting order field --- doc/release-notes/8671-sorting-licenses.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/8671-sorting-licenses.md b/doc/release-notes/8671-sorting-licenses.md index 34ad697d5a7..4ceb9ec056f 100644 --- a/doc/release-notes/8671-sorting-licenses.md +++ b/doc/release-notes/8671-sorting-licenses.md @@ -1,3 +1,7 @@ ## License sorting -Licenses as shown in the dropdown in UI can be now sorted by the superusers. See [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide for reference. \ No newline at end of file +Licenses as shown in the dropdown in UI can be now sorted by the superusers. See [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide for reference. + +## Backward Incompatibilities + +License files are now required to contain the new "sortOrder" column. When attempting to create a new license without this field, an error would be returned. See [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide for reference. \ No newline at end of file From 9ad8d6401eb0aa25fe4808e76322421376aa924b Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 8 Aug 2022 12:13:38 +0200 Subject: [PATCH 025/232] renamed: V5.11.0.1__8671-sorting_licenses.sql -> V5.11.1.2__8671-sorting_licenses.sql --- ...-sorting_licenses.sql => V5.11.1.2__8671-sorting_licenses.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.11.0.1__8671-sorting_licenses.sql => V5.11.1.2__8671-sorting_licenses.sql} (100%) diff --git a/src/main/resources/db/migration/V5.11.0.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.11.1.2__8671-sorting_licenses.sql similarity index 100% rename from src/main/resources/db/migration/V5.11.0.1__8671-sorting_licenses.sql rename to src/main/resources/db/migration/V5.11.1.2__8671-sorting_licenses.sql From ca0bac2a10828207c4ec9c00e0ebdfddd85339f2 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 8 Aug 2022 15:24:14 +0200 Subject: [PATCH 026/232] added sortOrder also in the error test license --- pom.xml | 2 +- src/test/resources/json/licenseError.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index eab64e522a5..19bdee46127 100644 --- a/pom.xml +++ b/pom.xml @@ -558,7 +558,7 @@ com.jayway.restassured rest-assured - 2.4.0 + 2.9.0 test diff --git a/src/test/resources/json/licenseError.json b/src/test/resources/json/licenseError.json index 552b6acadfb..533aa7ce7dc 100644 --- a/src/test/resources/json/licenseError.json +++ b/src/test/resources/json/licenseError.json @@ -4,5 +4,6 @@ "uri": "http://dataverse..org/licenses/test/ln6", "iconUrl": "http://dataverse.org/licenses/test/ln6/icon.png", "shortDescription": "A License that must have id 6.", - "active": true + "active": true, + "sortOrder": 1 } From 2af5cb31d1864588bfe7129c54bd61a7e653e55f Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 8 Aug 2022 15:26:00 +0200 Subject: [PATCH 027/232] revert by accident commited change in pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 19bdee46127..eab64e522a5 100644 --- a/pom.xml +++ b/pom.xml @@ -558,7 +558,7 @@ com.jayway.restassured rest-assured - 2.9.0 + 2.4.0 test From 255e30865162b649283737d585f7441103919715 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 8 Aug 2022 17:18:11 +0200 Subject: [PATCH 028/232] test license created last should be at the end of the list now because of the increased sortOrder --- src/test/resources/json/license.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/json/license.json b/src/test/resources/json/license.json index 00502ded9a6..d126b1d2280 100644 --- a/src/test/resources/json/license.json +++ b/src/test/resources/json/license.json @@ -4,5 +4,5 @@ "iconUrl": "http://dataverse.org/licenses/test/1.0/icon.png", "shortDescription": "Dataverse Test License v1.0.", "active": false, - "sortOrder": 1 + "sortOrder": 1000 } From 03c242bbc6d1213baa4d3ebb1225147999ff376f Mon Sep 17 00:00:00 2001 From: chenganj Date: Mon, 8 Aug 2022 14:28:23 -0400 Subject: [PATCH 029/232] license internationalization - first commit --- .../iq/dataverse/dataset/DatasetUtil.java | 31 +++++++++++++++---- .../java/propertyFiles/License.properties | 2 ++ .../migration/V4.13.0.1__3575-usernames.sql | 2 +- ...16.0.1__5303-addColumn-to-settingTable.sql | 6 +--- src/main/webapp/dataset-license-terms.xhtml | 4 +-- .../webapp/datasetLicenseInfoFragment.xhtml | 4 +-- 6 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 src/main/java/propertyFiles/License.properties diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index ccf947b8868..c6fc207163c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -24,12 +24,8 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.logging.Logger; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; import javax.imageio.ImageIO; import org.apache.commons.io.IOUtils; import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO; @@ -566,7 +562,30 @@ public static String getLicenseIcon(DatasetVersion dsv) { public static String getLicenseDescription(DatasetVersion dsv) { License license = dsv.getTermsOfUseAndAccess().getLicense(); - return license != null ? license.getShortDescription() : BundleUtil.getStringFromBundle("license.custom.description"); + + if (license != null) { + return getLocalizedLicenseDescription(license.getName()) ; + } else { + return BundleUtil.getStringFromBundle("license.custom.description"); + } + } + + public static String getLocalizedLicenseDescription(String licenseName) { + String key = "license." + licenseName.toLowerCase().replace(" ","_") + ".description"; + if (key != null) { + try { + String _description = BundleUtil.getStringFromPropertyFile(key, "License"); + if (_description == null) { + return BundleUtil.getStringFromBundle("license.custom.description"); + } else { + return _description; + } + } catch (MissingResourceException mre) { + return BundleUtil.getStringFromBundle("license.custom.description"); + } + } else { + return BundleUtil.getStringFromBundle("license.custom.description"); + } } public static String getLocaleExternalStatus(String status) { diff --git a/src/main/java/propertyFiles/License.properties b/src/main/java/propertyFiles/License.properties new file mode 100644 index 00000000000..f6def616a04 --- /dev/null +++ b/src/main/java/propertyFiles/License.properties @@ -0,0 +1,2 @@ +license.cc0_1.0.description=Creative Commons CC0 1.0 Universal Public Domain Dedication. +license.cc_by_4.0.description=Creative Commons Attribution 4.0 International License. \ No newline at end of file diff --git a/src/main/resources/db/migration/V4.13.0.1__3575-usernames.sql b/src/main/resources/db/migration/V4.13.0.1__3575-usernames.sql index 0b1804bdfc4..9e35623c455 100644 --- a/src/main/resources/db/migration/V4.13.0.1__3575-usernames.sql +++ b/src/main/resources/db/migration/V4.13.0.1__3575-usernames.sql @@ -1 +1 @@ -CREATE UNIQUE INDEX index_authenticateduser_lower_useridentifier ON authenticateduser (lower(useridentifier)); +CREATE UNIQUE INDEX IF NOT EXISTS index_authenticateduser_lower_useridentifier ON authenticateduser (lower(useridentifier)); diff --git a/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql b/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql index 8309dacf486..db08efdab7e 100644 --- a/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql +++ b/src/main/resources/db/migration/V4.16.0.1__5303-addColumn-to-settingTable.sql @@ -4,10 +4,6 @@ ALTER TABLE setting ADD COLUMN IF NOT EXISTS ID SERIAL PRIMARY KEY; ALTER TABLE setting ADD COLUMN IF NOT EXISTS lang text; -ALTER TABLE setting - ADD CONSTRAINT non_empty_lang - CHECK (lang <> ''); - -CREATE UNIQUE INDEX unique_settings +CREATE UNIQUE INDEX IF NOT EXISTS unique_settings ON setting (name, coalesce(lang, '')); diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 38f1f38e7d6..b81fed8a6d7 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -55,8 +55,8 @@

- - #{termsOfUseAndAccess.license.name} + + #{termsOfUseAndAccess.license.name}

diff --git a/src/main/webapp/datasetLicenseInfoFragment.xhtml b/src/main/webapp/datasetLicenseInfoFragment.xhtml index 554a3d95abf..e5d10c745dd 100644 --- a/src/main/webapp/datasetLicenseInfoFragment.xhtml +++ b/src/main/webapp/datasetLicenseInfoFragment.xhtml @@ -30,12 +30,12 @@ xmlns:jsf="http://xmlns.jcp.org/jsf">
+ jsf:rendered="#{!empty DatasetUtil:getLocalizedLicenseDescription(DatasetPage.workingVersion.termsOfUseAndAccess.license.name)} }">
- +
From 6f0c2600b6ea8a7ed7091f8a4e9698e1435b9391 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 9 Aug 2022 16:39:28 -0400 Subject: [PATCH 030/232] improve error handling --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index ef08444af69..a1d0ac86e8c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1813,6 +1813,9 @@ public Response submitDatasetVersionToArchive(@PathParam("id") String dsid, Dataset ds = findDatasetOrDie(dsid); DatasetVersion dv = datasetversionService.findByFriendlyVersionNumber(ds.getId(), versionNumber); + if(dv==null) { + return error(Status.BAD_REQUEST, "Requested version not found."); + } if (dv.getArchivalCopyLocation() == null) { String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName); // Note - the user is being sent via the createDataverseRequest(au) call to the @@ -1858,7 +1861,7 @@ public void run() { return error(Status.BAD_REQUEST, "Version was already submitted for archiving."); } } catch (WrappedResponse e1) { - return error(Status.UNAUTHORIZED, "api key required"); + return e1.getResponse(); } } @@ -1949,7 +1952,7 @@ public void run() { return error(Status.BAD_REQUEST, "No unarchived published dataset versions found"); } } catch (WrappedResponse e1) { - return error(Status.UNAUTHORIZED, "api key required"); + return e1.getResponse(); } } From 1aa61ea46ae35ca547cbd7fd801b991914967d52 Mon Sep 17 00:00:00 2001 From: xflv Date: Thu, 11 Aug 2022 11:23:02 +0800 Subject: [PATCH 031/232] In order to resolve #8838 in Add CSTR to Harvard Dataverse Related Publication ID Type list, we refer to the arXiv logo and Please review the code and make the changes. --- .../source/_static/api/ddi_dataset.xml | 12 +++++++++ ...dataset-create-new-all-default-fields.json | 26 +++++++++++++++++++ scripts/api/data/metadatablocks/citation.tsv | 3 ++- scripts/issues/2102/ready-state.sql | 2 ++ .../export/openaire/OpenAireExportUtil.java | 1 + .../java/propertyFiles/citation.properties | 1 + .../export/OpenAireExportUtilTest.java | 3 +++ .../export/SchemaDotOrgExporterTest.java | 3 ++- .../dataverse/export/dataset-all-defaults.txt | 26 +++++++++++++++++++ .../dataset-create-new-all-ddi-fields.json | 26 +++++++++++++++++++ .../iq/dataverse/export/ddi/exportfull.xml | 12 +++++++++ 11 files changed, 113 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml index 79e0581131e..8b5dddacb64 100644 --- a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml +++ b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml @@ -163,8 +163,10 @@ RelatedMaterial1 RelatedMaterial2 + RelatedMaterial3 RelatedDatasets1 RelatedDatasets2 + RelatedDatasets3 @@ -183,8 +185,18 @@ + + + + RelatedPublicationIDNumber3 + + RelatedPublicationCitation3 + + + OtherReferences1 OtherReferences2 + OtherReferences3 StudyLevelErrorNotes diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index d7ae8cefbf7..e51f4d9e1b5 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -369,6 +369,32 @@ "typeClass": "primitive", "value": "http://RelatedPublicationURL2.org" } + }, + { + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation3" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "CSTR" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber3" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL3.org" + } } ] }, diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index 94aa509334f..e17db407ae7 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -110,7 +110,8 @@ publicationIDType purl 13 publicationIDType upc 14 publicationIDType url 15 - publicationIDType urn 16 + publicationIDType urn 16 + publicationIDType CSTR 17 contributorType Data Collector 0 contributorType Data Curator 1 contributorType Data Manager 2 diff --git a/scripts/issues/2102/ready-state.sql b/scripts/issues/2102/ready-state.sql index 96ccf58d865..03ab805531e 100644 --- a/scripts/issues/2102/ready-state.sql +++ b/scripts/issues/2102/ready-state.sql @@ -3284,6 +3284,7 @@ COPY controlledvocabalternate (id, strvalue, controlledvocabularyvalue_id, datas 22 United States of America 472 79 23 U.S.A. 472 79 24 YEMEN 483 79 +25 CSTR 825 30 \. @@ -4116,6 +4117,7 @@ COPY controlledvocabularyvalue (id, displayorder, identifier, strvalue, datasetf 822 29 review article 154 823 30 translation 154 824 31 other 154 +825 17 CSTR 30 \. diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index 49fe203b96d..ffce432ce3b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -964,6 +964,7 @@ public static void writeRelatedIdentifierElement(XMLStreamWriter xmlw, DatasetVe relatedIdentifierTypeMap.put("URL".toLowerCase(), "URL"); relatedIdentifierTypeMap.put("URN".toLowerCase(), "URN"); relatedIdentifierTypeMap.put("WOS".toLowerCase(), "WOS"); + relatedIdentifierTypeMap.put("CSTR".toLowerCase(), "CSTR"); } for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index bdcc48b5bf1..47356b404b8 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -250,6 +250,7 @@ controlledvocabulary.subject.social_sciences=Social Sciences controlledvocabulary.subject.other=Other controlledvocabulary.publicationIDType.ark=ark controlledvocabulary.publicationIDType.arxiv=arXiv +controlledvocabulary.publicationIDType.CSTR=CSTR controlledvocabulary.publicationIDType.bibcode=bibcode controlledvocabulary.publicationIDType.doi=doi controlledvocabulary.publicationIDType.ean13=ean13 diff --git a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java index 7f7cc203506..8064b8e20f5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java @@ -609,6 +609,9 @@ public void testWriteRelatedIdentifierElement() throws XMLStreamException, IOExc + "RelatedPublicationIDNumber1" + "" + "RelatedPublicationIDNumber2" + + "" + + "" + + "RelatedPublicationIDNumber3" + "", stringWriter.toString()); } diff --git a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java index b5453e75fe5..5f0d4dfd106 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java @@ -408,7 +408,8 @@ private static void mockDatasetFieldSvc() { new ControlledVocabularyValue(3l, "bibcode", publicationIdTypes), new ControlledVocabularyValue(4l, "doi", publicationIdTypes), new ControlledVocabularyValue(5l, "ean13", publicationIdTypes), - new ControlledVocabularyValue(6l, "handle", publicationIdTypes) + new ControlledVocabularyValue(6l, "handle", publicationIdTypes), + new ControlledVocabularyValue(17l, "CSTR", publicationIdTypes) // Etc. There are more. )); publicationChildTypes.add(datasetFieldTypeSvc.add(publicationIdTypes)); diff --git a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt index a3f0dffc767..a2a6d9c0778 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt +++ b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt @@ -362,6 +362,32 @@ "typeClass": "primitive", "value": "http://RelatedPublicationURL2.org" } + }, + { + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation3" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "CSTR" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber3" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL3.org" + } } ] }, diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json index 1b327c15496..362a4ae4d90 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json @@ -321,6 +321,32 @@ "typeClass": "primitive", "value": "http://RelatedPublicationURL2.org" } + }, + { + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation3" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "CSTR" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber3" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL3.org" + } } ] }, diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml index 0570c832e4f..4314775c7a2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml @@ -166,8 +166,10 @@ RelatedMaterial1 RelatedMaterial2 + RelatedMaterial3 RelatedDatasets1 RelatedDatasets2 + RelatedDatasets3 @@ -185,9 +187,19 @@ RelatedPublicationCitation2 + + + + + RelatedPublicationIDNumber3 + + RelatedPublicationCitation3 + + OtherReferences1 OtherReferences2 + OtherReferences3 StudyLevelErrorNotes From 8948faa4c7215acfd5bf805c8f4b23477b3cf0d8 Mon Sep 17 00:00:00 2001 From: xflv Date: Mon, 22 Aug 2022 15:21:57 +0800 Subject: [PATCH 032/232] Hi reviewer, We have fixed the lowercase and sequential issues. Please review the code and make the changes. --- scripts/api/data/metadatablocks/citation.tsv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index e17db407ae7..7cc14043b2a 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -96,7 +96,8 @@ subject Other D12 13 publicationIDType ark 0 publicationIDType arXiv 1 - publicationIDType bibcode 2 + publicationIDType bibcode 2 + publicationIDType cstr 17 publicationIDType doi 3 publicationIDType ean13 4 publicationIDType eissn 5 @@ -110,8 +111,7 @@ publicationIDType purl 13 publicationIDType upc 14 publicationIDType url 15 - publicationIDType urn 16 - publicationIDType CSTR 17 + publicationIDType urn 16 contributorType Data Collector 0 contributorType Data Curator 1 contributorType Data Manager 2 From 2f664d0886031feee1ea006d0a77b269303ccc1c Mon Sep 17 00:00:00 2001 From: xflv Date: Mon, 29 Aug 2022 14:01:45 +0800 Subject: [PATCH 033/232] We have modified the AdminIT.java file as per example #8775 provided by qqmyers, please check if the conflict has been resolved. As we do not have access to the conflict details, if the modification does not resolve the conflict, please follow up with the file name and line number of the conflict and we will follow up with the modification. --- src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index c34ee2dd4bf..df505224817 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -761,7 +761,7 @@ public void testLoadMetadataBlock_NoErrorPath() { Map>> data = JsonPath.from(body).getMap("data"); assertEquals(1, data.size()); List> addedElements = data.get("added"); - assertEquals(321, addedElements.size()); + assertEquals(322, addedElements.size()); Map statistics = new HashMap<>(); for (Map unit : addedElements) { @@ -777,7 +777,7 @@ public void testLoadMetadataBlock_NoErrorPath() { assertEquals(3, statistics.size()); assertEquals(1, (int) statistics.get("MetadataBlock")); assertEquals(78, (int) statistics.get("DatasetField")); - assertEquals(242, (int) statistics.get("Controlled Vocabulary")); + assertEquals(243, (int) statistics.get("Controlled Vocabulary")); } @Test From aacff77fbcd210b6b212c34796d518d90f6342c5 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Fri, 9 Sep 2022 15:45:13 +0200 Subject: [PATCH 034/232] nullpointer fix when getting notification with newly created user --- .../iq/dataverse/authorization/AuthenticationServiceBean.java | 2 ++ .../iq/dataverse/authorization/users/AuthenticatedUser.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index b242cd2936f..6c401223cd5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -647,6 +647,8 @@ public AuthenticatedUser createAuthenticatedUser(UserRecordIdentifier userRecord actionLogSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.Auth, "createUser") .setInfo(authenticatedUser.getIdentifier())); + + authenticatedUser.postLoad(); return authenticatedUser; } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index b2b5fa92e76..84a13104d9d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -147,7 +147,7 @@ void prePersist() { } @PostLoad - void postLoad() { + public void postLoad() { mutedNotificationsSet = Type.tokenizeToSet(mutedNotifications); mutedEmailsSet = Type.tokenizeToSet(mutedEmails); } From 22cdaafd9df85267ef758460fd49b3a1d4cb8ef6 Mon Sep 17 00:00:00 2001 From: Bob Treacy Date: Sun, 11 Sep 2022 16:37:07 -0400 Subject: [PATCH 035/232] passes existing query params an signed urls in POST body as json --- .../iq/dataverse/api/AbstractApiBean.java | 4 +- .../edu/harvard/iq/dataverse/api/Users.java | 10 +- .../externaltools/ExternalToolHandler.java | 95 +++++++++++-------- .../iq/dataverse/util/URLTokenUtil.java | 17 ++++ .../iq/dataverse/util/UrlSignerUtil.java | 3 +- 5 files changed, 86 insertions(+), 43 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 7ddde7064fc..bedfac505db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -441,9 +441,9 @@ private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { String user = httpRequest.getParameter("user"); AuthenticatedUser targetUser = authSvc.getAuthenticatedUser(user); String key = System.getProperty(SystemConfig.API_SIGNING_SECRET,"") + authSvc.findApiTokenByUser(targetUser).getTokenString(); - String signedUrl = httpRequest.getRequestURL().toString(); + String signedUrl = httpRequest.getRequestURL().toString()+"?"+httpRequest.getQueryString(); String method = httpRequest.getMethod(); - + String queryString = httpRequest.getQueryString(); boolean validated = UrlSignerUtil.isValidUrl(signedUrl, user, method, key); if (validated){ authUser = targetUser; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index b1177531874..82ab236b92d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import javax.ejb.Stateless; import javax.json.JsonArray; @@ -201,7 +202,14 @@ public Response getAuthenticatedUserByToken() { AuthenticatedUser authenticatedUser = findUserByApiToken(tokenFromRequestAPI); if (authenticatedUser == null) { - return error(Response.Status.BAD_REQUEST, "User with token " + tokenFromRequestAPI + " not found."); + try { + authenticatedUser = findAuthenticatedUserOrDie(); + return ok(json(authenticatedUser)); + } catch (WrappedResponse ex) { + Logger.getLogger(Users.class.getName()).log(Level.SEVERE, null, ex); + return error(Response.Status.BAD_REQUEST, "User with token " + tokenFromRequestAPI + " not found."); + } + } else { return ok(json(authenticatedUser)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 54489953606..59260b82e5e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.URLTokenUtil; import edu.harvard.iq.dataverse.util.UrlSignerUtil; @@ -19,6 +20,7 @@ import java.net.http.HttpResponse; import java.util.ArrayList; import java.util.List; +import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; @@ -90,51 +92,66 @@ public String handleRequest(boolean preview) { JsonString method = obj.getJsonString("httpMethod"); requestMethod = method!=null?method.getString():HttpMethod.GET; JsonArray queryParams = obj.getJsonArray("queryParameters"); - if (queryParams == null || queryParams.isEmpty()) { - return ""; - } List params = new ArrayList<>(); - queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> { - queryParam.keySet().forEach((key) -> { - String value = queryParam.getString(key); - String param = getQueryParam(key, value); - if (param != null && !param.isEmpty()) { - params.add(param); - } + if (requestMethod.equals(HttpMethod.GET)) { + if (queryParams == null || queryParams.isEmpty()) { + return ""; + } + queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> { + queryParam.keySet().forEach((key) -> { + String value = queryParam.getString(key); + String param = getQueryParam(key, value); + if (param != null && !param.isEmpty()) { + params.add(param); + } + }); }); - }); - - String allowedApis; - JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); - - JsonObject objApis = JsonUtil.getJsonObject(externalTool.getAllowedApiCalls()); - - JsonArray apis = objApis.getJsonArray("apis"); - apis.getValuesAs(JsonObject.class).forEach(((apiObj) -> { - String name = apiObj.getJsonString("name").getString(); - String httpmethod = apiObj.getJsonString("method").getString(); - int timeout = apiObj.getInt("timeOut"); - String urlTemplate = apiObj.getJsonString("urlTemplate").getString(); - logger.fine("URL Template: " + urlTemplate); - String apiPath = replaceTokensWithValues(urlTemplate); - logger.fine("URL WithTokens: " + apiPath); - String url = UrlSignerUtil.signUrl(apiPath, timeout, user, httpmethod, getApiToken().getTokenString()); - logger.fine("Signed URL: " + url); - jsonArrayBuilder.add(Json.createObjectBuilder().add("name", name).add("httpMethod", httpmethod) - .add("signedUrl", url).add("timeOut", timeout)); - })); - JsonArray allowedApiCallsArray = jsonArrayBuilder.build(); - allowedApis = JsonUtil.prettyPrint(allowedApiCallsArray); - logger.fine("Sending these signed URLS: " + allowedApis); - + } + //ToDo - if the allowedApiCalls() are defined, could/should we send them to tools using GET as well? if (requestMethod.equals(HttpMethod.POST)) { + JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); try { - return postFormData(allowedApis); + queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> { + queryParam.keySet().forEach((key) -> { + String value = queryParam.getString(key); + String param = getPostBodyParam(key, value); + if (param != null && !param.isEmpty()) { + params.add(param); + } + }); + }); + String addVal = String.join(",", params); + String kvp = "{\"queryParameters\":{" + addVal; + + String allowedApis; + + JsonObject objApis = JsonUtil.getJsonObject(externalTool.getAllowedApiCalls()); + + JsonArray apis = objApis.getJsonArray("apis"); + apis.getValuesAs(JsonObject.class).forEach(((apiObj) -> { + String name = apiObj.getJsonString("name").getString(); + String httpmethod = apiObj.getJsonString("method").getString(); + int timeout = apiObj.getInt("timeOut"); + String urlTemplate = apiObj.getJsonString("urlTemplate").getString(); + logger.fine("URL Template: " + urlTemplate); + String apiPath = replaceTokensWithValues(urlTemplate); + logger.fine("URL WithTokens: " + apiPath); + String url = UrlSignerUtil.signUrl(apiPath, timeout, user, httpmethod, System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); + logger.fine("Signed URL: " + url); + jsonArrayBuilder.add(Json.createObjectBuilder().add("name", name).add("httpMethod", httpmethod) + .add("signedUrl", url).add("timeOut", timeout)); + })); + JsonArray allowedApiCallsArray = jsonArrayBuilder.build(); + allowedApis = "\"signedUrls\":" + JsonUtil.prettyPrint(allowedApiCallsArray) + "}"; + logger.fine("Sending these signed URLS: " + allowedApis); + String body = kvp + "}," + allowedApis; + logger.info(body); + return postFormData(body); } catch (IOException | InterruptedException ex) { - Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); - } + Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); + } } if (!preview) { return "?" + String.join("&", params); @@ -162,7 +179,7 @@ private String postFormData(String allowedApis ) throws IOException, Interrupted } if (redirect==true){ String newUrl = response.headers().firstValue("location").get(); - toolContext = "http://" + response.uri().getAuthority(); +// toolContext = "http://" + response.uri().getAuthority(); url = newUrl; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java index 1a1e92a2802..97dcb50dfea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java @@ -93,6 +93,23 @@ public String getQueryParam(String key, String value) { } } + + public String getPostBodyParam(String key, String value) { + String tokenValue = null; + tokenValue = getTokenValue(value); + if (tokenValue != null) { + try{ + int x =Integer.parseInt(tokenValue); + return "\""+ key + "\"" + ':' + tokenValue; + } catch (NumberFormatException nfe){ + return "\""+ key + "\"" + ':' + "\"" + tokenValue + "\""; + } + + } else { + return null; + } + } + /** * Tries to replace all occurrences of {} with the value for the * corresponding ReservedWord diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java index b11334520e6..85ae4c79190 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -57,6 +57,7 @@ public static String signUrl(String baseUrl, Integer timeout, String user, Strin } signedUrl.append(firstParam ? "?" : "&").append("token="); logger.fine("String to sign: " + signedUrl.toString() + ""); + signedUrl.append(DigestUtils.sha512Hex(signedUrl.toString() + key)); logger.fine("Generated Signed URL: " + signedUrl.toString()); if (logger.isLoggable(Level.FINE)) { @@ -119,7 +120,7 @@ public static boolean isValidUrl(String signedUrl, String user, String method, S // Assuming the token is last - doesn't have to be, but no reason for the URL // params to be rearranged either, and this should only cause false negatives if // it does happen - String urlToHash = signedUrl.substring(0, index + 7); + String urlToHash = signedUrl.substring(0, index + 7).toString(); logger.fine("String to hash: " + urlToHash + ""); String newHash = DigestUtils.sha512Hex(urlToHash + key); logger.fine("Calculated Hash: " + newHash); From 0bd68429d35e75f2c34e8090fa42dbda082423e4 Mon Sep 17 00:00:00 2001 From: cstr Date: Wed, 14 Sep 2022 08:46:04 +0800 Subject: [PATCH 036/232] Update OpenAireExportUtilTest.java --- .../edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java index 8064b8e20f5..40664527cfc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java @@ -609,7 +609,6 @@ public void testWriteRelatedIdentifierElement() throws XMLStreamException, IOExc + "RelatedPublicationIDNumber1" + "" + "RelatedPublicationIDNumber2" - + "" + "" + "RelatedPublicationIDNumber3" + "", From 561d8b7fecf9a1c64266bf17e6b1f0923f783e11 Mon Sep 17 00:00:00 2001 From: cstr Date: Wed, 14 Sep 2022 09:02:17 +0800 Subject: [PATCH 037/232] Update SchemaDotOrgExporterTest.java --- .../harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java index 5f0d4dfd106..d21d24a5432 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java @@ -409,7 +409,7 @@ private static void mockDatasetFieldSvc() { new ControlledVocabularyValue(4l, "doi", publicationIdTypes), new ControlledVocabularyValue(5l, "ean13", publicationIdTypes), new ControlledVocabularyValue(6l, "handle", publicationIdTypes), - new ControlledVocabularyValue(17l, "CSTR", publicationIdTypes) + new ControlledVocabularyValue(17l, "cstr", publicationIdTypes) // Etc. There are more. )); publicationChildTypes.add(datasetFieldTypeSvc.add(publicationIdTypes)); From 39fdcaba2db17e393d388ba97c83682d38ef0f96 Mon Sep 17 00:00:00 2001 From: cstr Date: Wed, 14 Sep 2022 09:12:34 +0800 Subject: [PATCH 038/232] Update ready-state.sql Override --- scripts/issues/2102/ready-state.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/issues/2102/ready-state.sql b/scripts/issues/2102/ready-state.sql index 03ab805531e..96ccf58d865 100644 --- a/scripts/issues/2102/ready-state.sql +++ b/scripts/issues/2102/ready-state.sql @@ -3284,7 +3284,6 @@ COPY controlledvocabalternate (id, strvalue, controlledvocabularyvalue_id, datas 22 United States of America 472 79 23 U.S.A. 472 79 24 YEMEN 483 79 -25 CSTR 825 30 \. @@ -4117,7 +4116,6 @@ COPY controlledvocabularyvalue (id, displayorder, identifier, strvalue, datasetf 822 29 review article 154 823 30 translation 154 824 31 other 154 -825 17 CSTR 30 \. From ffde80e9b09dec1bd0bfcf793a0ed53320edc332 Mon Sep 17 00:00:00 2001 From: cstr Date: Wed, 14 Sep 2022 09:22:23 +0800 Subject: [PATCH 039/232] Update citation.tsv Adjusted sequence number --- scripts/api/data/metadatablocks/citation.tsv | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index a6d54a9a1ad..1b1ff0ae819 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -97,22 +97,22 @@ publicationIDType ark 0 publicationIDType arXiv 1 publicationIDType bibcode 2 - publicationIDType cstr 17 - publicationIDType doi 3 - publicationIDType ean13 4 - publicationIDType eissn 5 - publicationIDType handle 6 - publicationIDType isbn 7 - publicationIDType issn 8 - publicationIDType istc 9 - publicationIDType lissn 10 - publicationIDType lsid 11 - publicationIDType pmid 12 - publicationIDType purl 13 - publicationIDType upc 14 - publicationIDType url 15 - publicationIDType urn 16 - publicationIDType DASH-NRS 17 + publicationIDType cstr 3 + publicationIDType doi 4 + publicationIDType ean13 5 + publicationIDType eissn 6 + publicationIDType handle 7 + publicationIDType isbn 8 + publicationIDType issn 9 + publicationIDType istc 10 + publicationIDType lissn 11 + publicationIDType lsid 12 + publicationIDType pmid 13 + publicationIDType purl 14 + publicationIDType upc 15 + publicationIDType url 16 + publicationIDType urn 17 + publicationIDType DASH-NRS 18 contributorType Data Collector 0 contributorType Data Curator 1 contributorType Data Manager 2 From 1007a6b1f1af2eb59f56e2a2d595139c064f289f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 14 Sep 2022 13:47:52 -0400 Subject: [PATCH 040/232] Update conf/solr/8.11.1/schema.xml Co-authored-by: Philip Durbin --- conf/solr/8.11.1/schema.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/solr/8.11.1/schema.xml b/conf/solr/8.11.1/schema.xml index 78cb0270532..381d72d2756 100644 --- a/conf/solr/8.11.1/schema.xml +++ b/conf/solr/8.11.1/schema.xml @@ -1110,7 +1110,7 @@ --> - + From 7afa2935d72410df2e7f3ad847ceea4037d9244f Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 17:40:25 +0800 Subject: [PATCH 041/232] Update dataset-create-new-all-ddi-fields.json override --- .../dataset-create-new-all-ddi-fields.json | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json index 362a4ae4d90..1b327c15496 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json @@ -321,32 +321,6 @@ "typeClass": "primitive", "value": "http://RelatedPublicationURL2.org" } - }, - { - "publicationCitation": { - "typeName": "publicationCitation", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationCitation3" - }, - "publicationIDType": { - "typeName": "publicationIDType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "CSTR" - }, - "publicationIDNumber": { - "typeName": "publicationIDNumber", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationIDNumber3" - }, - "publicationURL": { - "typeName": "publicationURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://RelatedPublicationURL3.org" - } } ] }, From 627adf7ad565928ef812ef425f32db0eaa07bb34 Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 17:41:01 +0800 Subject: [PATCH 042/232] Update exportfull.xml override --- .../harvard/iq/dataverse/export/ddi/exportfull.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml index 4314775c7a2..0570c832e4f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml @@ -166,10 +166,8 @@ RelatedMaterial1 RelatedMaterial2 - RelatedMaterial3 RelatedDatasets1 RelatedDatasets2 - RelatedDatasets3 @@ -187,19 +185,9 @@ RelatedPublicationCitation2 - - - - - RelatedPublicationIDNumber3 - - RelatedPublicationCitation3 - - OtherReferences1 OtherReferences2 - OtherReferences3 StudyLevelErrorNotes From 09c30a902ff776e721959a72d4d253e597b0730f Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 17:46:15 +0800 Subject: [PATCH 043/232] Update dataset-all-defaults.txt override --- .../dataverse/export/dataset-all-defaults.txt | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt index a2a6d9c0778..7348d54b7dd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt +++ b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt @@ -16,7 +16,7 @@ "createTime": "2015-09-24T16:47:50Z", "license": { "name": "CC0 1.0", - "uri": "http://creativecommons.org/publicdomain/zero/1.0/" + "uri": "https://creativecommons.org/publicdomain/zero/1.0/" }, "metadataBlocks": { "citation": { @@ -362,32 +362,6 @@ "typeClass": "primitive", "value": "http://RelatedPublicationURL2.org" } - }, - { - "publicationCitation": { - "typeName": "publicationCitation", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationCitation3" - }, - "publicationIDType": { - "typeName": "publicationIDType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "CSTR" - }, - "publicationIDNumber": { - "typeName": "publicationIDNumber", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationIDNumber3" - }, - "publicationURL": { - "typeName": "publicationURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://RelatedPublicationURL3.org" - } } ] }, From af57e5dad41a303ec969b9ff9270a2a55ed83d4c Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 18:12:31 +0800 Subject: [PATCH 044/232] Update ddi_dataset.xml override --- .../source/_static/api/ddi_dataset.xml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml index 0c5d5857b54..1e86f911a46 100644 --- a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml +++ b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml @@ -164,10 +164,8 @@ RelatedMaterial1 RelatedMaterial2 - RelatedMaterial3 RelatedDatasets1 RelatedDatasets2 - RelatedDatasets3 @@ -185,19 +183,9 @@ RelatedPublicationCitation2 - - - - - RelatedPublicationIDNumber3 - - RelatedPublicationCitation3 - - - + OtherReferences1 OtherReferences2 - OtherReferences3 StudyLevelErrorNotes From 6b6ab875203964c113e2cec2bebf9bd1c61917bd Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 18:17:03 +0800 Subject: [PATCH 045/232] Update dataset-create-new-all-default-fields.json override --- ...dataset-create-new-all-default-fields.json | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json index e51f4d9e1b5..d7ae8cefbf7 100644 --- a/scripts/api/data/dataset-create-new-all-default-fields.json +++ b/scripts/api/data/dataset-create-new-all-default-fields.json @@ -369,32 +369,6 @@ "typeClass": "primitive", "value": "http://RelatedPublicationURL2.org" } - }, - { - "publicationCitation": { - "typeName": "publicationCitation", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationCitation3" - }, - "publicationIDType": { - "typeName": "publicationIDType", - "multiple": false, - "typeClass": "controlledVocabulary", - "value": "CSTR" - }, - "publicationIDNumber": { - "typeName": "publicationIDNumber", - "multiple": false, - "typeClass": "primitive", - "value": "RelatedPublicationIDNumber3" - }, - "publicationURL": { - "typeName": "publicationURL", - "multiple": false, - "typeClass": "primitive", - "value": "http://RelatedPublicationURL3.org" - } } ] }, From f62474691479f4d1c246c72f53ceb7c2c650350f Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 18:17:54 +0800 Subject: [PATCH 046/232] Update ddi_dataset.xml remove space --- doc/sphinx-guides/source/_static/api/ddi_dataset.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml index 1e86f911a46..05eaadc3458 100644 --- a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml +++ b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml @@ -183,7 +183,7 @@ RelatedPublicationCitation2 - + OtherReferences1 OtherReferences2 From f1b1e3680cf0a4767d1535bff8aaf508a791ae02 Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 18:21:34 +0800 Subject: [PATCH 047/232] Create maven-publish.yml --- .github/workflows/maven-publish.yml | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/maven-publish.yml diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml new file mode 100644 index 00000000000..05c017789db --- /dev/null +++ b/.github/workflows/maven-publish.yml @@ -0,0 +1,54 @@ +name: Maven Unit Tests + +on: + push: + paths: + - "**.java" + - "pom.xml" + - "modules/**/pom.xml" + pull_request: + paths: + - "**.java" + - "pom.xml" + - "modules/**/pom.xml" + +jobs: + unittest: + name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Unit Tests + strategy: + fail-fast: false + matrix: + jdk: [ '11' ] + experimental: [false] + status: ["Stable"] + # + # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within + # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094 + # (This is a limitation of how Github is currently handling these things.) + # + #include: + # - jdk: '17' + # experimental: true + # status: "Experimental" + continue-on-error: ${{ matrix.experimental }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.jdk }} + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.jdk }} + distribution: 'adopt' + - name: Cache Maven packages + uses: actions/cache@v2 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + - name: Build with Maven + run: mvn -DcompilerArgument=-Xlint:unchecked -Dtarget.java.version=${{ matrix.jdk }} -P all-unit-tests clean test + - name: Maven Code Coverage + env: + CI_NAME: github + COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} + run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }} From 8e7509d906cce8e61d3bb8d15f20b35e98d4f8f8 Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 15 Sep 2022 18:32:24 +0800 Subject: [PATCH 048/232] Delete maven-publish.yml --- .github/workflows/maven-publish.yml | 54 ----------------------------- 1 file changed, 54 deletions(-) delete mode 100644 .github/workflows/maven-publish.yml diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml deleted file mode 100644 index 05c017789db..00000000000 --- a/.github/workflows/maven-publish.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Maven Unit Tests - -on: - push: - paths: - - "**.java" - - "pom.xml" - - "modules/**/pom.xml" - pull_request: - paths: - - "**.java" - - "pom.xml" - - "modules/**/pom.xml" - -jobs: - unittest: - name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Unit Tests - strategy: - fail-fast: false - matrix: - jdk: [ '11' ] - experimental: [false] - status: ["Stable"] - # - # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within - # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094 - # (This is a limitation of how Github is currently handling these things.) - # - #include: - # - jdk: '17' - # experimental: true - # status: "Experimental" - continue-on-error: ${{ matrix.experimental }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v2 - with: - java-version: ${{ matrix.jdk }} - distribution: 'adopt' - - name: Cache Maven packages - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Build with Maven - run: mvn -DcompilerArgument=-Xlint:unchecked -Dtarget.java.version=${{ matrix.jdk }} -P all-unit-tests clean test - - name: Maven Code Coverage - env: - CI_NAME: github - COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} - run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }} From b903e2abf5e4527349ec177b063e36f1d216d999 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 15 Sep 2022 19:19:35 -0400 Subject: [PATCH 049/232] release note and addition to search doc --- doc/release-notes/8239-geospatial-indexing.md | 1 + doc/sphinx-guides/source/api/search.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 doc/release-notes/8239-geospatial-indexing.md diff --git a/doc/release-notes/8239-geospatial-indexing.md b/doc/release-notes/8239-geospatial-indexing.md new file mode 100644 index 00000000000..3e6ba0e7a07 --- /dev/null +++ b/doc/release-notes/8239-geospatial-indexing.md @@ -0,0 +1 @@ +Support for indexing the Geographic Bounding Box fields from the Geospatial metadata block has been added. This allows trusted applications with access to solr to perform geospatial queries to find datasets, e.g. those near a given point. This is also a step towards enabling geospatial queries via the Dataverse API and UI. diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index d5e56543fb1..149ad132f79 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -730,3 +730,17 @@ Output from iteration example CORS + + +Geospatial Indexing +------------------- + +Dataverse indexes the Geospatial Bounding Box field from the Geospatial metadatablock as a solr.BBoxField enabling `Spatial Search`_. This capability is not yet exposed through the Dataverse API or UI but can be accessed by trusted applications with direct solr access. +For example, a query of the form + +.. code-block:: none + + q=*.*&fq={!bbox sfield=solr_srpt}=&pt=10,10&d=5 + + +would find datasets with information near the point latitude=10, longitude=10. From c231fb05e933e8a04b8ca9abdee19e723abc4336 Mon Sep 17 00:00:00 2001 From: cstr Date: Mon, 19 Sep 2022 08:45:02 +0800 Subject: [PATCH 050/232] Update OpenAireExportUtilTest.java override --- .../edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java index 40664527cfc..7f7cc203506 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/OpenAireExportUtilTest.java @@ -609,8 +609,6 @@ public void testWriteRelatedIdentifierElement() throws XMLStreamException, IOExc + "RelatedPublicationIDNumber1" + "" + "RelatedPublicationIDNumber2" - + "" - + "RelatedPublicationIDNumber3" + "", stringWriter.toString()); } From 1d47db41e4c7e2c92fe17eb61dff140808042b25 Mon Sep 17 00:00:00 2001 From: cstr Date: Mon, 19 Sep 2022 08:46:33 +0800 Subject: [PATCH 051/232] Update citation.properties --- src/main/java/propertyFiles/citation.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index edb418b0416..b382f8a5a1e 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -251,7 +251,7 @@ controlledvocabulary.subject.social_sciences=Social Sciences controlledvocabulary.subject.other=Other controlledvocabulary.publicationIDType.ark=ark controlledvocabulary.publicationIDType.arxiv=arXiv -controlledvocabulary.publicationIDType.CSTR=CSTR +controlledvocabulary.publicationIDType.cstr=CSTR controlledvocabulary.publicationIDType.bibcode=bibcode controlledvocabulary.publicationIDType.doi=doi controlledvocabulary.publicationIDType.ean13=ean13 @@ -346,7 +346,7 @@ controlledvocabulary.language.galician=Galician controlledvocabulary.language.georgian=Georgian controlledvocabulary.language.german=German controlledvocabulary.language.greek_(modern)=Greek (modern) -controlledvocabulary.language.guarani=Guaraní +controlledvocabulary.language.guarani=Guaraní controlledvocabulary.language.gujarati=Gujarati controlledvocabulary.language.haitian,_haitian_creole=Haitian, Haitian Creole controlledvocabulary.language.hausa=Hausa @@ -406,7 +406,7 @@ controlledvocabulary.language.navajo,_navaho=Navajo, Navaho controlledvocabulary.language.northern_ndebele=Northern Ndebele controlledvocabulary.language.nepali=Nepali controlledvocabulary.language.ndonga=Ndonga -controlledvocabulary.language.norwegian_bokmal=Norwegian Bokmål +controlledvocabulary.language.norwegian_bokmal=Norwegian BokmÃ¥l controlledvocabulary.language.norwegian_nynorsk=Norwegian Nynorsk controlledvocabulary.language.norwegian=Norwegian controlledvocabulary.language.nuosu=Nuosu @@ -468,7 +468,7 @@ controlledvocabulary.language.urdu=Urdu controlledvocabulary.language.uzbek=Uzbek controlledvocabulary.language.venda=Venda controlledvocabulary.language.vietnamese=Vietnamese -controlledvocabulary.language.volapuk=Volapük +controlledvocabulary.language.volapuk=Volapük controlledvocabulary.language.walloon=Walloon controlledvocabulary.language.welsh=Welsh controlledvocabulary.language.wolof=Wolof @@ -478,4 +478,4 @@ controlledvocabulary.language.yiddish=Yiddish controlledvocabulary.language.yoruba=Yoruba controlledvocabulary.language.zhuang,_chuang=Zhuang, Chuang controlledvocabulary.language.zulu=Zulu -controlledvocabulary.language.not_applicable=Not applicable \ No newline at end of file +controlledvocabulary.language.not_applicable=Not applicable From 8093406b5a700582a11cf7d2b016c564757ec479 Mon Sep 17 00:00:00 2001 From: cstr Date: Mon, 19 Sep 2022 08:48:22 +0800 Subject: [PATCH 052/232] Update dataset-all-defaults.txt --- .../edu/harvard/iq/dataverse/export/dataset-all-defaults.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt index 7348d54b7dd..a3f0dffc767 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt +++ b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt @@ -16,7 +16,7 @@ "createTime": "2015-09-24T16:47:50Z", "license": { "name": "CC0 1.0", - "uri": "https://creativecommons.org/publicdomain/zero/1.0/" + "uri": "http://creativecommons.org/publicdomain/zero/1.0/" }, "metadataBlocks": { "citation": { From 744030bd1c07d7ee62ff1d9aecd60f7b256dd57b Mon Sep 17 00:00:00 2001 From: cstr Date: Mon, 19 Sep 2022 09:49:22 +0800 Subject: [PATCH 053/232] Update SchemaDotOrgExporterTest.java --- .../iq/dataverse/export/SchemaDotOrgExporterTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java index d21d24a5432..644848d2776 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java @@ -406,10 +406,10 @@ private static void mockDatasetFieldSvc() { new ControlledVocabularyValue(1l, "ark", publicationIdTypes), new ControlledVocabularyValue(2l, "arXiv", publicationIdTypes), new ControlledVocabularyValue(3l, "bibcode", publicationIdTypes), - new ControlledVocabularyValue(4l, "doi", publicationIdTypes), - new ControlledVocabularyValue(5l, "ean13", publicationIdTypes), - new ControlledVocabularyValue(6l, "handle", publicationIdTypes), - new ControlledVocabularyValue(17l, "cstr", publicationIdTypes) + new ControlledVocabularyValue(4l, "cstr", publicationIdTypes), + new ControlledVocabularyValue(5l, "doi", publicationIdTypes), + new ControlledVocabularyValue(6l, "ean13", publicationIdTypes), + new ControlledVocabularyValue(7l, "handle", publicationIdTypes) // Etc. There are more. )); publicationChildTypes.add(datasetFieldTypeSvc.add(publicationIdTypes)); From c21082167d31c12354cab32544f5d7efeb100255 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 19 Sep 2022 13:48:20 -0400 Subject: [PATCH 054/232] add space --- doc/sphinx-guides/source/api/search.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 149ad132f79..fdebfdb8b10 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -735,7 +735,7 @@ Output from iteration example Geospatial Indexing ------------------- -Dataverse indexes the Geospatial Bounding Box field from the Geospatial metadatablock as a solr.BBoxField enabling `Spatial Search`_. This capability is not yet exposed through the Dataverse API or UI but can be accessed by trusted applications with direct solr access. +Dataverse indexes the Geospatial Bounding Box field from the Geospatial metadatablock as a solr.BBoxField enabling `Spatial Search `_. This capability is not yet exposed through the Dataverse API or UI but can be accessed by trusted applications with direct solr access. For example, a query of the form .. code-block:: none From 8463424a725a8459c3e2c623f7e626000a164933 Mon Sep 17 00:00:00 2001 From: Jan Range Date: Tue, 20 Sep 2022 09:51:53 +0200 Subject: [PATCH 055/232] Added 'multiple' to metadatablock JSON export --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index e088122419d..1b7a52b1ea5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -550,6 +550,7 @@ public static JsonObjectBuilder json(DatasetFieldType fld) { fieldsBld.add("type", fld.getFieldType().toString()); fieldsBld.add("watermark", fld.getWatermark()); fieldsBld.add("description", fld.getDescription()); + fieldsBld.add("multiple", fld.isAllowMultiples()); if (!fld.getChildDatasetFieldTypes().isEmpty()) { JsonObjectBuilder subFieldsBld = jsonObjectBuilder(); for (DatasetFieldType subFld : fld.getChildDatasetFieldTypes()) { From 417ae473ac4a897f0b68e8c351ed8da0eaa962e1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 20 Sep 2022 18:22:01 +0200 Subject: [PATCH 056/232] feat(upload): make upload file storage path configurable #6656 As outlined in IQSS#6656, files will be stored in `domaindir/generated/jsp/dataverse` during upload before being moved to our temporary ingest file space at `$dataverse.files.directory/temp`. With this commit, we enable to configure a different place for these kind of generated temporary files by using MPCONFIG variable substitution inside of glassfish-web.xml. Also sorts the content of glassfish-web.xml into order as specified by the XSD. Documentation of the setting is provided. --- doc/sphinx-guides/source/installation/config.rst | 11 +++++++++++ src/main/webapp/WEB-INF/glassfish-web.xml | 8 +++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 17d88c8ea31..72edaa0b456 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1406,6 +1406,17 @@ dataverse.files.directory This is how you configure the path Dataverse uses for temporary files. (File store specific dataverse.files.\.directory options set the permanent data storage locations.) +dataverse.files.uploads ++++++++++++++++++++++++ + +Configure a folder to store the incoming file stream during uploads (before transfering to `${dataverse.files.directory}/temp`). +You can use an absolute path or a relative, which is relative to the application server domain directory. + +Defaults to ``./uploads``, which resolves to ``/usr/local/payara5/glassfish/domains/domain1/uploads`` in a default +installation. + +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_UPLOADS``. + dataverse.auth.password-reset-timeout-in-minutes ++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/src/main/webapp/WEB-INF/glassfish-web.xml b/src/main/webapp/WEB-INF/glassfish-web.xml index ecd3ba15c40..e56d7013abf 100644 --- a/src/main/webapp/WEB-INF/glassfish-web.xml +++ b/src/main/webapp/WEB-INF/glassfish-web.xml @@ -8,9 +8,15 @@ Keep a copy of the generated servlet class' java code. + + - + + From 43c0681e568856f330f183b4585c2d4535cc2a99 Mon Sep 17 00:00:00 2001 From: cstr Date: Wed, 21 Sep 2022 17:55:37 +0800 Subject: [PATCH 057/232] Update OpenAireExportUtil.java --- .../iq/dataverse/export/openaire/OpenAireExportUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index ffce432ce3b..87604cdc988 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -964,7 +964,7 @@ public static void writeRelatedIdentifierElement(XMLStreamWriter xmlw, DatasetVe relatedIdentifierTypeMap.put("URL".toLowerCase(), "URL"); relatedIdentifierTypeMap.put("URN".toLowerCase(), "URN"); relatedIdentifierTypeMap.put("WOS".toLowerCase(), "WOS"); - relatedIdentifierTypeMap.put("CSTR".toLowerCase(), "CSTR"); + relatedIdentifierTypeMap.put("CSTR".toLowerCase(), "cstr"); } for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { From c715bb88e979f2fa21dd1b27fa9cf7b3108ee60f Mon Sep 17 00:00:00 2001 From: cstr Date: Wed, 21 Sep 2022 18:06:38 +0800 Subject: [PATCH 058/232] Update OpenAireExportUtil.java --- .../harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index 87604cdc988..49fe203b96d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -964,7 +964,6 @@ public static void writeRelatedIdentifierElement(XMLStreamWriter xmlw, DatasetVe relatedIdentifierTypeMap.put("URL".toLowerCase(), "URL"); relatedIdentifierTypeMap.put("URN".toLowerCase(), "URN"); relatedIdentifierTypeMap.put("WOS".toLowerCase(), "WOS"); - relatedIdentifierTypeMap.put("CSTR".toLowerCase(), "cstr"); } for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { From 5c79de8fb1ea1c50455595819864262f8f891e74 Mon Sep 17 00:00:00 2001 From: cstr Date: Fri, 23 Sep 2022 11:29:23 +0800 Subject: [PATCH 059/232] Update AdminIT.java resolve AdminIT junit.framework.AssertionFailedError: expected:<322> but was:<323> --- src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index cf06fd9937b..91f78ca6238 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -762,7 +762,7 @@ public void testLoadMetadataBlock_NoErrorPath() { assertEquals(1, data.size()); List> addedElements = data.get("added"); //Note -test depends on the number of elements in the production citation block, so any changes to the # of elements there can break this test - assertEquals(322, addedElements.size()); + assertEquals(323, addedElements.size()); Map statistics = new HashMap<>(); for (Map unit : addedElements) { From 4bfca4a243ae2795329a8df8ffd3a2f3f1aa36d6 Mon Sep 17 00:00:00 2001 From: cstr Date: Mon, 26 Sep 2022 09:16:25 +0800 Subject: [PATCH 060/232] Update AdminIT.java I should find the reason for the failure, please rebuild --- src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java index 91f78ca6238..bcee8d18e17 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java @@ -778,7 +778,7 @@ public void testLoadMetadataBlock_NoErrorPath() { assertEquals(3, statistics.size()); assertEquals(1, (int) statistics.get("MetadataBlock")); assertEquals(78, (int) statistics.get("DatasetField")); - assertEquals(243, (int) statistics.get("Controlled Vocabulary")); + assertEquals(244, (int) statistics.get("Controlled Vocabulary")); } @Test From b7ea43047fce4491284b12b1ae7403fc248b6f05 Mon Sep 17 00:00:00 2001 From: chenganj Date: Mon, 26 Sep 2022 16:02:16 -0400 Subject: [PATCH 061/232] license internationalization --- .../edu/harvard/iq/dataverse/dataset/DatasetUtil.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 16ea09919af..2db20377169 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -25,12 +25,8 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.logging.Logger; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; import javax.imageio.ImageIO; import org.apache.commons.io.IOUtils; import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO; @@ -577,8 +573,6 @@ public static String getLicenseIcon(DatasetVersion dsv) { public static String getLicenseDescription(DatasetVersion dsv) { License license = DatasetUtil.getLicense(dsv); - return license != null ? license.getShortDescription() : BundleUtil.getStringFromBundle("license.custom.description"); - License license = dsv.getTermsOfUseAndAccess().getLicense(); if (license != null) { return getLocalizedLicenseDescription(license.getName()) ; From 4af2c05a80949974989baf6f5d61348a4d31b3de Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:33:05 -0400 Subject: [PATCH 062/232] support signedURL in findUser --- .../harvard/iq/dataverse/api/AbstractApiBean.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 3b22fb83836..22d1f668949 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -363,7 +363,7 @@ protected AuthenticatedUser findUserByApiToken( String apiKey ) { protected User findUserOrDie() throws WrappedResponse { final String requestApiKey = getRequestApiKey(); final String requestWFKey = getRequestWorkflowInvocationID(); - if (requestApiKey == null && requestWFKey == null) { + if (requestApiKey == null && requestWFKey == null && getRequestParameter("token")==null) { return GuestUser.get(); } PrivateUrlUser privateUrlUser = privateUrlSvc.getPrivateUrlUserFromToken(requestApiKey); @@ -437,19 +437,19 @@ private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { // that as a secret in validation the signedURL. If the signature can't be // validating with their key, the user (or their API key) has been changed and // we reject the request. - //ToDo - add null checks/ verify that calling methods catch things. + // ToDo - add null checks/ verify that calling methods catch things. String user = httpRequest.getParameter("user"); AuthenticatedUser targetUser = authSvc.getAuthenticatedUser(user); - String key = System.getProperty(SystemConfig.API_SIGNING_SECRET,"") + authSvc.findApiTokenByUser(targetUser).getTokenString(); - String signedUrl = httpRequest.getRequestURL().toString()+"?"+httpRequest.getQueryString(); + String key = System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + + authSvc.findApiTokenByUser(targetUser).getTokenString(); + String signedUrl = httpRequest.getRequestURL().toString() + "?" + httpRequest.getQueryString(); String method = httpRequest.getMethod(); - String queryString = httpRequest.getQueryString(); boolean validated = UrlSignerUtil.isValidUrl(signedUrl, user, method, key); - if (validated){ + if (validated) { authUser = targetUser; } return authUser; - } + } protected Dataverse findDataverseOrDie( String dvIdtf ) throws WrappedResponse { Dataverse dv = findDataverse(dvIdtf); From 0cd3e4a5212537b19adf5b040882f59277b5ca2e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:42:55 -0400 Subject: [PATCH 063/232] update/fix requestSignedURL - use the user if supplied - require superuser --- .../edu/harvard/iq/dataverse/api/Admin.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 7a145143306..c9ce12fec98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2249,29 +2249,34 @@ public Response getBannerMessages(@PathParam("id") Long id) throws WrappedRespon @Consumes("application/json") @Path("/requestSignedUrl") public Response getSignedUrl(JsonObject urlInfo) throws WrappedResponse { - AuthenticatedUser superuser = authSvc.getAdminUser(); + AuthenticatedUser superuser = findAuthenticatedUserOrDie(); - if (superuser == null) { + if (superuser == null || !superuser.isSuperuser()) { return error(Response.Status.FORBIDDEN, "Requesting signed URLs is restricted to superusers."); } String userId = urlInfo.getString("user"); String key=null; - if(userId!=null) { - AuthenticatedUser user = authSvc.getAuthenticatedUser(userId); - if(user!=null) { - ApiToken apiToken = authSvc.findApiTokenByUser(user); - if(apiToken!=null && !apiToken.isExpired() && ! apiToken.isDisabled()) { - key = apiToken.getTokenString(); + if (userId != null) { + AuthenticatedUser user = authSvc.getAuthenticatedUser(userId); + // If a user param was sent, we sign the URL for them, otherwise on behalf of + // the superuser who made this api call + if (user != null) { + ApiToken apiToken = authSvc.findApiTokenByUser(user); + if (apiToken != null && !apiToken.isExpired() && !apiToken.isDisabled()) { + key = apiToken.getTokenString(); + } + } else { + userId = superuser.getUserIdentifier(); + // We ~know this exists - the superuser just used it and it was unexpired/not + // disabled. (ToDo - if we want this to work with workflow tokens (or as a + // signed URL), we should do more checking as for the user above)) + key = authSvc.findApiTokenByUser(superuser).getTokenString(); } - } else { - userId=superuser.getIdentifier(); - //We ~know this exists - the superuser just used it and it was unexpired/not disabled. (ToDo - if we want this to work with workflow tokens (or as a signed URL, we should do more checking as for the user above)) - } - key = System.getProperty(SystemConfig.API_SIGNING_SECRET,"") + authSvc.findApiTokenByUser(superuser).getTokenString(); - } - if(key==null) { - return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); + if (key == null) { + return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); + } + key = System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + key; } String baseUrl = urlInfo.getString("url"); From 1095f96253a39c3438dceb29a66be49ad803480d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:43:22 -0400 Subject: [PATCH 064/232] reduce duplication --- .../harvard/iq/dataverse/externaltools/ExternalTool.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index bda9ebad063..97838b45cc5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -118,14 +118,7 @@ public ExternalTool() { } public ExternalTool(String displayName, String toolName, String description, List externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType) { - this.displayName = displayName; - this.toolName = toolName; - this.description = description; - this.externalToolTypes = externalToolTypes; - this.scope = scope; - this.toolUrl = toolUrl; - this.toolParameters = toolParameters; - this.contentType = contentType; + this(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType, null); } public ExternalTool(String displayName, String toolName, String description, List externalToolTypes, Scope scope, String toolUrl, String toolParameters, String contentType, String allowedApiCalls) { From 01fb249eb8dda62da4b7232ffacd0449dd1888d6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:45:32 -0400 Subject: [PATCH 065/232] cleanup, add hasToken call --- .../iq/dataverse/util/UrlSignerUtil.java | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java index b11334520e6..ee3dd127196 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.util; +import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; import java.util.List; @@ -34,7 +35,7 @@ public class UrlSignerUtil { * @return - the signed URL */ public static String signUrl(String baseUrl, Integer timeout, String user, String method, String key) { - StringBuilder signedUrl = new StringBuilder(baseUrl); + StringBuilder signedUrlBuilder = new StringBuilder(baseUrl); boolean firstParam = true; if (baseUrl.contains("?")) { @@ -44,26 +45,26 @@ public static String signUrl(String baseUrl, Integer timeout, String user, Strin LocalDateTime validTime = LocalDateTime.now(); validTime = validTime.plusMinutes(timeout); validTime.toString(); - signedUrl.append(firstParam ? "?" : "&").append("until=").append(validTime); + signedUrlBuilder.append(firstParam ? "?" : "&").append("until=").append(validTime); firstParam = false; } if (user != null) { - signedUrl.append(firstParam ? "?" : "&").append("user=").append(user); + signedUrlBuilder.append(firstParam ? "?" : "&").append("user=").append(user); firstParam = false; } if (method != null) { - signedUrl.append(firstParam ? "?" : "&").append("method=").append(method); + signedUrlBuilder.append(firstParam ? "?" : "&").append("method=").append(method); firstParam=false; } - signedUrl.append(firstParam ? "?" : "&").append("token="); - logger.fine("String to sign: " + signedUrl.toString() + ""); - signedUrl.append(DigestUtils.sha512Hex(signedUrl.toString() + key)); - logger.fine("Generated Signed URL: " + signedUrl.toString()); + signedUrlBuilder.append(firstParam ? "?" : "&").append("token="); + logger.fine("String to sign: " + signedUrlBuilder.toString() + ""); + String signedUrl = signedUrlBuilder.toString(); + signedUrl= signedUrl + (DigestUtils.sha512Hex(signedUrl + key)); if (logger.isLoggable(Level.FINE)) { logger.fine( - "URL signature is " + (isValidUrl(signedUrl.toString(), user, method, key) ? "valid" : "invalid")); + "URL signature is " + (isValidUrl(signedUrl, user, method, key) ? "valid" : "invalid")); } - return signedUrl.toString(); + return signedUrl; } /** @@ -148,4 +149,18 @@ public static boolean isValidUrl(String signedUrl, String user, String method, S return valid; } + public static boolean hasToken(String urlString) { + try { + URL url = new URL(urlString); + List params = URLEncodedUtils.parse(url.getQuery(), Charset.forName("UTF-8")); + for (NameValuePair nvp : params) { + if (nvp.getName().equals("token")) { + return true; + } + } + } catch (MalformedURLException mue) { + logger.fine("Bad url string: " + urlString); + } + return false; + } } From 12d98fad38d1bbec0f523aefc769f4459e83e488 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:46:28 -0400 Subject: [PATCH 066/232] remove ; in jvm option name --- src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index e0d016c6137..13d12ce79ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -125,7 +125,7 @@ public class SystemConfig { // A secret used in signing URLs - individual urls are signed using this and the // intended user's apiKey, creating an aggregate key that is unique to the user // but not known to the user (as their apiKey is) - public final static String API_SIGNING_SECRET = "dataverse.api-signing-secret;"; + public final static String API_SIGNING_SECRET = "dataverse.api-signing-secret"; public String getVersion() { return getVersion(false); From 2287438382c242e50f9e91536eae9e7c534fbf25 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:48:17 -0400 Subject: [PATCH 067/232] remove unnecessary 'apis' json object from manifest structure --- .../externaltools/ExternalToolServiceBean.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java index 432aa26714d..a65ad2427ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBean.java @@ -169,8 +169,8 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { String toolUrl = getRequiredTopLevelField(jsonObject, TOOL_URL); JsonObject toolParametersObj = jsonObject.getJsonObject(TOOL_PARAMETERS); JsonArray queryParams = toolParametersObj.getJsonArray("queryParameters"); - JsonObject allowedApiCallsObj = jsonObject.getJsonObject(ALLOWED_API_CALLS); - JsonArray apis = allowedApiCallsObj.getJsonArray("apis"); + JsonArray allowedApiCallsArray = jsonObject.getJsonArray(ALLOWED_API_CALLS); + boolean allRequiredReservedWordsFound = false; if (scope.equals(Scope.FILE)) { List requiredReservedWordCandidates = new ArrayList<>(); @@ -223,9 +223,11 @@ public static ExternalTool parseAddExternalToolManifest(String manifest) { } String toolParameters = toolParametersObj.toString(); - String allowedApiCalls = allowedApiCallsObj.toString(); + String allowedApiCalls = null; + if(allowedApiCallsArray !=null) { + allowedApiCalls = allowedApiCallsArray.toString(); + } -// return new ExternalTool(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType); return new ExternalTool(displayName, toolName, description, externalToolTypes, scope, toolUrl, toolParameters, contentType, allowedApiCalls); } From 1f82b191a546b9a01f5dffee8773ba6313b06730 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:51:17 -0400 Subject: [PATCH 068/232] refactor, remove extra user variable --- .../iq/dataverse/ConfigureFragmentBean.java | 1 - .../iq/dataverse/api/ExternalTools.java | 1 - .../externaltools/ExternalToolHandler.java | 230 +++++++++++------- .../iq/dataverse/util/URLTokenUtil.java | 23 +- 4 files changed, 143 insertions(+), 112 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java b/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java index 58752af8520..d51a73fd2dc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ConfigureFragmentBean.java @@ -106,7 +106,6 @@ public void generateApiToken() { ApiToken apiToken = new ApiToken(); User user = session.getUser(); if (user instanceof AuthenticatedUser) { - toolHandler.setUser(((AuthenticatedUser) user).getUserIdentifier()); apiToken = authService.findApiTokenByUser((AuthenticatedUser) user); if (apiToken == null) { //No un-expired token diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java b/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java index aef30bfb0c2..e53b54482b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/ExternalTools.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.api; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; -import static edu.harvard.iq.dataverse.api.AbstractApiBean.error; import edu.harvard.iq.dataverse.externaltools.ExternalTool; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import java.util.logging.Logger; diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index be99e78c4d6..085c2a7b3bb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -16,18 +16,24 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.util.ArrayList; -import java.util.List; +import java.util.Map.Entry; import java.util.logging.Level; import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; +import javax.json.JsonNumber; import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; import javax.json.JsonString; +import javax.json.JsonValue; import javax.ws.rs.HttpMethod; +import org.apache.commons.codec.binary.StringUtils; + +import com.github.scribejava.core.java8.Base64; + /** * Handles an operation on a specific file. Requires a file id in order to be * instantiated. Applies logic based on an {@link ExternalTool} specification, @@ -35,29 +41,21 @@ */ public class ExternalToolHandler extends URLTokenUtil { - /** - * @param user the user to set - */ - public void setUser(String user) { - this.user = user; - } - private final ExternalTool externalTool; private String requestMethod; private String toolContext; - private String user; - /** * File level tool * * @param externalTool The database entity. - * @param dataFile Required. - * @param apiToken The apiToken can be null because "explore" tools can be - * used anonymously. + * @param dataFile Required. + * @param apiToken The apiToken can be null because "explore" tools can be + * used anonymously. */ - public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToken apiToken, FileMetadata fileMetadata, String localeCode) { + public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToken apiToken, + FileMetadata fileMetadata, String localeCode) { super(dataFile, apiToken, fileMetadata, localeCode); this.externalTool = externalTool; toolContext = externalTool.getToolUrl(); @@ -67,125 +65,169 @@ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToke * Dataset level tool * * @param externalTool The database entity. - * @param dataset Required. - * @param apiToken The apiToken can be null because "explore" tools can be - * used anonymously. + * @param dataset Required. + * @param apiToken The apiToken can be null because "explore" tools can be + * used anonymously. */ public ExternalToolHandler(ExternalTool externalTool, Dataset dataset, ApiToken apiToken, String localeCode) { super(dataset, apiToken, localeCode); this.externalTool = externalTool; } - // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. + // TODO: rename to handleRequest() to someday handle sending headers as well as + // query parameters. public String handleRequest() { return handleRequest(false); } - - // TODO: rename to handleRequest() to someday handle sending headers as well as query parameters. + + // TODO: rename to handleRequest() to someday handle sending headers as well as + // query parameters. public String handleRequest(boolean preview) { - JsonObject obj = JsonUtil.getJsonObject(externalTool.getToolParameters()); - JsonString method = obj.getJsonString("httpMethod"); - requestMethod = method!=null?method.getString():HttpMethod.GET; - JsonArray queryParams = obj.getJsonArray("queryParameters"); - List params = new ArrayList<>(); + JsonObject toolParameters = JsonUtil.getJsonObject(externalTool.getToolParameters()); + JsonString method = toolParameters.getJsonString("httpMethod"); + requestMethod = method != null ? method.getString() : HttpMethod.GET; + JsonObject params = getParams(toolParameters); + logger.fine("Found params: " + JsonUtil.prettyPrint(params)); if (requestMethod.equals(HttpMethod.GET)) { - if (queryParams == null || queryParams.isEmpty()) { - return ""; + String paramsString = ""; + if (externalTool.getAllowedApiCalls() == null) { + // Legacy, using apiKey + logger.fine("Legacy Case"); + + for (Entry entry : params.entrySet()) { + paramsString = paramsString + (paramsString.isEmpty() ? "?" : "&") + entry.getKey() + "="; + JsonValue val = entry.getValue(); + if (val.getValueType().equals(JsonValue.ValueType.NUMBER)) { + paramsString += ((JsonNumber) val).intValue(); + } else { + paramsString += ((JsonString) val).getString(); + } + } + } else { + //Send a signed callback to get params and signedURLs + String callback = null; + switch (externalTool.getScope()) { + case DATASET: + callback=SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/" + + dataset.getId() + "/versions/:latest/toolparams/" + externalTool.getId(); + case FILE: + callback= SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/files/" + + dataFile.getId() + "/metadata/" + fileMetadata.getId() + "/toolparams/" + + externalTool.getId(); + } + if (apiToken != null) { + callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(), HttpMethod.GET, + System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + apiToken.getTokenString()); + } + paramsString= "?callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback)); + if (getLocaleCode() != null) { + paramsString += "&locale=" + getLocaleCode(); + } + } + if (preview) { + paramsString += "&preview=true"; + } + logger.fine("GET return is: " + paramsString); + return paramsString; + + } else { + // ToDo - if the allowedApiCalls() are defined, could/should we send them to + // tools using GET as well? + + if (requestMethod.equals(HttpMethod.POST)) { + String body = JsonUtil.prettyPrint(createPostBody(params).build()); + try { + logger.info("POST Body: " + body); + return postFormData(body); + } catch (IOException | InterruptedException ex) { + Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); + } } + } + return null; + } + + public JsonObject getParams(JsonObject toolParameters) { + JsonArray queryParams = toolParameters.getJsonArray("queryParameters"); + + // ToDo return json and print later + JsonObjectBuilder paramsBuilder = Json.createObjectBuilder(); + if (!(queryParams == null) && !queryParams.isEmpty()) { queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> { queryParam.keySet().forEach((key) -> { String value = queryParam.getString(key); - String param = getQueryParam(key, value); - if (param != null && !param.isEmpty()) { - params.add(param); + JsonValue param = getParam(value); + if (param != null) { + paramsBuilder.add(key, param); } }); }); } + return paramsBuilder.build(); + } - //ToDo - if the allowedApiCalls() are defined, could/should we send them to tools using GET as well? + public JsonObjectBuilder createPostBody(JsonObject params) { + JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); + + bodyBuilder.add("queryParameters", params); + + JsonArray apiArray = JsonUtil.getJsonArray(externalTool.getAllowedApiCalls()); + JsonArrayBuilder apisBuilder = Json.createArrayBuilder(); - if (requestMethod.equals(HttpMethod.POST)) { - JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); - try { - queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> { - queryParam.keySet().forEach((key) -> { - String value = queryParam.getString(key); - String param = getPostBodyParam(key, value); - if (param != null && !param.isEmpty()) { - params.add(param); - } - }); - }); - String addVal = String.join(",", params); - String kvp = "{\"queryParameters\":{" + addVal; - - String allowedApis; - - JsonObject objApis = JsonUtil.getJsonObject(externalTool.getAllowedApiCalls()); - - JsonArray apis = objApis.getJsonArray("apis"); - apis.getValuesAs(JsonObject.class).forEach(((apiObj) -> { - String name = apiObj.getJsonString("name").getString(); - String httpmethod = apiObj.getJsonString("method").getString(); - int timeout = apiObj.getInt("timeOut"); - String urlTemplate = apiObj.getJsonString("urlTemplate").getString(); - logger.fine("URL Template: " + urlTemplate); - String apiPath = replaceTokensWithValues(urlTemplate); - logger.fine("URL WithTokens: " + apiPath); - String url = UrlSignerUtil.signUrl(apiPath, timeout, user, httpmethod, System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); - logger.fine("Signed URL: " + url); - jsonArrayBuilder.add(Json.createObjectBuilder().add("name", name).add("httpMethod", httpmethod) - .add("signedUrl", url).add("timeOut", timeout)); - })); - JsonArray allowedApiCallsArray = jsonArrayBuilder.build(); - allowedApis = "\"signedUrls\":" + JsonUtil.prettyPrint(allowedApiCallsArray) + "}"; - logger.fine("Sending these signed URLS: " + allowedApis); - String body = kvp + "}," + allowedApis; - logger.info(body); - return postFormData(body); - } catch (IOException | InterruptedException ex) { - Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex); + apiArray.getValuesAs(JsonObject.class).forEach(((apiObj) -> { + logger.info(JsonUtil.prettyPrint(apiObj)); + String name = apiObj.getJsonString("name").getString(); + String httpmethod = apiObj.getJsonString("method").getString(); + int timeout = apiObj.getInt("timeOut"); + String urlTemplate = apiObj.getJsonString("urlTemplate").getString(); + logger.fine("URL Template: " + urlTemplate); + urlTemplate = SystemConfig.getDataverseSiteUrlStatic() + urlTemplate; + String apiPath = replaceTokensWithValues(urlTemplate); + logger.fine("URL WithTokens: " + apiPath); + String url = apiPath; + // Sign if apiToken exists, otherwise send unsigned URL (i.e. for guest users) + ApiToken apiToken = getApiToken(); + logger.info("Fullkey create: " + System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); + if (apiToken != null) { + url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(), httpmethod, + System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); } - } - if (!preview) { - return "?" + String.join("&", params); - } else { - return "?" + String.join("&", params) + "&preview=true"; - } + logger.fine("Signed URL: " + url); + apisBuilder.add(Json.createObjectBuilder().add("name", name).add("httpMethod", httpmethod) + .add("signedUrl", url).add("timeOut", timeout)); + })); + bodyBuilder.add("signedUrls", apisBuilder); + return bodyBuilder; } - - private String postFormData(String allowedApis ) throws IOException, InterruptedException{ + private String postFormData(String allowedApis) throws IOException, InterruptedException { String url = null; HttpClient client = HttpClient.newHttpClient(); - HttpRequest request = HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(allowedApis)).uri(URI.create(externalTool.getToolUrl())) - .header("Content-Type", "application/json") - .build(); + HttpRequest request = HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(allowedApis)) + .uri(URI.create(externalTool.getToolUrl())).header("Content-Type", "application/json").build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - boolean redirect=false; + boolean redirect = false; int status = response.statusCode(); if (status != HttpURLConnection.HTTP_OK) { - if (status == HttpURLConnection.HTTP_MOVED_TEMP - || status == HttpURLConnection.HTTP_MOVED_PERM + if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) { redirect = true; } } - if (redirect==true){ + if (redirect == true) { String newUrl = response.headers().firstValue("location").get(); // toolContext = "http://" + response.uri().getAuthority(); - + url = newUrl; } return url; } - + public String getToolUrlWithQueryParams() { String params = ExternalToolHandler.this.handleRequest(); return toolContext + params; } - + public String getToolUrlForPreviewMode() { return externalTool.getToolUrl() + handleRequest(true); } @@ -199,9 +241,9 @@ public void setApiToken(ApiToken apiToken) { } /** - * @return Returns Javascript that opens the explore tool in a new browser - * tab if the browser allows it.If not, it shows an alert that popups must - * be enabled in the browser. + * @return Returns Javascript that opens the explore tool in a new browser tab + * if the browser allows it.If not, it shows an alert that popups must + * be enabled in the browser. */ public String getExploreScript() { String toolUrl = this.getToolUrlWithQueryParams(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java index 00c93cda1f9..4acf2d544e8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java @@ -5,6 +5,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.json.Json; +import javax.json.JsonValue; + import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.FileMetadata; @@ -95,29 +98,17 @@ public ApiToken getApiToken() { public String getLocaleCode() { return localeCode; } - - public String getQueryParam(String key, String value) { - String tokenValue = null; - tokenValue = getTokenValue(value); - if (tokenValue != null) { - return key + '=' + tokenValue; - } else { - return null; - } - } - - public String getPostBodyParam(String key, String value) { + public JsonValue getParam(String value) { String tokenValue = null; tokenValue = getTokenValue(value); - if (tokenValue != null) { + if (tokenValue != null && !tokenValue.isBlank()) { try{ int x =Integer.parseInt(tokenValue); - return "\""+ key + "\"" + ':' + tokenValue; + return Json.createValue(x); } catch (NumberFormatException nfe){ - return "\""+ key + "\"" + ':' + "\"" + tokenValue + "\""; + return Json.createValue(tokenValue); } - } else { return null; } From a5ca4e29280556094bdde5c961f204f896f13fc8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:51:40 -0400 Subject: [PATCH 069/232] cleanup, note :me works in UI --- src/main/java/edu/harvard/iq/dataverse/api/Users.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 181458bfd6c..7568c7caff6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -201,19 +201,17 @@ public Response getAuthenticatedUserByToken() { String tokenFromRequestAPI = getRequestApiKey(); AuthenticatedUser authenticatedUser = findUserByApiToken(tokenFromRequestAPI); + // This allows use of the :me API call from an active login session. Not sure + // this is a good idea if (authenticatedUser == null) { try { authenticatedUser = findAuthenticatedUserOrDie(); - return ok(json(authenticatedUser)); } catch (WrappedResponse ex) { Logger.getLogger(Users.class.getName()).log(Level.SEVERE, null, ex); return error(Response.Status.BAD_REQUEST, "User with token " + tokenFromRequestAPI + " not found."); } - - } else { - return ok(json(authenticatedUser)); } - + return ok(json(authenticatedUser)); } @POST From a413a13f3a04c2dca2e2c39d80791f1cf47a939a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:52:06 -0400 Subject: [PATCH 070/232] add tool callback methods for dataset and datafile --- .../harvard/iq/dataverse/api/Datasets.java | 38 +++++++++++++++++-- .../edu/harvard/iq/dataverse/api/Files.java | 37 +++++++++++++++++- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index aff543e643c..84f03ed275c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -57,10 +57,11 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetThumbnailCommand; import edu.harvard.iq.dataverse.export.DDIExportServiceBean; import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.externaltools.ExternalTool; +import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; - -import edu.harvard.iq.dataverse.S3PackageImporter; +import edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.dataaccess.DataAccess; @@ -142,7 +143,6 @@ import javax.ws.rs.core.*; import javax.ws.rs.core.Response.Status; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; -import javax.ws.rs.core.UriInfo; import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrServerException; @@ -3581,4 +3581,36 @@ private boolean isSingleVersionArchiving() { } return false; } + + // This method provides a callback for an external tool to retrieve it's + // parameters/api URLs. If the request is authenticated, e.g. by it being + // signed, the api URLs will be signed. If a guest request is made, the URLs + // will be plain/unsigned. + // This supports the cases where a tool is accessing a restricted resource (e.g. + // for a draft dataset), or public case. + @GET + @Path("{id}/versions/{version}/toolparams/{tid}") + public Response getExternalToolDVParams(@PathParam("tid") long externalToolId, + @PathParam("id") String datasetId, @PathParam("version") String version, @QueryParam(value = "locale") String locale) { + try { + DataverseRequest req = createDataverseRequest(findUserOrDie()); + DatasetVersion target = getDatasetVersionOrDie(req, version, findDatasetOrDie(datasetId), null, null); + if (target == null) { + return error(BAD_REQUEST, "DatasetVersion not found."); + } + + ExternalTool externalTool = externalToolService.findById(externalToolId); + ApiToken apiToken = null; + User u = findUserOrDie(); + if (u instanceof AuthenticatedUser) { + apiToken = authSvc.findApiTokenByUser((AuthenticatedUser) u); + } + + + ExternalToolHandler eth = new ExternalToolHandler(externalTool, target.getDataset(), apiToken, locale); + return ok(eth.createPostBody(eth.getParams(JsonUtil.getJsonObject(externalTool.getToolParameters())))); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 9dc0c3be524..1bfa9ee1d7b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -14,6 +14,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.TermsOfUseAndAccessValidator; import edu.harvard.iq.dataverse.UserNotificationServiceBean; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; @@ -31,6 +32,8 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; import edu.harvard.iq.dataverse.export.ExportException; import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.externaltools.ExternalTool; +import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.ingest.IngestRequest; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.ingest.IngestUtil; @@ -40,6 +43,7 @@ import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import java.io.InputStream; import java.io.StringReader; @@ -451,7 +455,8 @@ public Response updateFileMetadata(@FormDataParam("jsonData") String jsonData, @GET @Path("{id}/metadata") public Response getFileMetadata(@PathParam("id") String fileIdOrPersistentId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response, Boolean getDraft) throws WrappedResponse, Exception { - DataverseRequest req; + //ToDo - versionId is not used - can't get metadata for earlier versions + DataverseRequest req; try { req = createDataverseRequest(findUserOrDie()); } catch (Exception e) { @@ -639,4 +644,34 @@ private void exportDatasetMetadata(SettingsServiceBean settingsServiceBean, Data } } + // This method provides a callback for an external tool to retrieve it's + // parameters/api URLs. If the request is authenticated, e.g. by it being + // signed, the api URLs will be signed. If a guest request is made, the URLs + // will be plain/unsigned. + // This supports the cases where a tool is accessing a restricted resource (e.g. + // preview of a draft file), or public case. + @GET + @Path("{id}/metadata/{fmid}/toolparams/{tid}") + public Response getExternalToolFMParams(@PathParam("tid") long externalToolId, + @PathParam("id") long fileId, @PathParam("fmid") long fmid, @QueryParam(value = "locale") String locale) { + try { + ExternalTool externalTool = externalToolService.findById(externalToolId); + ApiToken apiToken = null; + User u = findUserOrDie(); + if (u instanceof AuthenticatedUser) { + apiToken = authSvc.findApiTokenByUser((AuthenticatedUser) u); + } + FileMetadata target = fileSvc.findFileMetadata(fmid); + if (target == null) { + return error(BAD_REQUEST, "FileMetadata not found."); + } + + ExternalToolHandler eth = null; + + eth = new ExternalToolHandler(externalTool, target.getDataFile(), apiToken, target, locale); + return ok(eth.createPostBody(eth.getParams(JsonUtil.getJsonObject(externalTool.getToolParameters())))); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } } From 2ed2414a622fe4e93dc2f05703b3d57eb41ccddb Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 12:52:17 -0400 Subject: [PATCH 071/232] add flyway script --- .../db/migration/V5.12.0.1__7715-signed-urls-for-tools.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/main/resources/db/migration/V5.12.0.1__7715-signed-urls-for-tools.sql diff --git a/src/main/resources/db/migration/V5.12.0.1__7715-signed-urls-for-tools.sql b/src/main/resources/db/migration/V5.12.0.1__7715-signed-urls-for-tools.sql new file mode 100644 index 00000000000..b47529800d3 --- /dev/null +++ b/src/main/resources/db/migration/V5.12.0.1__7715-signed-urls-for-tools.sql @@ -0,0 +1 @@ +ALTER TABLE externaltool ADD COLUMN IF NOT EXISTS allowedapicalls VARCHAR; From 6b9d69f92198aebed388b94c8d5d00bb49b7dc4b Mon Sep 17 00:00:00 2001 From: Robert Treacy Date: Mon, 26 Sep 2022 14:39:33 -0400 Subject: [PATCH 072/232] uses JsonObjectBuilder, elininating some string building that was messy and brittle, Probably still a little string building could be cleaned up --- .../harvard/iq/dataverse/externaltools/ExternalToolHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 085c2a7b3bb..6308b3bed1d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -168,7 +168,6 @@ public JsonObject getParams(JsonObject toolParameters) { public JsonObjectBuilder createPostBody(JsonObject params) { JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); - bodyBuilder.add("queryParameters", params); JsonArray apiArray = JsonUtil.getJsonArray(externalTool.getAllowedApiCalls()); From b098ba6646c3c9a344f6c7a51ac15bd057ef6345 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 13:17:58 -0400 Subject: [PATCH 073/232] use httpMethod in toolParams and allowedapicalls --- .../harvard/iq/dataverse/externaltools/ExternalToolHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 6308b3bed1d..5003a06692c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -176,7 +176,7 @@ public JsonObjectBuilder createPostBody(JsonObject params) { apiArray.getValuesAs(JsonObject.class).forEach(((apiObj) -> { logger.info(JsonUtil.prettyPrint(apiObj)); String name = apiObj.getJsonString("name").getString(); - String httpmethod = apiObj.getJsonString("method").getString(); + String httpmethod = apiObj.getJsonString("httpMethod").getString(); int timeout = apiObj.getInt("timeOut"); String urlTemplate = apiObj.getJsonString("urlTemplate").getString(); logger.fine("URL Template: " + urlTemplate); From 48b2e04285b6e1c8be04865be74b6160851a99a1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 14:06:44 -0400 Subject: [PATCH 074/232] doc updates --- .../admin/dataverse-external-tools.tsv | 2 +- .../external-tools/dynamicDatasetTool.json | 12 ++++++-- .../root/external-tools/fabulousFileTool.json | 16 ++++++++-- .../source/api/external-tools.rst | 29 ++++++++++++++++++- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv index 61db5dfed93..fd1f0f27bc5 100644 --- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv +++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv @@ -1,5 +1,5 @@ Tool Type Scope Description Data Explorer explore file A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse. Whole Tale explore dataset A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_. -File Previewers explore file A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, text, video, tabular data, spreadsheets, and GeoJSON - allowing them to be viewed without downloading. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers +File Previewers explore file A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, text, video, tabular data, spreadsheets, GeoJSON, and ZipFiles - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers Data Curation Tool configure file A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions. diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json index e30c067a86b..47413c8a625 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json @@ -12,8 +12,16 @@ "PID": "{datasetPid}" }, { - "apiToken": "{apiToken}" + "locale":"{localeCode}" } - ] + ], + "allowedApiCalls": [ + { + "name":"retrieveDatasetJson", + "httpMethod":"GET", + "urlTemplate":"/api/v1/datasets/{datasetId}", + "timeOut":10 + } + ] } } diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json index 14f71a280b3..83594929a96 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json @@ -1,6 +1,6 @@ { "displayName": "Fabulous File Tool", - "description": "Fabulous Fun for Files!", + "description": "A non-existent tool that is Fabulous Fun for Files!", "toolName": "fabulous", "scope": "file", "types": [ @@ -9,13 +9,25 @@ ], "toolUrl": "https://fabulousfiletool.com", "contentType": "text/tab-separated-values", + "httpMethod":"GET", "toolParameters": { "queryParameters": [ { "fileid": "{fileId}" }, { - "key": "{apiToken}" + "datasetPid": "{datasetPid}" + }, + { + "locale":"{localeCode}" + } + ], + "allowedApiCalls": [ + { + "name":"retrieveDataFile", + "httpMethod":"GET", + "urlTemplate":"/api/v1/access/datafile/{fileId}", + "timeOut":270 } ] } diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index d72a6f62004..c5b1c43745e 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -92,7 +92,9 @@ Terminology contentType File level tools operate on a specific **file type** (content type or MIME type such as "application/pdf") and this must be specified. Dataset level tools do not use contentType. - toolParameters **Query parameters** are supported and described below. + toolParameters **httpMethod**, **Query parameters**, and **allowedApiCalls** are supported and described below. + + httpMethod Either **GET** or **POST**. queryParameters **Key/value combinations** that can be appended to the toolUrl. For example, once substitution takes place (described below) the user may be redirected to ``https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org``. @@ -102,6 +104,16 @@ Terminology reserved words A **set of strings surrounded by curly braces** such as ``{fileId}`` or ``{datasetId}`` that will be inserted into query parameters. See the table below for a complete list. + allowedApiCalls An array of objects defining callbacks the tool is allowed to make to the Dataverse API. If the dataset or file being accessed is not public, the callback URLs will be signed to allow the tool access for a defined time. + + allowdApiCalls name A name the tool will use to identify this callback URL + + allowedApiCalls urlTemplate The relative URL for the callback using the reserved words to indicate where values should by dynamically substituted + + allowdApiCalls httpMethod Which HTTP method the specified callback uses + + allowedApiCalls timeOut For non-public datasets and datafiles, how long the signed URLs given to the tool should be valid for. + toolName A **name** of an external tool that is used to differentiate between external tools and also used in bundle.properties for localization in the Dataverse installation web interface. For example, the toolName for Data Explorer is ``explorer``. For the Data Curation Tool the toolName is ``dct``. This is an optional parameter in the manifest JSON file. =========================== ========== @@ -131,6 +143,21 @@ Reserved Words ``{localeCode}`` optional The code for the language ("en" for English, "fr" for French, etc.) that user has selected from the language toggle in a Dataverse installation. See also :ref:`i18n`. =========================== ========== =========== +Authorization Options ++++++++++++++++++++++ + +When called for Datasets or DataFiles that are not public, i.e. in a draft dataset or for a restricted file, external tools are allowed access via the user's credentials. This is accomplished by one of two mechanisms: + +* Signed URLs (more secure, recommended) + Configured via the allowedApiCalls section of the manifest. The tool will be provided with signed URLs allowing the specified access to the given dataset or datafile for the specified amount of time. The tool will not be able to access any other datasets or files the user may have access to and will not be able to make calls other than those specified. + For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest. + For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool. + +* ApiToken (deprecated, less secure, not recommended) + Configured via the queryParameters by including an {apiToken} value. When this is present Dataverse will send the user's apiToken to the tool. With the user's apiToken, the tool can perform any action via the Dataverse api that the user could. External tools configured via this method should be assessed for their trustworthiness. + For tools invoked via GET, this will be done via a query parameter in the request URL which could be cached in the browser's history. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool. + For tools invoked via POST, Dataverse will send a JSON body including the apiToken. + Internationalization of Your External Tool ++++++++++++++++++++++++++++++++++++++++++ From 2eb90f18ca579f82ef707c96727f0df70bfc4211 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 14:06:58 -0400 Subject: [PATCH 075/232] use java.util.Base64 --- .../iq/dataverse/externaltools/ExternalToolHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 5003a06692c..26f0f5a7c4b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -16,6 +16,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.util.Base64; import java.util.Map.Entry; import java.util.logging.Level; import java.util.logging.Logger; @@ -32,8 +33,6 @@ import org.apache.commons.codec.binary.StringUtils; -import com.github.scribejava.core.java8.Base64; - /** * Handles an operation on a specific file. Requires a file id in order to be * instantiated. Applies logic based on an {@link ExternalTool} specification, @@ -148,6 +147,7 @@ public String handleRequest(boolean preview) { } public JsonObject getParams(JsonObject toolParameters) { + //ToDo - why an array of object each with a single key/value pair instead of one object? JsonArray queryParams = toolParameters.getJsonArray("queryParameters"); // ToDo return json and print later From 86f910dc276fdc1a464a0edcda8a86e2f5b68b45 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 14:11:00 -0400 Subject: [PATCH 076/232] release note --- doc/release-notes/7715-signed-urls-for-external-tools.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/7715-signed-urls-for-external-tools.md diff --git a/doc/release-notes/7715-signed-urls-for-external-tools.md b/doc/release-notes/7715-signed-urls-for-external-tools.md new file mode 100644 index 00000000000..00b5cff24b3 --- /dev/null +++ b/doc/release-notes/7715-signed-urls-for-external-tools.md @@ -0,0 +1,3 @@ +# Improved Security for External Tools + +This release adds support for configuring external tools to use signed URLs to access the Dataverse API. This eliminates the need for tools to have access to the user's apiToken in order to access draft or restricted datasets and datafiles. \ No newline at end of file From 6a9ab48a42d48a34346eb2c6838dbf1c590c6ee7 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 15:27:53 -0400 Subject: [PATCH 077/232] Tests --- .../ExternalToolHandlerTest.java | 36 ++++++++++++++++ .../ExternalToolServiceBeanTest.java | 43 +++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index 8e70934b4ad..70c835839bb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -6,9 +6,16 @@ import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.util.json.JsonUtil; + import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.List; @@ -198,4 +205,33 @@ public void testGetToolUrlWithOptionalQueryParameters() { } + @Test + public void testGetToolUrlWithallowedApiCalls() { + + System.out.println("allowedApiCalls test"); + Dataset ds = new Dataset(); + ds.setId(1L); + ApiToken at = new ApiToken(); + AuthenticatedUser au = new AuthenticatedUser(); + au.setUserIdentifier("dataverseAdmin"); + at.setAuthenticatedUser(au); + at.setTokenString("1234"); + ExternalTool et = ExternalToolServiceBeanTest.getAllowedApiCallsTool(); + assertTrue(et != null); + System.out.println("allowedApiCalls et created"); + System.out.println(et.getAllowedApiCalls()); + ExternalToolHandler externalToolHandler = new ExternalToolHandler(et, ds, at, null); + System.out.println("allowedApiCalls eth created"); + JsonObject jo = externalToolHandler + .createPostBody(externalToolHandler.getParams(JsonUtil.getJsonObject(et.getToolParameters()))).build(); + assertEquals(1, jo.getJsonObject("queryParameters").getInt("datasetId")); + String signedUrl = jo.getJsonArray("signedUrls").getJsonObject(0).getString("signedUrl"); + // The date and token will change each time but check for the constant parts of + // the response + assertTrue(signedUrl.contains("https://librascholar.org/api/v1/datasets/1")); + assertTrue(signedUrl.contains("&user=dataverseAdmin")); + assertTrue(signedUrl.contains("&method=GET")); + assertTrue(signedUrl.contains("&token=")); + System.out.println(JsonUtil.prettyPrint(jo)); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java index 304898f0fb0..74e10d67352 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java @@ -501,4 +501,47 @@ public void testParseAddToolWithLegacyType() { assertNull(externalTool.getContentType()); } + @Test + public void testParseAddDatasetToolAllowedApiCalls() { + + ExternalTool externalTool = null; + try { + externalTool = getAllowedApiCallsTool(); + } catch (Exception ex) { + System.out.println(ex.getMessage()); + } + assertNotNull(externalTool); + assertNull(externalTool.getContentType()); + } + + protected static ExternalTool getAllowedApiCallsTool() { + JsonObjectBuilder job = Json.createObjectBuilder(); + job.add("displayName", "AwesomeTool"); + job.add("toolName", "explorer"); + job.add("description", "This tool is awesome."); + job.add("types", Json.createArrayBuilder().add("explore")); + job.add("scope", "dataset"); + job.add("toolUrl", "http://awesometool.com"); + job.add("hasPreviewMode", "true"); + + job.add("toolParameters", Json.createObjectBuilder() + .add("httpMethod", "GET") + .add("queryParameters", + Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetId", "{datasetId}") + ) + ) + ).add("allowedApiCalls", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("name", "getDataset") + .add("httpMethod", "GET") + .add("urlTemplate", "/api/v1/datasets/{datasetId}") + .add("timeOut", 10)) + ); + String tool = job.build().toString(); + System.out.println("tool: " + tool); + + return ExternalToolServiceBean.parseAddExternalToolManifest(tool); + } } From 4da5be02ecdf35d7b6e513fd055475afd801c23f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 27 Sep 2022 15:37:18 -0400 Subject: [PATCH 078/232] remove test logging --- .../harvard/iq/dataverse/externaltools/ExternalToolHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 26f0f5a7c4b..7f2087c1e31 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -186,7 +186,6 @@ public JsonObjectBuilder createPostBody(JsonObject params) { String url = apiPath; // Sign if apiToken exists, otherwise send unsigned URL (i.e. for guest users) ApiToken apiToken = getApiToken(); - logger.info("Fullkey create: " + System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); if (apiToken != null) { url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(), httpmethod, System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); From 724f88deca52645b8f17852f8a2b641e0fb81e31 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 28 Sep 2022 09:41:11 -0400 Subject: [PATCH 079/232] remove toolContext - wasn't set for dataset tools, isn't needed --- .../iq/dataverse/externaltools/ExternalToolHandler.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 7f2087c1e31..c9da22081b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -43,7 +43,6 @@ public class ExternalToolHandler extends URLTokenUtil { private final ExternalTool externalTool; private String requestMethod; - private String toolContext; /** * File level tool @@ -57,7 +56,6 @@ public ExternalToolHandler(ExternalTool externalTool, DataFile dataFile, ApiToke FileMetadata fileMetadata, String localeCode) { super(dataFile, apiToken, fileMetadata, localeCode); this.externalTool = externalTool; - toolContext = externalTool.getToolUrl(); } /** @@ -223,7 +221,7 @@ private String postFormData(String allowedApis) throws IOException, InterruptedE public String getToolUrlWithQueryParams() { String params = ExternalToolHandler.this.handleRequest(); - return toolContext + params; + return externalTool.getToolUrl() + params; } public String getToolUrlForPreviewMode() { From 88bc73c14855697d5032a82acf7cd5b1df115330 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 28 Sep 2022 12:14:25 -0400 Subject: [PATCH 080/232] handle unsigned urls --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 22d1f668949..a6e1f4d9ef1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -420,11 +420,11 @@ private AuthenticatedUser findAuthenticatedUserOrDie( String key, String wfid ) } else { throw new WrappedResponse(badWFKey(wfid)); } - } else { + } else if (getRequestParameter("token") != null) { AuthenticatedUser authUser = getAuthenticatedUserFromSignedUrl(); if (authUser != null) { return authUser; - } + } } //Just send info about the apiKey - workflow users will learn about invocationId elsewhere throw new WrappedResponse(badApiKey(null)); From 34585864b847f2ad1954ec64f8cfeeb05e3b2529 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 4 Oct 2022 14:56:36 -0400 Subject: [PATCH 081/232] document release tasks #9019 --- .../source/developers/making-releases.rst | 83 ++++++++++++++++--- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 53fc11a5915..084d0fbba3e 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -10,7 +10,7 @@ Introduction See :doc:`version-control` for background on our branching strategy. -The steps below describe making both normal releases and hotfix releases. +The steps below describe making both regular releases and hotfix releases. Write Release Notes ------------------- @@ -43,49 +43,106 @@ Increment the version number to the milestone (e.g. 5.10.1) in the following two - modules/dataverse-parent/pom.xml -> ```` -> ```` (e.g. `pom.xml commit `_) - doc/sphinx-guides/source/conf.py (two places, e.g. `conf.py commit `_) -Add the version being released to the lists in the following two files: +Add the version being released to the lists in the following file: - doc/sphinx-guides/source/versions.rst (e.g. `versions.rst commit `_) Check in the Changes Above into a Release Branch and Merge It ------------------------------------------------------------- -For any ordinary release, make the changes above in the release branch you created, make a pull request, and merge it into the "develop" branch. Like usual, you can safely delete the branch after the merge is complete. +For a regular release, make the changes above in the release branch you created, make a pull request, and merge it into the "develop" branch. Like usual, you can safely delete the branch after the merge is complete. If you are making a hotfix release, make the pull request against the "master" branch. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later. -Either way, as usual, you should ensure that all tests are passing. Please note that you might need to bump the version in `jenkins.yml `_ in dataverse-ansible to get the tests to run. +Either way, as usual, you should ensure that all tests are passing. Please note that you will need to bump the version in `jenkins.yml `_ in dataverse-ansible to get the tests to pass. Consider doing this before making the pull request. Alternatively, you can bump jenkins.yml after making the pull request and re-run the Jenkins job to make sure tests pass. Merge "develop" into "master" ----------------------------- -Note: If you are making a hotfix release, the "develop" branch is not involved so you can skip this step. +If this is a regular (non-hotfix) release, create a pull request to merge the "develop" branch into the "master" branch using this "compare" link: https://github.com/IQSS/dataverse/compare/master...develop -The "develop" branch should be merged into "master" before tagging. +Once the tests have passed, merge the pull request. + +If this is a hotfix release, skip this step (the "develop" branch is not involved until later). + +Build the Guides for the Release +-------------------------------- + +Go to https://jenkins.dataverse.org/job/guides.dataverse.org/ and make the following adjustments to the config: + +- Repository URL: ``https://github.com/IQSS/dataverse.git`` +- Branch Specifier (blank for 'any'): ``*/master`` +- ``VERSION`` (under "Build Steps"): ``5.10.1`` (for example) + +Click "Save" then "Build Now". + +Make sure the guides directory appears in the expected location such as https://guides.dataverse.org/en/5.10.1/ + +As described below, we'll soon point the "latest" symlink to that new directory. Create a Draft Release on GitHub -------------------------------- -Create a draft release at https://github.com/IQSS/dataverse/releases/new +Go to https://github.com/IQSS/dataverse/releases/new to start creating a draft release. + +- Under "Choose a tag" you will be creating a new tag. Have it start with a "v" such as ``v5.10.1``. Click "Create new tag on publish". +- Under "Target" select ``master`` as the branch. +- Under "Release title" use the same name as the tag such as ``v5.10.1``. +- In the description, copy and paste the content from the release notes .md file created in the "Write Release Notes" steps above. +- Click "Save draft" because we do not want to publish the release yet. + +At this point you can send around the draft release for any final feedback. Links to the guides for this release should be working now, since you build them above. + +Make corrections to the draft, if necessary. It will be out of sync with the .md file, but that's ok (`#7988 `_ is tracking this). -The "tag version" and "title" should be the number of the milestone with a "v" in front (i.e. v5.10.1). +Run a Build to Create the War File +---------------------------------- -Copy in the content from the .md file created in the "Write Release Notes" steps above. +Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the following adjustments to the config: + +- Repository URL: ``https://github.com/IQSS/dataverse.git`` +- Branch Specifier (blank for 'any'): ``*/master`` +- Execute shell: Update version in filenames to ``dataverse-5.10.1.war`` (for example) + +Click "Save" then "Build Now". + +Build Installer (dvinstall.zip) +------------------------------- + +ssh into the dataverse-internal server and do the following: + +- In a git checkout of the dataverse source switch to the master branch and pull the latest. +- Copy the war file from the previous step to the ``target`` directory in the root of the repo (create it, if necessary). +- ``cd scripts/installer`` +- ``make`` + +A zip file called ``dvinstall.zip`` should be produced. Make Artifacts Available for Download ------------------------------------- Upload the following artifacts to the draft release you created: -- war file (``mvn package`` from Jenkins) -- installer (``cd scripts/installer && make``) -- other files as needed, such as updated Solr schema and config files +- the war file (e.g. ``dataverse-5.10.1.war``, from above) +- the installer (``dvinstall.zip``, from above) +- other files as needed: + + - updated Solr schema + - metadata block tsv files + - config files Publish the Release ------------------- Click the "Publish release" button. +Update Guides Link +------------------ + +"latest" at https://guides.dataverse.org/en/latest/ is a symlink to the directory with the latest release. That directory (e.g. ``5.10.1``) was put into place by the Jenkins "guides" job described above. + +ssh into the guides server and update the symlink to point to the latest release. + Close Milestone on GitHub and Create a New One ---------------------------------------------- @@ -115,7 +172,7 @@ For Hotfixes, Merge Hotfix Branch into "develop" and Rename SQL Scripts Note: this only applies to hotfixes! -We've merged the hotfix into the "master" branch but now we need the fixes (and version bump) in the "develop" branch. Make a new branch off the hotfix branch and create a pull request against develop. Merge conflicts are possible and this pull request should go through review and QA like normal. Afterwards it's fine to delete this branch and the hotfix brach that was merged into master. +We've merged the hotfix into the "master" branch but now we need the fixes (and version bump) in the "develop" branch. Make a new branch off the hotfix branch and create a pull request against develop. Merge conflicts are possible and this pull request should go through review and QA like normal. Afterwards it's fine to delete this branch and the hotfix branch that was merged into master. Because of the hotfix version, any SQL scripts in "develop" should be renamed (from "5.11.0" to "5.11.1" for example). To read more about our naming conventions for SQL scripts, see :doc:`sql-upgrade-scripts`. From 53fd76d93bc4c77124ad8f43e0531c263f599999 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 4 Oct 2022 17:41:18 -0400 Subject: [PATCH 082/232] add addition details and corrections #9019 --- .../source/developers/making-releases.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 084d0fbba3e..55f5f550dd9 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -61,9 +61,9 @@ Merge "develop" into "master" If this is a regular (non-hotfix) release, create a pull request to merge the "develop" branch into the "master" branch using this "compare" link: https://github.com/IQSS/dataverse/compare/master...develop -Once the tests have passed, merge the pull request. +Once important tests have passed (compile, unit tests, etc.), merge the pull request. Don't worry about style tests failing such as for shell scripts. -If this is a hotfix release, skip this step (the "develop" branch is not involved until later). +If this is a hotfix release, skip this whole "merge develop to master" step (the "develop" branch is not involved until later). Build the Guides for the Release -------------------------------- @@ -86,7 +86,7 @@ Create a Draft Release on GitHub Go to https://github.com/IQSS/dataverse/releases/new to start creating a draft release. - Under "Choose a tag" you will be creating a new tag. Have it start with a "v" such as ``v5.10.1``. Click "Create new tag on publish". -- Under "Target" select ``master`` as the branch. +- Under "Target" go to "Recent Commits" and select the merge commit from when you merged ``develop`` into ``master`` above. This commit will appear in ``/api/info/version`` from a running installation. - Under "Release title" use the same name as the tag such as ``v5.10.1``. - In the description, copy and paste the content from the release notes .md file created in the "Write Release Notes" steps above. - Click "Save draft" because we do not want to publish the release yet. @@ -98,6 +98,8 @@ Make corrections to the draft, if necessary. It will be out of sync with the .md Run a Build to Create the War File ---------------------------------- +ssh into the dataverse-internal server and undeploy the current war file. + Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the following adjustments to the config: - Repository URL: ``https://github.com/IQSS/dataverse.git`` @@ -106,6 +108,8 @@ Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the fo Click "Save" then "Build Now". +The build number will appear in ``/api/info/version`` (along with the commit mentioned above) from a running installation (e.g. ``{"version":"5.10.1","build":"907-b844672``). + Build Installer (dvinstall.zip) ------------------------------- From 63c175d2079410c29321f2f70976503b1b7e7c0c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 14:47:42 -0400 Subject: [PATCH 083/232] add sleepForReindex call --- .../edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 7579ab265fd..3d81b51df56 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -226,9 +226,8 @@ public void testOaiFunctionality() throws InterruptedException { // created and published: // - however, publish command is executed asynchronously, i.e. it may // still be running after we received the OK from the publish API. - // So let's give it a couple of extra seconds to finish, to make sure - // the dataset is published, exported and indexed - because the OAI - // set create API requires all of the above. + // So let's wait for it to finish. + UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); Thread.sleep(3000L); String setName = identifier; String setQuery = "dsPersistentId:" + identifier; From d3bc18015f7616f124d4f078d9aed67877cbd3b1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 14:48:59 -0400 Subject: [PATCH 084/232] and remove old sleep --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 3d81b51df56..5f0a6cec340 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -228,7 +228,7 @@ public void testOaiFunctionality() throws InterruptedException { // still be running after we received the OK from the publish API. // So let's wait for it to finish. UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); - Thread.sleep(3000L); + String setName = identifier; String setQuery = "dsPersistentId:" + identifier; String apiPath = String.format("/api/harvest/server/oaisets/%s", setName); From ceb3968167de4e78d9b2ed1a48665ffdf8048ae1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 14:56:09 -0400 Subject: [PATCH 085/232] change sleep to be in seconds --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 7107ee783d7..c0a0a18bf63 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2429,16 +2429,18 @@ static Boolean sleepForLock(String idOrPersistentId, String lockType, String api } - static boolean sleepForReindex(String idOrPersistentId, String apiToken, int duration) { + static boolean sleepForReindex(String idOrPersistentId, String apiToken, int durationInSeconds) { int i = 0; Response timestampResponse; + int sleepStep=200; + int repeats = durationInSeconds*1000/sleepStep; do { timestampResponse = UtilIT.getDatasetTimestamps(idOrPersistentId, apiToken); try { - Thread.sleep(200); + Thread.sleep(sleepStep); i++; - if (i > duration) { + if (i > repeats) { break; } } catch (InterruptedException ex) { @@ -2446,7 +2448,7 @@ static boolean sleepForReindex(String idOrPersistentId, String apiToken, int dur } } while (timestampResponse.body().jsonPath().getBoolean("data.hasStaleIndex")); - return i <= duration; + return i <= repeats; } From 8fb50a861dd463cc3fb16d854b70b37ee1b95a35 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 16:52:05 -0400 Subject: [PATCH 086/232] change other use of sleepForReindex --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 529af5f746c..6e42e478863 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3020,7 +3020,7 @@ public void testArchivalStatusAPI() throws IOException { } assertEquals(OK.getStatusCode(), status); - if (!UtilIT.sleepForReindex(datasetPersistentId, apiToken, 3000)) { + if (!UtilIT.sleepForReindex(datasetPersistentId, apiToken, 3)) { logger.info("Still indexing after 3 seconds"); } From a8258ea8b9362eaf3a897de91ba6dd31fe0f6d56 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 14:47:42 -0400 Subject: [PATCH 087/232] add sleepForReindex call --- .../edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 7579ab265fd..3d81b51df56 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -226,9 +226,8 @@ public void testOaiFunctionality() throws InterruptedException { // created and published: // - however, publish command is executed asynchronously, i.e. it may // still be running after we received the OK from the publish API. - // So let's give it a couple of extra seconds to finish, to make sure - // the dataset is published, exported and indexed - because the OAI - // set create API requires all of the above. + // So let's wait for it to finish. + UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); Thread.sleep(3000L); String setName = identifier; String setQuery = "dsPersistentId:" + identifier; From 48d0f4cbbd281af0a5f258504ef03ef7a7596748 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 14:48:59 -0400 Subject: [PATCH 088/232] and remove old sleep --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 3d81b51df56..5f0a6cec340 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -228,7 +228,7 @@ public void testOaiFunctionality() throws InterruptedException { // still be running after we received the OK from the publish API. // So let's wait for it to finish. UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); - Thread.sleep(3000L); + String setName = identifier; String setQuery = "dsPersistentId:" + identifier; String apiPath = String.format("/api/harvest/server/oaisets/%s", setName); From 463b033c6951ecf3a828c515fb469cba4a42b418 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 14:56:09 -0400 Subject: [PATCH 089/232] change sleep to be in seconds --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 7107ee783d7..c0a0a18bf63 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2429,16 +2429,18 @@ static Boolean sleepForLock(String idOrPersistentId, String lockType, String api } - static boolean sleepForReindex(String idOrPersistentId, String apiToken, int duration) { + static boolean sleepForReindex(String idOrPersistentId, String apiToken, int durationInSeconds) { int i = 0; Response timestampResponse; + int sleepStep=200; + int repeats = durationInSeconds*1000/sleepStep; do { timestampResponse = UtilIT.getDatasetTimestamps(idOrPersistentId, apiToken); try { - Thread.sleep(200); + Thread.sleep(sleepStep); i++; - if (i > duration) { + if (i > repeats) { break; } } catch (InterruptedException ex) { @@ -2446,7 +2448,7 @@ static boolean sleepForReindex(String idOrPersistentId, String apiToken, int dur } } while (timestampResponse.body().jsonPath().getBoolean("data.hasStaleIndex")); - return i <= duration; + return i <= repeats; } From 85a3306e6ab3d20aa206a9669926cbc462b5a76c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 16:56:14 -0400 Subject: [PATCH 090/232] update test to use seconds --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 529af5f746c..6e42e478863 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3020,7 +3020,7 @@ public void testArchivalStatusAPI() throws IOException { } assertEquals(OK.getStatusCode(), status); - if (!UtilIT.sleepForReindex(datasetPersistentId, apiToken, 3000)) { + if (!UtilIT.sleepForReindex(datasetPersistentId, apiToken, 3)) { logger.info("Still indexing after 3 seconds"); } From 735f8a121334ffba6693b3153d50ce7c1bf2cdb9 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 17:01:37 -0400 Subject: [PATCH 091/232] cleanup, slower steps, add logging --- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index c0a0a18bf63..29b25a07983 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2432,22 +2432,19 @@ static Boolean sleepForLock(String idOrPersistentId, String lockType, String api static boolean sleepForReindex(String idOrPersistentId, String apiToken, int durationInSeconds) { int i = 0; Response timestampResponse; - int sleepStep=200; - int repeats = durationInSeconds*1000/sleepStep; + int sleepStep=500; + int repeats = durationInSeconds*(1000/sleepStep); do { timestampResponse = UtilIT.getDatasetTimestamps(idOrPersistentId, apiToken); try { Thread.sleep(sleepStep); i++; - if (i > repeats) { - break; - } } catch (InterruptedException ex) { Logger.getLogger(UtilIT.class.getName()).log(Level.SEVERE, null, ex); } - } while (timestampResponse.body().jsonPath().getBoolean("data.hasStaleIndex")); - + } while ((i <= repeats) && timestampResponse.body().jsonPath().getBoolean("data.hasStaleIndex")); + Logger.getLogger(UtilIT.class.getName()).info("Waited " + (i * (sleepStep/1000)) + " seconds"); return i <= repeats; } From 1f0c7e7e1b4ec5404eb26c7c0f73b4aae1698dfc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 18:18:15 -0400 Subject: [PATCH 092/232] more logging, handle Interrupted better --- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 29b25a07983..ce7aefb3820 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2432,19 +2432,24 @@ static Boolean sleepForLock(String idOrPersistentId, String lockType, String api static boolean sleepForReindex(String idOrPersistentId, String apiToken, int durationInSeconds) { int i = 0; Response timestampResponse; - int sleepStep=500; - int repeats = durationInSeconds*(1000/sleepStep); + int sleepStep = 500; + int repeats = durationInSeconds * (1000 / sleepStep); + boolean stale=true; do { timestampResponse = UtilIT.getDatasetTimestamps(idOrPersistentId, apiToken); + String hasStaleIndex = timestampResponse.body().jsonPath().getString("data.hasStaleIndex"); + System.out.println(hasStaleIndex); + stale = Boolean.parseBoolean(hasStaleIndex); try { Thread.sleep(sleepStep); i++; } catch (InterruptedException ex) { Logger.getLogger(UtilIT.class.getName()).log(Level.SEVERE, null, ex); + i = repeats + 1; } - } while ((i <= repeats) && timestampResponse.body().jsonPath().getBoolean("data.hasStaleIndex")); - Logger.getLogger(UtilIT.class.getName()).info("Waited " + (i * (sleepStep/1000)) + " seconds"); + } while ((i <= repeats) && stale); + System.out.println("Waited " + (i * (sleepStep / 1000)) + " seconds"); return i <= repeats; } From 0861ea84dc56725cd1e8082b8ea2d9aae14b05d3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 5 Oct 2022 18:18:27 -0400 Subject: [PATCH 093/232] try longer wait for export --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 5f0a6cec340..290cde7b3e6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -246,7 +246,7 @@ public void testOaiFunctionality() throws InterruptedException { Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); //SEK 09/04/2019 resonable wait time for export OAI? #6128 - Thread.sleep(5000L); + Thread.sleep(10000L); Response getSet = given() .get(apiPath); From 31247d85e59feaf3aaf06d0a62a9d101dcff43ad Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 6 Oct 2022 12:15:31 -0400 Subject: [PATCH 094/232] tweak timing --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 290cde7b3e6..ca5654fa49f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -227,6 +227,7 @@ public void testOaiFunctionality() throws InterruptedException { // - however, publish command is executed asynchronously, i.e. it may // still be running after we received the OK from the publish API. // So let's wait for it to finish. + Thread.sleep(1000L); UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); String setName = identifier; @@ -246,7 +247,7 @@ public void testOaiFunctionality() throws InterruptedException { Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); //SEK 09/04/2019 resonable wait time for export OAI? #6128 - Thread.sleep(10000L); + Thread.sleep(5000L); Response getSet = given() .get(apiPath); From 0b0c3b937e4eeef584a0313a76d01bc51fe40ea3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 6 Oct 2022 14:52:35 -0400 Subject: [PATCH 095/232] add multivalued in schema --- conf/solr/8.11.1/schema.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/solr/8.11.1/schema.xml b/conf/solr/8.11.1/schema.xml index 381d72d2756..063ffa9bd29 100644 --- a/conf/solr/8.11.1/schema.xml +++ b/conf/solr/8.11.1/schema.xml @@ -679,9 +679,9 @@ - + - + From 547b4a410edcb8b3f64290cc3c9a0cf7f7fdb5f8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 6 Oct 2022 15:00:53 -0400 Subject: [PATCH 096/232] more testing of timing and sleepForReindex --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 4 ++-- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index ca5654fa49f..7498c71bfc5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -227,9 +227,9 @@ public void testOaiFunctionality() throws InterruptedException { // - however, publish command is executed asynchronously, i.e. it may // still be running after we received the OK from the publish API. // So let's wait for it to finish. - Thread.sleep(1000L); + Thread.sleep(200L); UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); - + Thread.sleep(5000L); String setName = identifier; String setQuery = "dsPersistentId:" + identifier; String apiPath = String.format("/api/harvest/server/oaisets/%s", setName); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index ce7aefb3820..4ea2cc5f2d2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2437,6 +2437,7 @@ static boolean sleepForReindex(String idOrPersistentId, String apiToken, int dur boolean stale=true; do { timestampResponse = UtilIT.getDatasetTimestamps(idOrPersistentId, apiToken); + System.out.println(timestampResponse.body().asString()); String hasStaleIndex = timestampResponse.body().jsonPath().getString("data.hasStaleIndex"); System.out.println(hasStaleIndex); stale = Boolean.parseBoolean(hasStaleIndex); From cf9b4aee9c5914ff6046fabd188466eb7d3bba1f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 6 Oct 2022 16:19:56 -0400 Subject: [PATCH 097/232] case matters --- conf/solr/8.11.1/schema.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/solr/8.11.1/schema.xml b/conf/solr/8.11.1/schema.xml index 063ffa9bd29..e9fbb35403e 100644 --- a/conf/solr/8.11.1/schema.xml +++ b/conf/solr/8.11.1/schema.xml @@ -679,9 +679,9 @@ - + - + From f542925e24e301b4c9c5336fbfc8f5bbc01c6f79 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 6 Oct 2022 16:43:24 -0400 Subject: [PATCH 098/232] north < south latitude is an error we may want to test for that, but not here. --- doc/sphinx-guides/source/_static/api/ddi_dataset.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml index 05eaadc3458..850e6e72ba2 100644 --- a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml +++ b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml @@ -88,8 +88,8 @@ 10 20 - 30 - 40 + 40 + 30 80 From 9cae388d96a9e794b93cb83375f48a6963721e3d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 7 Oct 2022 09:09:08 -0400 Subject: [PATCH 099/232] remove sleep so error logging shows --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 7498c71bfc5..a9043d49032 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -229,7 +229,7 @@ public void testOaiFunctionality() throws InterruptedException { // So let's wait for it to finish. Thread.sleep(200L); UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); - Thread.sleep(5000L); + String setName = identifier; String setQuery = "dsPersistentId:" + identifier; String apiPath = String.format("/api/harvest/server/oaisets/%s", setName); From 4f9434e9e09689ea8afcbbf7fb36656b7b77e2ae Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 12 Oct 2022 15:34:45 -0400 Subject: [PATCH 100/232] fix another non-physical box --- doc/sphinx-guides/source/_static/api/ddi_dataset.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml index 850e6e72ba2..014ebb8c581 100644 --- a/doc/sphinx-guides/source/_static/api/ddi_dataset.xml +++ b/doc/sphinx-guides/source/_static/api/ddi_dataset.xml @@ -92,8 +92,8 @@ 30 - 80 - 70 + 70 + 80 60 50 From 1db095fa8cbb947beddc1792761ed6bebf099db8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 12 Oct 2022 16:58:48 -0400 Subject: [PATCH 101/232] handle multiples - make bbox a single surrounding box --- conf/solr/8.11.1/schema.xml | 3 --- .../iq/dataverse/search/IndexServiceBean.java | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/conf/solr/8.11.1/schema.xml b/conf/solr/8.11.1/schema.xml index e9fbb35403e..10f1d8f1f4f 100644 --- a/conf/solr/8.11.1/schema.xml +++ b/conf/solr/8.11.1/schema.xml @@ -647,9 +647,6 @@ - - - diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 63412c59b56..766d2a05e6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.io.InputStream; import java.sql.Timestamp; +import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.time.LocalDate; import java.util.ArrayList; @@ -951,6 +952,10 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Float.parseFloat(westLon)) { + minWestLon=westLon; + } + if(maxEastLon==null || Float.parseFloat(maxEastLon) < Float.parseFloat(eastLon)) { + maxEastLon=eastLon; + } + if(minSouthLat==null || Float.parseFloat(minSouthLat) > Float.parseFloat(southLat)) { + minSouthLat=southLat; + } + if(maxNorthLat==null || Float.parseFloat(maxNorthLat) < Float.parseFloat(northLat)) { + maxNorthLat=northLat; + } //W, E, N, S solrInputDocument.addField("solr_srpt", "ENVELOPE(" + westLon + "," + eastLon + "," + northLat + "," + southLat + ")"); } + //Only one bbox per dataset + //W, E, N, S + solrInputDocument.addField("solr_bbox", "ENVELOPE(" + minWestLon + "," + maxEastLon + "," + maxNorthLat + "," + minSouthLat + ")"); + } } } From 202438af792eef1ff5db5835b5ca350eea366c32 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 12 Oct 2022 21:50:21 -0400 Subject: [PATCH 102/232] typo --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 766d2a05e6d..05947ee1224 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1007,7 +1007,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Thu, 13 Oct 2022 13:31:39 -0400 Subject: [PATCH 103/232] wrong scope --- .../edu/harvard/iq/dataverse/search/IndexServiceBean.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 05947ee1224..6c4fb3f1332 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1005,11 +1005,10 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Fri, 14 Oct 2022 11:13:59 +0800 Subject: [PATCH 104/232] encoding change. From de6be210c10c491eeba8fb5dce9d74a3cff06926 Mon Sep 17 00:00:00 2001 From: xflv Date: Fri, 14 Oct 2022 11:27:52 +0800 Subject: [PATCH 105/232] update encoding change. --- src/main/java/propertyFiles/citation.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index b382f8a5a1e..ef8b44d7114 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -346,7 +346,7 @@ controlledvocabulary.language.galician=Galician controlledvocabulary.language.georgian=Georgian controlledvocabulary.language.german=German controlledvocabulary.language.greek_(modern)=Greek (modern) -controlledvocabulary.language.guarani=Guaraní +controlledvocabulary.language.guarani=Guaraní controlledvocabulary.language.gujarati=Gujarati controlledvocabulary.language.haitian,_haitian_creole=Haitian, Haitian Creole controlledvocabulary.language.hausa=Hausa @@ -406,7 +406,7 @@ controlledvocabulary.language.navajo,_navaho=Navajo, Navaho controlledvocabulary.language.northern_ndebele=Northern Ndebele controlledvocabulary.language.nepali=Nepali controlledvocabulary.language.ndonga=Ndonga -controlledvocabulary.language.norwegian_bokmal=Norwegian BokmÃ¥l +controlledvocabulary.language.norwegian_bokmal=Norwegian Bokmål controlledvocabulary.language.norwegian_nynorsk=Norwegian Nynorsk controlledvocabulary.language.norwegian=Norwegian controlledvocabulary.language.nuosu=Nuosu @@ -468,7 +468,7 @@ controlledvocabulary.language.urdu=Urdu controlledvocabulary.language.uzbek=Uzbek controlledvocabulary.language.venda=Venda controlledvocabulary.language.vietnamese=Vietnamese -controlledvocabulary.language.volapuk=Volapük +controlledvocabulary.language.volapuk=Volapük controlledvocabulary.language.walloon=Walloon controlledvocabulary.language.welsh=Welsh controlledvocabulary.language.wolof=Wolof @@ -478,4 +478,4 @@ controlledvocabulary.language.yiddish=Yiddish controlledvocabulary.language.yoruba=Yoruba controlledvocabulary.language.zhuang,_chuang=Zhuang, Chuang controlledvocabulary.language.zulu=Zulu -controlledvocabulary.language.not_applicable=Not applicable +controlledvocabulary.language.not_applicable=Not applicable \ No newline at end of file From c2f9c58e9fdd623a711d652d434de76466600f7e Mon Sep 17 00:00:00 2001 From: chenganj Date: Fri, 14 Oct 2022 10:48:41 -0400 Subject: [PATCH 106/232] license name translation --- .../iq/dataverse/dataset/DatasetUtil.java | 31 +++++++++++++------ .../java/propertyFiles/License.properties | 4 ++- src/main/webapp/dataset-license-terms.xhtml | 6 ++-- .../webapp/datasetLicenseInfoFragment.xhtml | 6 ++-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 2db20377169..31e45aebf18 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -575,27 +575,38 @@ public static String getLicenseDescription(DatasetVersion dsv) { License license = DatasetUtil.getLicense(dsv); if (license != null) { - return getLocalizedLicenseDescription(license.getName()) ; + return getLocalizedLicense(license.getName(),"description") ; } else { return BundleUtil.getStringFromBundle("license.custom.description"); } } - public static String getLocalizedLicenseDescription(String licenseName) { - String key = "license." + licenseName.toLowerCase().replace(" ","_") + ".description"; - if (key != null) { + public static String getLocalizedLicense(String licenseName,String keyPart) { + String key = "license." + licenseName.toLowerCase().replace(" ", "_") + "." + keyPart; + + String second_key = ""; + if (keyPart == "description") + { + second_key = "license.custom.description"; + } + else + { + second_key = "license.custom"; + } + + if (key != null) { try { - String _description = BundleUtil.getStringFromPropertyFile(key, "License"); - if (_description == null) { - return BundleUtil.getStringFromBundle("license.custom.description"); + String propertyValue = BundleUtil.getStringFromPropertyFile(key, "License"); + if (propertyValue == null) { + return BundleUtil.getStringFromBundle(second_key); } else { - return _description; + return propertyValue; } } catch (MissingResourceException mre) { - return BundleUtil.getStringFromBundle("license.custom.description"); + return BundleUtil.getStringFromBundle(second_key); } } else { - return BundleUtil.getStringFromBundle("license.custom.description"); + return BundleUtil.getStringFromBundle(second_key); } } diff --git a/src/main/java/propertyFiles/License.properties b/src/main/java/propertyFiles/License.properties index f6def616a04..2347fed9db6 100644 --- a/src/main/java/propertyFiles/License.properties +++ b/src/main/java/propertyFiles/License.properties @@ -1,2 +1,4 @@ license.cc0_1.0.description=Creative Commons CC0 1.0 Universal Public Domain Dedication. -license.cc_by_4.0.description=Creative Commons Attribution 4.0 International License. \ No newline at end of file +license.cc_by_4.0.description=Creative Commons Attribution 4.0 International License. +license.cc0_1.0.name=CC0 1.0 +license.cc_by_4.0.name=CC-BY 4.0 diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 3669d199283..429dee9b14a 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -46,7 +46,7 @@

+ var="license" itemLabel="#{DatasetUtil:getLocalizedLicense(license.name, 'name')}" itemValue="#{license}"/> @@ -55,8 +55,8 @@

- - #{termsOfUseAndAccess.license.name} + + #{termsOfUseAndAccess.license.name}

diff --git a/src/main/webapp/datasetLicenseInfoFragment.xhtml b/src/main/webapp/datasetLicenseInfoFragment.xhtml index e5d10c745dd..797d20b8a25 100644 --- a/src/main/webapp/datasetLicenseInfoFragment.xhtml +++ b/src/main/webapp/datasetLicenseInfoFragment.xhtml @@ -30,12 +30,12 @@ xmlns:jsf="http://xmlns.jcp.org/jsf">
+ jsf:rendered="#{!empty DatasetUtil:getLocalizedLicense(DatasetPage.workingVersion.termsOfUseAndAccess.license.name,'description')} }">
- +
@@ -121,4 +121,4 @@ xmlns:jsf="http://xmlns.jcp.org/jsf"> - \ No newline at end of file + From 73cd9852e54c3c3a791fced3631d39c3d4aeef7c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 14 Oct 2022 12:35:36 -0400 Subject: [PATCH 107/232] try a wait loop up to 10 seconds --- .../iq/dataverse/api/HarvestingServerIT.java | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index a9043d49032..ed9d46f945c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -20,6 +20,7 @@ import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * extremely minimal API tests for creating OAI sets. @@ -227,7 +228,6 @@ public void testOaiFunctionality() throws InterruptedException { // - however, publish command is executed asynchronously, i.e. it may // still be running after we received the OK from the publish API. // So let's wait for it to finish. - Thread.sleep(200L); UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); String setName = identifier; @@ -247,7 +247,6 @@ public void testOaiFunctionality() throws InterruptedException { Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); //SEK 09/04/2019 resonable wait time for export OAI? #6128 - Thread.sleep(5000L); Response getSet = given() .get(apiPath); @@ -255,21 +254,33 @@ public void testOaiFunctionality() throws InterruptedException { logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); logger.info("getSet printresponse: " + getSet.prettyPrint()); assertEquals(200, getSet.getStatusCode()); - - // Run ListIdentifiers on this newly-created set: - Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); - List ret = listIdentifiersResponse.getBody().xmlPath().getList("OAI-PMH.ListIdentifiers.header"); - - assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); - assertNotNull(ret); - logger.info("setName: " + setName); - logger.info("listIdentifiersResponse.prettyPrint:..... "); - listIdentifiersResponse.prettyPrint(); - // There should be 1 and only 1 record in the response: - assertEquals(1, ret.size()); - // And the record should be the dataset we have just created: - assertEquals(datasetPersistentId, listIdentifiersResponse.getBody().xmlPath().getString("OAI-PMH.ListIdentifiers.header.identifier")); - + int i = 0; + for (i = 1; i < 10; i++) { + Thread.sleep(1000L); + + // Run ListIdentifiers on this newly-created set: + Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); + List ret = listIdentifiersResponse.getBody().xmlPath().getList("OAI-PMH.ListIdentifiers.header"); + + assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + assertNotNull(ret); + logger.info("setName: " + setName); + logger.info("listIdentifiersResponse.prettyPrint:..... "); + listIdentifiersResponse.prettyPrint(); + if (ret.size() != 1) { + i++; + } else { + // There should be 1 and only 1 record in the response: + assertEquals(1, ret.size()); + // And the record should be the dataset we have just created: + assertEquals(datasetPersistentId, listIdentifiersResponse.getBody().xmlPath() + .getString("OAI-PMH.ListIdentifiers.header.identifier")); + break; + } + } + System.out.println("Waited " + i + " seconds for OIA export."); + //Fail if we didn't find the exported record before the timeout + assertTrue(i < 10); Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc"); assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); List listRecords = listRecordsResponse.getBody().xmlPath().getList("OAI-PMH.ListRecords.record"); From 68b921e4069c361a668e9722f586c7d1b01514fd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 14 Oct 2022 13:44:09 -0400 Subject: [PATCH 108/232] change to do while --- .../edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index ed9d46f945c..f88d1d7411f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -255,7 +255,7 @@ public void testOaiFunctionality() throws InterruptedException { logger.info("getSet printresponse: " + getSet.prettyPrint()); assertEquals(200, getSet.getStatusCode()); int i = 0; - for (i = 1; i < 10; i++) { + do { Thread.sleep(1000L); // Run ListIdentifiers on this newly-created set: @@ -277,10 +277,10 @@ public void testOaiFunctionality() throws InterruptedException { .getString("OAI-PMH.ListIdentifiers.header.identifier")); break; } - } + } while (i<15); System.out.println("Waited " + i + " seconds for OIA export."); //Fail if we didn't find the exported record before the timeout - assertTrue(i < 10); + assertTrue(i < 15); Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc"); assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); List listRecords = listRecordsResponse.getBody().xmlPath().getList("OAI-PMH.ListRecords.record"); From 962dde77db3d24c6cdea719b381774469ab5e2db Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 14 Oct 2022 13:48:30 -0400 Subject: [PATCH 109/232] rename sql script post 5.12 release #8671 --- ...-sorting_licenses.sql => V5.12.0.1__8671-sorting_licenses.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.11.1.2__8671-sorting_licenses.sql => V5.12.0.1__8671-sorting_licenses.sql} (100%) diff --git a/src/main/resources/db/migration/V5.11.1.2__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql similarity index 100% rename from src/main/resources/db/migration/V5.11.1.2__8671-sorting_licenses.sql rename to src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql From 9c3b2a75867d55aba1edc6e9ac23c426ba2d64d5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 14 Oct 2022 14:36:02 -0400 Subject: [PATCH 110/232] restore a sleep --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index f88d1d7411f..4709d0452ef 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -247,7 +247,7 @@ public void testOaiFunctionality() throws InterruptedException { Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); //SEK 09/04/2019 resonable wait time for export OAI? #6128 - + Thread.sleep(5000L); Response getSet = given() .get(apiPath); From c7c16d4f9138ed557998ccbf34d17e29a300dfd9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 14 Oct 2022 16:50:21 -0400 Subject: [PATCH 111/232] add geo_point and geo_radius #8239 --- doc/sphinx-guides/source/api/search.rst | 2 ++ .../edu/harvard/iq/dataverse/api/Search.java | 21 ++++++++++++- .../search/SearchIncludeFragment.java | 4 +-- .../dataverse/search/SearchServiceBean.java | 25 +++++++++++++--- .../savedsearch/SavedSearchServiceBean.java | 4 ++- .../iq/dataverse/api/DataversesIT.java | 30 ++++++++++++++++++- 6 files changed, 77 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index fdebfdb8b10..c4e62e05bb7 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -35,6 +35,8 @@ show_relevance boolean Whether or not to show details of which fields were ma show_facets boolean Whether or not to show facets that can be operated on by the "fq" parameter. False by default. See :ref:`advanced search example `. fq string A filter query on the search term. Multiple "fq" parameters can be used. See :ref:`advanced search example `. show_entity_ids boolean Whether or not to show the database IDs of the search results (for developer use). +geo_point string Latitude and longitude in the form ``geo_point=42.3,-71.1``. +geo_radius string Radial distance in kilometers such as ``geo_radius=5``. metadata_fields string Includes the requested fields for each dataset in the response. Multiple "metadata_fields" parameters can be used to include several fields. The value must be in the form "{metadata_block_name}:{field_name}" to include a specific field from a metadata block (see :ref:`example `) or "{metadata_field_set_name}:\*" to include all the fields for a metadata block (see :ref:`example `). "{field_name}" cannot be a subfield of a compound field. If "{field_name}" is a compound field, all subfields are included. =============== ======= =========== diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index 71cb59ff62a..737fc7d1e20 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -72,6 +72,8 @@ public Response search( @QueryParam("show_my_data") boolean showMyData, @QueryParam("query_entities") boolean queryEntities, @QueryParam("metadata_fields") List metadataFields, + @QueryParam("geo_point") String geoPointRequested, + @QueryParam("geo_radius") String geoRadiusRequested, @Context HttpServletResponse response ) { @@ -87,6 +89,8 @@ public Response search( // sanity checking on user-supplied arguments SortBy sortBy; int numResultsPerPage; + String geoPoint; + String geoRadius; List dataverseSubtrees = new ArrayList<>(); try { @@ -119,6 +123,9 @@ public Response search( throw new IOException("Filter is empty, which should never happen, as this allows unfettered searching of our index"); } + geoPoint = getGeoPoint(geoPointRequested); + geoRadius = getGeoRadius(geoRadiusRequested); + } catch (Exception ex) { return error(Response.Status.BAD_REQUEST, ex.getLocalizedMessage()); } @@ -137,7 +144,9 @@ public Response search( paginationStart, dataRelatedToMe, numResultsPerPage, - true //SEK get query entities always for search API additional Dataset Information 6300 12/6/2019 + true, //SEK get query entities always for search API additional Dataset Information 6300 12/6/2019 + geoPoint, + geoRadius ); } catch (SearchException ex) { Throwable cause = ex; @@ -340,4 +349,14 @@ private Dataverse getSubtree(String alias) throws Exception { } } + private String getGeoPoint(String geoPointRequested) { + // TODO add error checking + return geoPointRequested; + } + + private String getGeoRadius(String geoRadiusRequested) { + // TODO add error checking + return geoRadiusRequested; + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java index 9bb83c88add..2b40347828a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java @@ -355,7 +355,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused DataverseRequest dataverseRequest = new DataverseRequest(session.getUser(), httpServletRequest); List dataverses = new ArrayList<>(); dataverses.add(dataverse); - solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false); + solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null); if (solrQueryResponse.hasError()){ logger.info(solrQueryResponse.getError()); setSolrErrorEncountered(true); @@ -363,7 +363,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused // This 2nd search() is for populating the "type" ("dataverse", "dataset", "file") facets: -- L.A. // (why exactly do we need it, again?) // To get the counts we display in the types facets particulary for unselected types - SEK 08/25/2021 - solrQueryResponseAllTypes = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalAllTypes, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false); + solrQueryResponseAllTypes = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalAllTypes, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null); if (solrQueryResponse.hasError()){ logger.info(solrQueryResponse.getError()); setSolrErrorEncountered(true); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index ca158198204..aee0465ddb1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -100,7 +100,7 @@ public class SearchServiceBean { * @throws SearchException */ public SolrQueryResponse search(DataverseRequest dataverseRequest, List dataverses, String query, List filterQueries, String sortField, String sortOrder, int paginationStart, boolean onlyDatatRelatedToMe, int numResultsPerPage) throws SearchException { - return search(dataverseRequest, dataverses, query, filterQueries, sortField, sortOrder, paginationStart, onlyDatatRelatedToMe, numResultsPerPage, true); + return search(dataverseRequest, dataverses, query, filterQueries, sortField, sortOrder, paginationStart, onlyDatatRelatedToMe, numResultsPerPage, true, null, null); } /** @@ -121,10 +121,24 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, List dataverses, String query, List filterQueries, String sortField, String sortOrder, int paginationStart, boolean onlyDatatRelatedToMe, int numResultsPerPage, boolean retrieveEntities) throws SearchException { + public SolrQueryResponse search( + DataverseRequest dataverseRequest, + List dataverses, + String query, + List filterQueries, + String sortField, String sortOrder, + int paginationStart, + boolean onlyDatatRelatedToMe, + int numResultsPerPage, + boolean retrieveEntities, + String geoPoint, + String geoRadius + ) throws SearchException { if (paginationStart < 0) { throw new IllegalArgumentException("paginationStart must be 0 or greater"); @@ -204,8 +218,11 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, List Date: Tue, 18 Oct 2022 11:03:17 -0400 Subject: [PATCH 112/232] double sleep --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 4709d0452ef..71ee313b0c5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -247,7 +247,7 @@ public void testOaiFunctionality() throws InterruptedException { Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); //SEK 09/04/2019 resonable wait time for export OAI? #6128 - Thread.sleep(5000L); + Thread.sleep(10000L); Response getSet = given() .get(apiPath); From 73c3362e363cc5d706d25bbb2f249e1c912e0bd6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 18 Oct 2022 19:36:26 -0400 Subject: [PATCH 113/232] add waitForRexport test --- .../iq/dataverse/api/HarvestingServerIT.java | 4 +-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 71ee313b0c5..7056cf31f59 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -228,7 +228,7 @@ public void testOaiFunctionality() throws InterruptedException { // - however, publish command is executed asynchronously, i.e. it may // still be running after we received the OK from the publish API. // So let's wait for it to finish. - UtilIT.sleepForReindex(datasetPersistentId, adminUserAPIKey, 10); + UtilIT.sleepForReexport(datasetPersistentId, adminUserAPIKey, 10); String setName = identifier; String setQuery = "dsPersistentId:" + identifier; @@ -247,7 +247,7 @@ public void testOaiFunctionality() throws InterruptedException { Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); //SEK 09/04/2019 resonable wait time for export OAI? #6128 - Thread.sleep(10000L); + Thread.sleep(5000L); Response getSet = given() .get(apiPath); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 4ea2cc5f2d2..425156c1652 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -17,6 +17,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.time.LocalDateTime; import java.util.logging.Level; import edu.harvard.iq.dataverse.api.datadeposit.SwordConfigurationImpl; import com.jayway.restassured.path.xml.XmlPath; @@ -2454,6 +2455,38 @@ static boolean sleepForReindex(String idOrPersistentId, String apiToken, int dur return i <= repeats; } + static boolean sleepForReexport(String idOrPersistentId, String apiToken, int durationInSeconds) { + int i = 0; + Response timestampResponse; + int sleepStep = 500; + int repeats = durationInSeconds * (1000 / sleepStep); + boolean staleExport=true; + do { + timestampResponse = UtilIT.getDatasetTimestamps(idOrPersistentId, apiToken); + System.out.println(timestampResponse.body().asString()); + String updateTimeString = timestampResponse.body().jsonPath().getString("data.lastUpdateTime"); + String exportTimeString = timestampResponse.body().jsonPath().getString("data.lastMetadataExportTime"); + if (updateTimeString != null && exportTimeString != null) { + LocalDateTime updateTime = LocalDateTime.parse(updateTimeString); + LocalDateTime exportTime = LocalDateTime.parse(exportTimeString); + if (exportTime.isAfter(updateTime)) { + staleExport = false; + } + } + try { + Thread.sleep(sleepStep); + i++; + } catch (InterruptedException ex) { + Logger.getLogger(UtilIT.class.getName()).log(Level.SEVERE, null, ex); + i = repeats + 1; + } + } while ((i <= repeats) && staleExport); + System.out.println("Waited " + (i * (sleepStep / 1000)) + " seconds for export"); + return i <= repeats; + + } + + //Helper function that returns true if a given search returns a non-zero response within a fixed time limit From d08841df22d0c593500cae802caff80df40da423 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 19 Oct 2022 06:51:33 -0400 Subject: [PATCH 114/232] cleanup --- .../iq/dataverse/api/HarvestingServerIT.java | 20 +++++++++++-------- .../edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 7056cf31f59..17ad077c39e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import java.util.logging.Level; import java.util.logging.Logger; import com.jayway.restassured.RestAssured; import static com.jayway.restassured.RestAssured.given; @@ -227,7 +228,9 @@ public void testOaiFunctionality() throws InterruptedException { // created and published: // - however, publish command is executed asynchronously, i.e. it may // still be running after we received the OK from the publish API. - // So let's wait for it to finish. + // The oaiExport step also requires the metadata exports to be done and this + // takes longer than just publish/reindex. + // So wait for all of this to finish. UtilIT.sleepForReexport(datasetPersistentId, adminUserAPIKey, 10); String setName = identifier; @@ -246,15 +249,14 @@ public void testOaiFunctionality() throws InterruptedException { // (this is asynchronous - so we should probably wait a little) Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); - //SEK 09/04/2019 resonable wait time for export OAI? #6128 - Thread.sleep(5000L); Response getSet = given() .get(apiPath); logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); - logger.info("getSet printresponse: " + getSet.prettyPrint()); + logger.fine("getSet printresponse: " + getSet.prettyPrint()); assertEquals(200, getSet.getStatusCode()); int i = 0; + int maxWait=10; do { Thread.sleep(1000L); @@ -265,8 +267,10 @@ public void testOaiFunctionality() throws InterruptedException { assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); assertNotNull(ret); logger.info("setName: " + setName); - logger.info("listIdentifiersResponse.prettyPrint:..... "); - listIdentifiersResponse.prettyPrint(); + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint:..... "); + listIdentifiersResponse.prettyPrint(); + } if (ret.size() != 1) { i++; } else { @@ -277,10 +281,10 @@ public void testOaiFunctionality() throws InterruptedException { .getString("OAI-PMH.ListIdentifiers.header.identifier")); break; } - } while (i<15); + } while (i Date: Wed, 19 Oct 2022 08:56:55 -0400 Subject: [PATCH 115/232] report fractional second waits --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index a919b222f58..1d0398900fb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2451,7 +2451,7 @@ static boolean sleepForReindex(String idOrPersistentId, String apiToken, int dur i = repeats + 1; } } while ((i <= repeats) && stale); - System.out.println("Waited " + (i * (sleepStep / 1000)) + " seconds"); + System.out.println("Waited " + (i * (sleepStep / 1000.0)) + " seconds"); return i <= repeats; } @@ -2481,7 +2481,7 @@ static boolean sleepForReexport(String idOrPersistentId, String apiToken, int du i = repeats + 1; } } while ((i <= repeats) && staleExport); - System.out.println("Waited " + (i * (sleepStep / 1000)) + " seconds for export"); + System.out.println("Waited " + (i * (sleepStep / 1000.0)) + " seconds for export"); return i <= repeats; } From 4927ac3eaea41a1d00bfe61833f58d48892c0cc3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 19 Oct 2022 09:53:07 -0400 Subject: [PATCH 116/232] only wait if needed --- .../java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 17ad077c39e..fdd034ab12e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -258,7 +258,7 @@ public void testOaiFunctionality() throws InterruptedException { int i = 0; int maxWait=10; do { - Thread.sleep(1000L); + // Run ListIdentifiers on this newly-created set: Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); @@ -281,6 +281,7 @@ public void testOaiFunctionality() throws InterruptedException { .getString("OAI-PMH.ListIdentifiers.header.identifier")); break; } + Thread.sleep(1000L); } while (i Date: Fri, 21 Oct 2022 13:09:01 +0200 Subject: [PATCH 117/232] Added controlled vocab bool and values to JSON --- .../edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 1b7a52b1ea5..a8c3013c2ef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -551,6 +551,14 @@ public static JsonObjectBuilder json(DatasetFieldType fld) { fieldsBld.add("watermark", fld.getWatermark()); fieldsBld.add("description", fld.getDescription()); fieldsBld.add("multiple", fld.isAllowMultiples()); + fieldsBld.add("isControlledVocabulary", fld.isControlledVocabulary()); + if (fld.isControlledVocabulary()) { + // If the field has a controlled vocabulary, + // add all values to the resulting JSON + fieldsBld.add( + "controlledVocabularyValues", + fld.getControlledVocabularyValues().toArray().toString()); + } if (!fld.getChildDatasetFieldTypes().isEmpty()) { JsonObjectBuilder subFieldsBld = jsonObjectBuilder(); for (DatasetFieldType subFld : fld.getChildDatasetFieldTypes()) { From 3d647f43aa2021121c5a45b94ec4697cfcc0b6e5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 24 Oct 2022 20:37:54 -0400 Subject: [PATCH 118/232] move hard coded strings to SearchFields class #8239 --- .../edu/harvard/iq/dataverse/search/IndexServiceBean.java | 4 ++-- .../java/edu/harvard/iq/dataverse/search/SearchFields.java | 5 +++++ .../edu/harvard/iq/dataverse/search/SearchServiceBean.java | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 6c4fb3f1332..8bd3f7f443d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1003,12 +1003,12 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Mon, 24 Oct 2022 22:03:38 -0400 Subject: [PATCH 119/232] add geospatial search test #8239 --- .../harvard/iq/dataverse/api/SearchIT.java | 154 +++++++++++++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 22 +++ 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 789b60a34e7..0f2c77de717 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -36,6 +36,7 @@ import org.junit.After; import static org.junit.Assert.assertNotEquals; import static java.lang.Thread.sleep; +import javax.json.JsonObjectBuilder; public class SearchIT { @@ -1084,7 +1085,158 @@ public void testSubtreePermissions() { .statusCode(OK.getStatusCode()) .body("data.total_count", CoreMatchers.equalTo(1)); } - + + @Test + public void testGeospatialSearch() { + + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response setMetadataBlocks = UtilIT.setMetadataBlocks(dataverseAlias, Json.createArrayBuilder().add("citation").add("geospatial"), apiToken); + setMetadataBlocks.prettyPrint(); + setMetadataBlocks.then().assertThat().statusCode(OK.getStatusCode()); + + JsonObjectBuilder datasetJson = Json.createObjectBuilder() + .add("datasetVersion", Json.createObjectBuilder() + .add("metadataBlocks", Json.createObjectBuilder() + .add("citation", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "title") + .add("value", "Dataverse HQ") + .add("typeClass", "primitive") + .add("multiple", false) + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "Simpson, Homer") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorName")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "author") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetContactEmail", + Json.createObjectBuilder() + .add("value", "hsimpson@mailinator.com") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "datasetContactEmail")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "datasetContact") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dsDescriptionValue", + Json.createObjectBuilder() + .add("value", "Headquarters for Dataverse.") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "dsDescriptionValue")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "dsDescription") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add("Other") + ) + .add("typeClass", "controlledVocabulary") + .add("multiple", true) + .add("typeName", "subject") + ) + ) + ) + .add("geospatial", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "geographicBoundingBox") + .add("typeClass", "compound") + .add("multiple", true) + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + // The box is roughly on Cambridge, MA + // See https://linestrings.com/bbox/#-71.187346,42.33661,-71.043056,42.409599 + .add("westLongitude", + Json.createObjectBuilder() + .add("value", "-71.187346") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "westLongitude") + ) + .add("southLongitude", + Json.createObjectBuilder() + .add("value", "42.33661") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "southLongitude") + ) + .add("eastLongitude", + Json.createObjectBuilder() + .add("value", "-71.043056") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "eastLongitude") + ) + .add("northLongitude", + Json.createObjectBuilder() + .add("value", "42.409599") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "northLongitude") + ) + ) + ) + ) + ) + ) + )); + + Response createDatasetResponse = UtilIT.createDataset(dataverseAlias, datasetJson, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + + // Plymouth rock (41.9580775,-70.6621063) is within 50 km of Cambridge. Hit. + Response search1 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken, "&show_entity_ids=true&geo_point=41.9580775,-70.6621063&geo_radius=50"); + search1.prettyPrint(); + search1.then().assertThat() + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].entity_id", CoreMatchers.is(datasetId)) + .statusCode(OK.getStatusCode()); + + // Plymouth rock (41.9580775,-70.6621063) is not within 1 km of Cambridge. Miss. + Response search2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken, "&geo_point=41.9580775,-70.6621063&geo_radius=1"); + search2.prettyPrint(); + search2.then().assertThat() + .body("data.total_count", CoreMatchers.is(0)) + .body("data.count_in_response", CoreMatchers.is(0)) + .statusCode(OK.getStatusCode()); + + } + @After public void tearDownDataverse() { File treesThumb = new File("scripts/search/data/binary/trees.png.thumb48"); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 7107ee783d7..3bffdaf5188 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -407,6 +407,20 @@ static Response createDatasetViaNativeApi(String dataverseAlias, String pathToJs return createDatasetResponse; } + static Response createDataset(String dataverseAlias, JsonObjectBuilder datasetJson, String apiToken) { + return createDataset(dataverseAlias, datasetJson.build().toString(), apiToken); + } + + static Response createDataset(String dataverseAlias, String datasetJson, String apiToken) { + System.out.println("creating with " + datasetJson); + Response createDatasetResponse = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(datasetJson) + .contentType("application/json") + .post("/api/dataverses/" + dataverseAlias + "/datasets"); + return createDatasetResponse; + } + static String getDatasetJson(String pathToJsonFile) { File datasetVersionJson = new File(pathToJsonFile); try { @@ -544,6 +558,14 @@ static Response loadMetadataBlock(String apiToken, byte[] body) { .post("/api/admin/datasetfield/load"); } + static Response setMetadataBlocks(String dataverseAlias, JsonArrayBuilder blocks, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/json") + .body(blocks.build().toString()) + .post("/api/dataverses/" + dataverseAlias + "/metadatablocks"); + } + static private String getDatasetXml(String title, String author, String description) { String nullLicense = null; String nullRights = null; From eca4c2d57ac92a07c3be19ea2676f59b776952c8 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 25 Oct 2022 17:38:34 +0200 Subject: [PATCH 120/232] fix for the test with license sorting --- src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java index e189336b61e..50d3c5b34ea 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java @@ -91,7 +91,8 @@ public void testLicenses(){ getLicensesResponse.prettyPrint(); body = getLicensesResponse.getBody().asString(); status = JsonPath.from(body).getString("status"); - long licenseId = JsonPath.from(body).getLong("data[-1].id"); + //Last added licens; with the highest id + long licenseId = JsonPath.from(body).getList("data[*].id").stream().max((x, y) -> Long.compare(x, y)).get(); //Assumes the first license is active, which should be true on a test server long activeLicenseId = JsonPath.from(body).getLong("data[0].id"); assertEquals("OK", status); From e5187b2af4e1915f5eff4575c6b2ccadd62d150a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 25 Oct 2022 14:28:40 -0400 Subject: [PATCH 121/232] Avoid DatasetCreate exception with only one coordinate #8239 With only westLongitude added and the other three coordinates left empty, we were getting the following exception. A null check was added to prevent this. Command [DatasetCreate dataset:132] failed: Exception thrown from bean: javax.ejb.EJBTransactionRolledbackException: Exception thrown from bean: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://localhost:8983/solr/collection1: ERROR: [doc=dataset_132_draft] Error adding field 'solr_bboxtype'='ENVELOPE(null,null,null,null)' msg=Unable to parse shape given formats "lat,lon", "x y" or as WKT because java.text.ParseException: Expected a number input: ENVELOPE(null,null,null,null) --- .../edu/harvard/iq/dataverse/search/IndexServiceBean.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 8bd3f7f443d..f5a5570c831 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1008,7 +1008,10 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Tue, 25 Oct 2022 16:14:14 -0400 Subject: [PATCH 122/232] rename solr_srpt to geolocation and solr_bboxtype to boundingBox #8239 --- conf/solr/8.11.1/schema.xml | 12 +++++++----- doc/sphinx-guides/source/api/search.rst | 2 +- .../iq/dataverse/search/IndexServiceBean.java | 4 ++-- .../harvard/iq/dataverse/search/SearchFields.java | 6 +++--- .../iq/dataverse/search/SearchServiceBean.java | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/conf/solr/8.11.1/schema.xml b/conf/solr/8.11.1/schema.xml index 10f1d8f1f4f..655cf1bc3cc 100644 --- a/conf/solr/8.11.1/schema.xml +++ b/conf/solr/8.11.1/schema.xml @@ -228,6 +228,11 @@ + + + + + - - + @@ -1107,7 +1109,7 @@ --> - + diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index c4e62e05bb7..c2311ead089 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -742,7 +742,7 @@ For example, a query of the form .. code-block:: none - q=*.*&fq={!bbox sfield=solr_srpt}=&pt=10,10&d=5 + q=*.*&fq={!bbox sfield=geolocation}=&pt=10,10&d=5 would find datasets with information near the point latitude=10, longitude=10. diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index f5a5570c831..4661e9c1cd5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1003,13 +1003,13 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set Date: Wed, 26 Oct 2022 15:41:41 -0400 Subject: [PATCH 123/232] add error checking for geo_point and geo_radius #8239 --- .../edu/harvard/iq/dataverse/api/Search.java | 14 ++++-- .../iq/dataverse/search/SearchUtil.java | 46 ++++++++++++++++++- .../harvard/iq/dataverse/api/SearchIT.java | 38 +++++++++++++-- .../iq/dataverse/search/SearchUtilTest.java | 38 +++++++++++++++ 4 files changed, 127 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index 737fc7d1e20..cef509b1ec5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -126,6 +126,14 @@ public Response search( geoPoint = getGeoPoint(geoPointRequested); geoRadius = getGeoRadius(geoRadiusRequested); + if (geoPoint != null && geoRadius == null) { + return error(Response.Status.BAD_REQUEST, "If you supply geo_point you must also supply geo_radius."); + } + + if (geoRadius != null && geoPoint == null) { + return error(Response.Status.BAD_REQUEST, "If you supply geo_radius you must also supply geo_point."); + } + } catch (Exception ex) { return error(Response.Status.BAD_REQUEST, ex.getLocalizedMessage()); } @@ -350,13 +358,11 @@ private Dataverse getSubtree(String alias) throws Exception { } private String getGeoPoint(String geoPointRequested) { - // TODO add error checking - return geoPointRequested; + return SearchUtil.getGeoPoint(geoPointRequested); } private String getGeoRadius(String geoRadiusRequested) { - // TODO add error checking - return geoRadiusRequested; + return SearchUtil.getGeoRadius(geoRadiusRequested); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java index c226d77f885..8a1045a842c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java @@ -181,5 +181,49 @@ public static String constructQuery(List queryStrings, boolean isAnd, bo return queryBuilder.toString().trim(); } - + + /** + * @return Null if supplied point is null or whitespace. + * @throws IllegalArgumentException If the lat/long is not separated by a + * comma. + * @throws NumberFormatException If the lat/long values are not numbers. + */ + public static String getGeoPoint(String userSuppliedGeoPoint) throws IllegalArgumentException, NumberFormatException { + if (userSuppliedGeoPoint == null || userSuppliedGeoPoint.isBlank()) { + return null; + } + String[] parts = userSuppliedGeoPoint.split(","); + // We'll supply our own errors but Solr gives a decent one: + // "Point must be in 'lat, lon' or 'x y' format: 42.3;-71.1" + if (parts.length != 2) { + String msg = "Must contain a single comma to separate latitude and longitude."; + throw new IllegalArgumentException(msg); + } + float latitude = Float.parseFloat(parts[0]); + float longitude = Float.parseFloat(parts[1]); + return latitude + "," + longitude; + } + + /** + * @return Null if supplied radius is null or whitespace. + * @throws NumberFormatException If the radius is not a positive number. + */ + public static String getGeoRadius(String userSuppliedGeoRadius) throws NumberFormatException { + if (userSuppliedGeoRadius == null || userSuppliedGeoRadius.isBlank()) { + return null; + } + float radius = 0; + try { + radius = Float.parseFloat(userSuppliedGeoRadius); + } catch (NumberFormatException ex) { + String msg = "Non-number radius supplied."; + throw new NumberFormatException(msg); + } + if (radius <= 0) { + String msg = "The supplied radius must be greater than zero."; + throw new NumberFormatException(msg); + } + return userSuppliedGeoRadius; + } + } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 0f2c77de717..fc3b911c0a5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -17,6 +17,7 @@ import java.io.UnsupportedEncodingException; import java.util.Base64; import javax.json.JsonArray; +import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.OK; import static javax.ws.rs.core.Response.Status.FORBIDDEN; import org.hamcrest.CoreMatchers; @@ -1222,18 +1223,47 @@ public void testGeospatialSearch() { Response search1 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken, "&show_entity_ids=true&geo_point=41.9580775,-70.6621063&geo_radius=50"); search1.prettyPrint(); search1.then().assertThat() + .statusCode(OK.getStatusCode()) .body("data.total_count", CoreMatchers.is(1)) .body("data.count_in_response", CoreMatchers.is(1)) - .body("data.items[0].entity_id", CoreMatchers.is(datasetId)) - .statusCode(OK.getStatusCode()); + .body("data.items[0].entity_id", CoreMatchers.is(datasetId)); // Plymouth rock (41.9580775,-70.6621063) is not within 1 km of Cambridge. Miss. Response search2 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken, "&geo_point=41.9580775,-70.6621063&geo_radius=1"); search2.prettyPrint(); search2.then().assertThat() + .statusCode(OK.getStatusCode()) .body("data.total_count", CoreMatchers.is(0)) - .body("data.count_in_response", CoreMatchers.is(0)) - .statusCode(OK.getStatusCode()); + .body("data.count_in_response", CoreMatchers.is(0)); + + } + + @Test + public void testGeospatialSearchInvalid() { + + Response noRadius = UtilIT.search("*", null, "&geo_point=40,60"); + noRadius.prettyPrint(); + noRadius.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", CoreMatchers.equalTo("If you supply geo_point you must also supply geo_radius.")); + + Response noPoint = UtilIT.search("*", null, "&geo_radius=5"); + noPoint.prettyPrint(); + noPoint.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", CoreMatchers.equalTo("If you supply geo_radius you must also supply geo_point.")); + + Response junkPoint = UtilIT.search("*", null, "&geo_point=junk&geo_radius=5"); + junkPoint.prettyPrint(); + junkPoint.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", CoreMatchers.equalTo("Must contain a single comma to separate latitude and longitude.")); + + Response junkRadius = UtilIT.search("*", null, "&geo_point=40,60&geo_radius=junk"); + junkRadius.prettyPrint(); + junkRadius.then().assertThat() + .statusCode(BAD_REQUEST.getStatusCode()) + .body("message", CoreMatchers.equalTo("Non-number radius supplied.")); } diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java index 525e03f8302..33f50c9a4c0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java @@ -91,4 +91,42 @@ public void testdetermineFinalQuery() { assertEquals("*", SearchUtil.determineFinalQuery("")); assertEquals("foo", SearchUtil.determineFinalQuery("foo")); } + + @Test + public void testGetGeoPoint() { + // valid + assertEquals("42.3,-71.1", SearchUtil.getGeoPoint("42.3,-71.1")); + // user doesn't want geospatial search + assertEquals(null, SearchUtil.getGeoPoint(null)); + // invalid + assertThrows(IllegalArgumentException.class, () -> { + SearchUtil.getGeoRadius("42.3;-71.1"); + }, "Must have a comma."); + assertThrows(IllegalArgumentException.class, () -> { + SearchUtil.getGeoRadius("-71.187346,42.33661,-71.043056,42.409599"); + }, "Must have only one comma."); + assertThrows(IllegalArgumentException.class, () -> { + SearchUtil.getGeoRadius("junk"); + }, "Must have a comma."); + assertThrows(NumberFormatException.class, () -> { + SearchUtil.getGeoRadius("somejunk,morejunk"); + }, "Must be numbers."); + } + + @Test + public void testGetGeoRadius() { + // valid + assertEquals("5", SearchUtil.getGeoRadius("5")); + assertEquals("1.5", SearchUtil.getGeoRadius("1.5")); + // user doesn't want geospatial search + assertEquals(null, SearchUtil.getGeoRadius(null)); + assertEquals(null, SearchUtil.getGeoRadius("")); + // invalid + assertThrows(NumberFormatException.class, () -> { + SearchUtil.getGeoRadius("nonNumber"); + }, "Must be a number."); + assertThrows(NumberFormatException.class, () -> { + SearchUtil.getGeoRadius("-1"); + }, "Must be greater than zero."); + } } From 6e7499e7be3586b1129d9598716e4c2e6ba4b27d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 26 Oct 2022 17:34:55 -0400 Subject: [PATCH 124/232] update docs and release note (supported via API) #8239 --- doc/release-notes/8239-geospatial-indexing.md | 6 +++++- doc/sphinx-guides/source/api/search.rst | 18 ++---------------- .../source/user/find-use-data.rst | 7 +++++++ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/doc/release-notes/8239-geospatial-indexing.md b/doc/release-notes/8239-geospatial-indexing.md index 3e6ba0e7a07..165cb9031ba 100644 --- a/doc/release-notes/8239-geospatial-indexing.md +++ b/doc/release-notes/8239-geospatial-indexing.md @@ -1 +1,5 @@ -Support for indexing the Geographic Bounding Box fields from the Geospatial metadata block has been added. This allows trusted applications with access to solr to perform geospatial queries to find datasets, e.g. those near a given point. This is also a step towards enabling geospatial queries via the Dataverse API and UI. +Support for indexing the "Geographic Bounding Box" fields ("West Longitude", "East Longitude", "North Latitude", and "South Latitude") from the Geospatial metadata block has been added. + +Geospatial search is supported but only via API using two new parameters: `geo_point` and `geo_radius`. + +A Solr schema update is required. diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index c2311ead089..b941064f173 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -35,8 +35,8 @@ show_relevance boolean Whether or not to show details of which fields were ma show_facets boolean Whether or not to show facets that can be operated on by the "fq" parameter. False by default. See :ref:`advanced search example `. fq string A filter query on the search term. Multiple "fq" parameters can be used. See :ref:`advanced search example `. show_entity_ids boolean Whether or not to show the database IDs of the search results (for developer use). -geo_point string Latitude and longitude in the form ``geo_point=42.3,-71.1``. -geo_radius string Radial distance in kilometers such as ``geo_radius=5``. +geo_point string Latitude and longitude in the form ``geo_point=42.3,-71.1``. You must supply ``geo_radius`` as well. See also :ref:`geospatial-search`. +geo_radius string Radial distance in kilometers from ``geo_point`` (which must be supplied as well) such as ``geo_radius=1.5``. metadata_fields string Includes the requested fields for each dataset in the response. Multiple "metadata_fields" parameters can be used to include several fields. The value must be in the form "{metadata_block_name}:{field_name}" to include a specific field from a metadata block (see :ref:`example `) or "{metadata_field_set_name}:\*" to include all the fields for a metadata block (see :ref:`example `). "{field_name}" cannot be a subfield of a compound field. If "{field_name}" is a compound field, all subfields are included. =============== ======= =========== @@ -732,17 +732,3 @@ Output from iteration example CORS - - -Geospatial Indexing -------------------- - -Dataverse indexes the Geospatial Bounding Box field from the Geospatial metadatablock as a solr.BBoxField enabling `Spatial Search `_. This capability is not yet exposed through the Dataverse API or UI but can be accessed by trusted applications with direct solr access. -For example, a query of the form - -.. code-block:: none - - q=*.*&fq={!bbox sfield=geolocation}=&pt=10,10&d=5 - - -would find datasets with information near the point latitude=10, longitude=10. diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 42e1a2b23d4..2e82a1482b4 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -39,6 +39,13 @@ enter search terms for Dataverse collections, dataset metadata (citation and dom metadata. If you are searching for tabular data files you can also search at the variable level for name and label. To find out more about what each field searches, hover over the field name for a detailed description of the field. +.. _geospatial-search: + +Geospatial Search +----------------- + +Geospatial search is available from the :doc:`/api/search` (look for "geo" parameters). The metadata fields that are geospatially indexed are "West Longitude", "East Longitude", "North Latitude", and "South Latitude" from the "Geographic Bounding Box" field in the "Geospatial Metadata" block. + Browsing a Dataverse Installation --------------------------------- From c80a06f38b679641d80a05b5215106006da52667 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 28 Oct 2022 18:07:07 -0400 Subject: [PATCH 125/232] fix for cvv and editMetadata replace=true, and test --- .../source/_static/api/dataset-add-subject-metadata.json | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 1 + .../java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 8 +++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/_static/api/dataset-add-subject-metadata.json b/doc/sphinx-guides/source/_static/api/dataset-add-subject-metadata.json index ea0922dadc8..c81c5b32aab 100644 --- a/doc/sphinx-guides/source/_static/api/dataset-add-subject-metadata.json +++ b/doc/sphinx-guides/source/_static/api/dataset-add-subject-metadata.json @@ -2,7 +2,7 @@ "typeName": "subject", "value": ["Astronomy and Astrophysics", "Agricultural Sciences", -"Arts and Humanities", "Physics"] +"Arts and Humanities", "Physics", "Mathematical Sciences"] } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index aff543e643c..2ae4544ae68 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -986,6 +986,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque dsf.setSingleValue(""); dsf.setSingleControlledVocabularyValue(null); } + cvvDisplay=""; } if (updateField.getDatasetFieldType().isControlledVocabulary()) { if (dsf.getDatasetFieldType().isAllowMultiples()) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 529af5f746c..326b3963217 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -67,6 +67,7 @@ import javax.xml.stream.XMLStreamReader; import static org.junit.Assert.assertEquals; import org.hamcrest.CoreMatchers; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.CoreMatchers.nullValue; @@ -76,7 +77,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.junit.matchers.JUnitMatchers.containsString; + public class DatasetsIT { @@ -272,9 +273,10 @@ public void testAddUpdateDatasetViaNativeAPI() { String pathToJsonFileSingle = "doc/sphinx-guides/source/_static/api/dataset-simple-update-metadata.json"; Response addSubjectSingleViaNative = UtilIT.updateFieldLevelDatasetMetadataViaNative(datasetPersistentId, pathToJsonFileSingle, apiToken); - addSubjectSingleViaNative.prettyPrint(); + String responseString = addSubjectSingleViaNative.prettyPrint(); addSubjectSingleViaNative.then().assertThat() - .statusCode(OK.getStatusCode()); + .statusCode(OK.getStatusCode()).body(containsString("Mathematical Sciences")).body(containsString("Social Sciences")); + //Trying to blank out required field should fail... From 5993be8a5adecd41bf138e35ceda212cac522f92 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 28 Oct 2022 18:15:35 -0400 Subject: [PATCH 126/232] check math exists before update --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 326b3963217..18afb88fb3b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -268,7 +268,7 @@ public void testAddUpdateDatasetViaNativeAPI() { addSubjectViaNative = UtilIT.addDatasetMetadataViaNative(datasetPersistentId, pathToJsonFile, apiToken); addSubjectViaNative.prettyPrint(); addSubjectViaNative.then().assertThat() - .statusCode(OK.getStatusCode()); + .statusCode(OK.getStatusCode()).body(containsString("Mathematical Sciences")); String pathToJsonFileSingle = "doc/sphinx-guides/source/_static/api/dataset-simple-update-metadata.json"; From 25521d8459605ba12f7a3bf036d1e4957ef2f658 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 31 Oct 2022 16:37:31 -0400 Subject: [PATCH 127/232] A quick implementation of a mechanism for disabling new account signups for remote auth. of specific type (without blocking all the existing accounts of the type). #9111 --- .../oauth2/OAuth2LoginBackingBean.java | 52 +++++++++++++++++-- .../settings/SettingsServiceBean.java | 38 +++++++++++++- .../iq/dataverse/util/SystemConfig.java | 11 ++++ src/main/webapp/oauth2/callback.xhtml | 2 +- 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java index 225352dec43..58412d6d51f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.authorization.providers.oauth2; import edu.harvard.iq.dataverse.DataverseSession; +import edu.harvard.iq.dataverse.authorization.AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -27,6 +28,8 @@ import static edu.harvard.iq.dataverse.util.StringUtil.toOption; import edu.harvard.iq.dataverse.util.SystemConfig; +import java.util.ArrayList; +import java.util.Collections; import org.omnifaces.util.Faces; /** @@ -45,6 +48,8 @@ public class OAuth2LoginBackingBean implements Serializable { private String responseBody; Optional redirectPage = Optional.empty(); private OAuth2Exception error; + private boolean disabled = false; + private boolean signUpDisabled = false; /** * TODO: Only used in exchangeCodeForToken(). Make local var in method. */ @@ -96,13 +101,26 @@ public void exchangeCodeForToken() throws IOException { AbstractOAuth2AuthenticationProvider idp = oIdp.get(); oauthUser = idp.getUserRecord(code.get(), systemConfig.getOAuth2CallbackUrl()); + // Throw an error if this authentication method is disabled: + if (isProviderDisabled(idp.getId())) { + disabled = true; + throw new OAuth2Exception(-1, "", "This authentication method ("+idp.getId()+") is currently disabled. Please log in using one of the supported methods."); + } + UserRecordIdentifier idtf = oauthUser.getUserRecordIdentifier(); AuthenticatedUser dvUser = authenticationSvc.lookupUser(idtf); if (dvUser == null) { - // need to create the user - newAccountPage.setNewUser(oauthUser); - Faces.redirect("/oauth2/firstLogin.xhtml"); + // need to create the user - unless signups are disabled + // for this authentication method; in which case, throw + // an error: + if (systemConfig.isSignupDisabledForRemoteAuthProvider(idp.getId())) { + signUpDisabled = true; + throw new OAuth2Exception(-1, "", "Sorry, signup for new accounts using "+idp.getId()+" authentication is currently disabled."); + } else { + newAccountPage.setNewUser(oauthUser); + Faces.redirect("/oauth2/firstLogin.xhtml"); + } } else { // login the user and redirect to HOME of intended page (if any). @@ -271,4 +289,32 @@ public List getProviders() { public boolean isOAuth2ProvidersDefined() { return !authenticationSvc.getOAuth2Providers().isEmpty(); } + + public boolean isDisabled() { + return disabled; + } + + public boolean isSignUpDisabled() { + return signUpDisabled; + } + + private boolean isProviderDisabled(String providerId) { + // Compare this provider id against the list of *enabled* auth providers + // returned by the Authentication Service: + List idps = new ArrayList<>(authenticationSvc.getAuthenticationProviders()); + + // for the tests to work: + if (idps.isEmpty()) { + return false; + } + + for (AuthenticationProvider idp : idps) { + if (idp != null) { + if (providerId.equals(idp.getId())) { + return false; + } + } + } + return true; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 50e29d2a333..46b4ae5cbc5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -563,7 +563,11 @@ Whether Harvesting (OAI) service is enabled /* * Allow a custom JavaScript to control values of specific fields. */ - ControlledVocabularyCustomJavaScript + ControlledVocabularyCustomJavaScript, + /** + * A compound setting for disabling signup for remote Auth providers: + */ + AllowRemoteAuthSignUp ; @Override @@ -668,7 +672,39 @@ public Long getValueForCompoundKeyAsLong(Key key, String param){ } } + + /** + * Same, but with Booleans + * (returns null if not set; the calling method will decide what that shouldall + * default to) + * Example: + * :AllowRemoteAuthSignUp {"default":"true","google":"false"} + */ + public Boolean getValueForCompoundKeyAsBoolean(Key key, String param) { + + String val = this.getValueForKey(key); + + if (val == null) { + return null; + } + + try (StringReader rdr = new StringReader(val)) { + JsonObject settings = Json.createReader(rdr).readObject(); + if (settings.containsKey(param)) { + return Boolean.parseBoolean(settings.getString(param)); + } else if (settings.containsKey("default")) { + return Boolean.parseBoolean(settings.getString("default")); + } else { + return null; + } + + } catch (Exception e) { + logger.log(Level.WARNING, "Incorrect setting. Could not convert \"{0}\" from setting {1} to boolean: {2}", new Object[]{val, key.toString(), e.getMessage()}); + return null; + } + + } /** * Return the value stored, or the default value, in case no setting by that * name exists. The main difference between this method and the other {@code get()}s diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 7abd0d02065..3010ce208a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1228,4 +1228,15 @@ public Map getCurationLabels() { } return labelMap; } + + public boolean isSignupDisabledForRemoteAuthProvider(String providerId) { + Boolean ret = settingsService.getValueForCompoundKeyAsBoolean(SettingsServiceBean.Key.AllowRemoteAuthSignUp, providerId); + + // we default to false if it's null, i.e. if not present: + if (ret == null) { + return false; + } + + return !ret; + } } diff --git a/src/main/webapp/oauth2/callback.xhtml b/src/main/webapp/oauth2/callback.xhtml index f0d66b2fa74..76a17b6e113 100644 --- a/src/main/webapp/oauth2/callback.xhtml +++ b/src/main/webapp/oauth2/callback.xhtml @@ -21,7 +21,7 @@
+ jsf:rendered="#{!empty DatasetUtil:getLocalizedLicenseDetails(DatasetPage.workingVersion.termsOfUseAndAccess.license.name,'.description')} }">
- +
From 4c25878b6d7e903274b5d25623548867783acf17 Mon Sep 17 00:00:00 2001 From: cstr Date: Thu, 3 Nov 2022 15:02:44 +0800 Subject: [PATCH 132/232] Update citation.properties --- src/main/java/propertyFiles/citation.properties | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index ef8b44d7114..f35ede79b50 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -251,7 +251,7 @@ controlledvocabulary.subject.social_sciences=Social Sciences controlledvocabulary.subject.other=Other controlledvocabulary.publicationIDType.ark=ark controlledvocabulary.publicationIDType.arxiv=arXiv -controlledvocabulary.publicationIDType.cstr=CSTR +controlledvocabulary.publicationIDType.cstr=cstr controlledvocabulary.publicationIDType.bibcode=bibcode controlledvocabulary.publicationIDType.doi=doi controlledvocabulary.publicationIDType.ean13=ean13 @@ -346,7 +346,7 @@ controlledvocabulary.language.galician=Galician controlledvocabulary.language.georgian=Georgian controlledvocabulary.language.german=German controlledvocabulary.language.greek_(modern)=Greek (modern) -controlledvocabulary.language.guarani=Guaraní +controlledvocabulary.language.guarani=Guaraní controlledvocabulary.language.gujarati=Gujarati controlledvocabulary.language.haitian,_haitian_creole=Haitian, Haitian Creole controlledvocabulary.language.hausa=Hausa @@ -406,7 +406,7 @@ controlledvocabulary.language.navajo,_navaho=Navajo, Navaho controlledvocabulary.language.northern_ndebele=Northern Ndebele controlledvocabulary.language.nepali=Nepali controlledvocabulary.language.ndonga=Ndonga -controlledvocabulary.language.norwegian_bokmal=Norwegian Bokmål +controlledvocabulary.language.norwegian_bokmal=Norwegian BokmÃ¥l controlledvocabulary.language.norwegian_nynorsk=Norwegian Nynorsk controlledvocabulary.language.norwegian=Norwegian controlledvocabulary.language.nuosu=Nuosu @@ -468,7 +468,7 @@ controlledvocabulary.language.urdu=Urdu controlledvocabulary.language.uzbek=Uzbek controlledvocabulary.language.venda=Venda controlledvocabulary.language.vietnamese=Vietnamese -controlledvocabulary.language.volapuk=Volapük +controlledvocabulary.language.volapuk=Volapük controlledvocabulary.language.walloon=Walloon controlledvocabulary.language.welsh=Welsh controlledvocabulary.language.wolof=Wolof @@ -478,4 +478,4 @@ controlledvocabulary.language.yiddish=Yiddish controlledvocabulary.language.yoruba=Yoruba controlledvocabulary.language.zhuang,_chuang=Zhuang, Chuang controlledvocabulary.language.zulu=Zulu -controlledvocabulary.language.not_applicable=Not applicable \ No newline at end of file +controlledvocabulary.language.not_applicable=Not applicable From e068cabe90effe55e3078101dc85e91de2310eff Mon Sep 17 00:00:00 2001 From: chenganj Date: Thu, 3 Nov 2022 11:38:08 -0400 Subject: [PATCH 133/232] correction --- .../java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 75cde7b4bd9..fecfdc2bcfb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -584,11 +584,11 @@ public static String getLocalizedLicenseDetails(String licenseName,String keyPar localizedLicenseValue = BundleUtil.getStringFromPropertyFile(key, "License"); } catch (Exception e) { - localizedLicenseValue = licenseName.toLowerCase(); + localizedLicenseValue = licenseName; } if (localizedLicenseValue == null) { - localizedLicenseValue = licenseName.toLowerCase() ; + localizedLicenseValue = licenseName ; } return localizedLicenseValue; From 57dd54ae807d0d30d83b5d5a6064ab79c820f46c Mon Sep 17 00:00:00 2001 From: chenganj Date: Thu, 3 Nov 2022 12:00:55 -0400 Subject: [PATCH 134/232] added additional doc --- doc/sphinx-guides/source/installation/config.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 3cdac253cb3..d2ef3a165cf 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1044,6 +1044,14 @@ On a new Dataverse installation, users may select from the following licenses or (Note that existing Dataverse installations which are upgraded from 5.9 or previous will only offer CC0 1.0, added automatically during the upgrade to version 5.10.) +If the Dataverse Installation supports multiple languages, the license name/description translations should be added to the ``License`` properties files. (See :ref:`i18n` for more on properties files and internationalization in general.) +To create the key, the license name has to be converted to lowercase, replace space with underscore. + +Example:: + + license.cc0_1.0.description=Creative Commons CC0 1.0 Universal Public Domain Dedication. + license.cc0_1.0.name=CC0 1.0 + You have a lot of control over which licenses and terms are available. You can remove licenses and add new ones. You can decide which license is the default. You can remove "Custom Dataset Terms" as a option. You can remove all licenses and make "Custom Dataset Terms" the only option. Before making changes, you are encouraged to read the :ref:`license-terms` section of the User Guide about why CC0 is the default and what the "Custom Dataset Terms" option allows. From cf00e61a531dfd2b7287becd61af18fa9bbdab50 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 3 Nov 2022 14:56:56 -0400 Subject: [PATCH 135/232] clarify OAuth only, put like with like, "sign up" #9111 --- .../source/admin/user-administration.rst | 4 +- .../source/developers/troubleshooting.rst | 2 +- .../source/installation/config.rst | 46 +++++++++++-------- .../source/installation/oauth2.rst | 5 ++ 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/doc/sphinx-guides/source/admin/user-administration.rst b/doc/sphinx-guides/source/admin/user-administration.rst index 608a8ab2b72..a21263f6f17 100644 --- a/doc/sphinx-guides/source/admin/user-administration.rst +++ b/doc/sphinx-guides/source/admin/user-administration.rst @@ -57,9 +57,9 @@ See :ref:`deactivate-a-user` Confirm Email ------------- -A Dataverse installation encourages builtin/local users to verify their email address upon signup or email change so that sysadmins can be assured that users can be contacted. +A Dataverse installation encourages builtin/local users to verify their email address upon sign up or email change so that sysadmins can be assured that users can be contacted. -The app will send a standard welcome email with a URL the user can click, which, when activated, will store a ``lastconfirmed`` timestamp in the ``authenticateduser`` table of the database. Any time this is "null" for a user (immediately after signup and/or changing of their Dataverse installation email address), their current email on file is considered to not be verified. The link that is sent expires after a time (the default is 24 hours), but this is configurable by a superuser via the ``:MinutesUntilConfirmEmailTokenExpires`` config option. +The app will send a standard welcome email with a URL the user can click, which, when activated, will store a ``lastconfirmed`` timestamp in the ``authenticateduser`` table of the database. Any time this is "null" for a user (immediately after sign up and/or changing of their Dataverse installation email address), their current email on file is considered to not be verified. The link that is sent expires after a time (the default is 24 hours), but this is configurable by a superuser via the ``:MinutesUntilConfirmEmailTokenExpires`` config option. Should users' URL token expire, they will see a "Verify Email" button on the account information page to send another URL. diff --git a/doc/sphinx-guides/source/developers/troubleshooting.rst b/doc/sphinx-guides/source/developers/troubleshooting.rst index 0463a68d8c8..832785f9860 100755 --- a/doc/sphinx-guides/source/developers/troubleshooting.rst +++ b/doc/sphinx-guides/source/developers/troubleshooting.rst @@ -41,7 +41,7 @@ This command helps verify what host your domain is using to send mail. Even if i 2. From the left-side panel, select **JavaMail Sessions** 3. You should see one session named **mail/notifyMailSession** -- click on that. -From this window you can modify certain fields of your Dataverse installation's notifyMailSession, which is the JavaMail session for outgoing system email (such as on user signup or data publication). Two of the most important fields we need are: +From this window you can modify certain fields of your Dataverse installation's notifyMailSession, which is the JavaMail session for outgoing system email (such as on user sign up or data publication). Two of the most important fields we need are: - **Mail Host:** The DNS name of the default mail server (e.g. smtp.gmail.com) - **Default User:** The username provided to your Mail Host when you connect to it (e.g. johndoe@gmail.com) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 4cd28783908..c61bf451eb7 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -800,7 +800,7 @@ Refer to :ref:`:NavbarSupportUrl` for setting to a fully-qualified URL which wil Sign Up ####### -Refer to :ref:`:SignUpUrl` and :ref:`conf-allow-signup` for setting a relative path URL to which users will be sent for signup and for controlling the ability for creating local user accounts. +Refer to :ref:`:SignUpUrl` and :ref:`conf-allow-signup` for setting a relative path URL to which users will be sent for sign up and for controlling the ability for creating local user accounts. Custom Header ^^^^^^^^^^^^^ @@ -2126,23 +2126,6 @@ Notes: - For larger file upload sizes, you may need to configure your reverse proxy timeout. If using apache2 (httpd) with Shibboleth, add a timeout to the ProxyPass defined in etc/httpd/conf.d/ssl.conf (which is described in the :doc:`/installation/shibboleth` setup). -:AllowRemoteAuthSignUp -++++++++++++++++++++++ - -This is a **compound** setting that enables or disables signup for new accounts for individual OAuth2 authentication methods (such as Orcid, Google and GitHub). This way it is possible to continue allowing logins via an OAuth2 provider for already existing accounts, without letting new users create accounts with this method. - -By default, if the setting is not present, all remote signups are open. If the setting is present, but the value for this specific method is not specified, it is assumed that the signups are allowed for it. - -Examples: - -``curl -X PUT -d '{"default":"false"}' http://localhost:8080/api/admin/settings/:AllowRemoteAuthSignUp`` - -disables all signups. - -``curl -X PUT -d '{"default":"true","google":"false"}' http://localhost:8080/api/admin/settings/:AllowRemoteAuthSignUp`` - -keeps signups open for all the methods except google. (but note that the "default":"true" part in this example is redundant, since it would default to true anyway for all the methods other than google). - :MultipleUploadFilesLimit +++++++++++++++++++++++++ @@ -2214,7 +2197,7 @@ If ``:SolrFullTextIndexing`` is set to true, the content of files of any size wi :SignUpUrl ++++++++++ -The relative path URL to which users will be sent for signup. The default setting is below. +The relative path URL to which users will be sent for sign up. The default setting is below. ``curl -X PUT -d '/dataverseuser.xhtml?editMode=CREATE' http://localhost:8080/api/admin/settings/:SignUpUrl`` @@ -2290,6 +2273,31 @@ Set to false to disallow local accounts from being created. See also the section ``curl -X PUT -d 'false' http://localhost:8080/api/admin/settings/:AllowSignUp`` +.. _:AllowRemoteAuthSignUp: + +:AllowRemoteAuthSignUp +++++++++++++++++++++++ + +This is a **compound** setting that enables or disables sign up for new accounts for individual OAuth2 authentication methods (such as Orcid, Google and GitHub). This way it is possible to continue allowing logins via an OAuth2 provider for already existing accounts, without letting new users create accounts with this method. + +By default, if the setting is not present, all configured OAuth sign ups are allowed. If the setting is present, but the value for this specific method is not specified, it is assumed that the sign ups are allowed for it. + +Examples: + +This curl command... + +``curl -X PUT -d '{"default":"false"}' http://localhost:8080/api/admin/settings/:AllowRemoteAuthSignUp`` + +...disables all OAuth sign ups. + +This curl command... + +``curl -X PUT -d '{"default":"true","google":"false"}' http://localhost:8080/api/admin/settings/:AllowRemoteAuthSignUp`` + +...keeps sign ups open for all the OAuth login providers except google. (That said, note that the ``"default":"true"`` part in this example is redundant, since it would default to true anyway for all the methods other than google.) + +See also :doc:`oauth2`. + :FileFixityChecksumAlgorithm ++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/installation/oauth2.rst b/doc/sphinx-guides/source/installation/oauth2.rst index 0dfdb0393e0..8dffde87cc2 100644 --- a/doc/sphinx-guides/source/installation/oauth2.rst +++ b/doc/sphinx-guides/source/installation/oauth2.rst @@ -78,6 +78,11 @@ This template can be used for configuring this setting (**this is not something - :download:`orcid-sandbox.json <../_static/installation/files/root/auth-providers/orcid-sandbox.json>` +Disabling Sign Up +~~~~~~~~~~~~~~~~~ + +See :ref:`:AllowRemoteAuthSignUp`. + Converting Local Users to OAuth ------------------------------- From 3aacc4ebdf3844772d648ee58eed313cb65850d9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 3 Nov 2022 15:42:05 -0400 Subject: [PATCH 136/232] add 5.12.1 release notes --- doc/release-notes/5.12.1-release-notes.md | 102 ++++++++++++++++++ ...732-date-in-citation-harvested-datasets.md | 7 -- doc/release-notes/8733-oai_dc-date.md | 4 - .../9111-oauth2-disabling-signups | 3 - 4 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 doc/release-notes/5.12.1-release-notes.md delete mode 100644 doc/release-notes/8732-date-in-citation-harvested-datasets.md delete mode 100644 doc/release-notes/8733-oai_dc-date.md delete mode 100644 doc/release-notes/9111-oauth2-disabling-signups diff --git a/doc/release-notes/5.12.1-release-notes.md b/doc/release-notes/5.12.1-release-notes.md new file mode 100644 index 00000000000..8872fec060b --- /dev/null +++ b/doc/release-notes/5.12.1-release-notes.md @@ -0,0 +1,102 @@ +# Dataverse Software 5.12.1 + +This release brings new features, enhancements, and bug fixes to the Dataverse Software. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +## Release Highlights + +### Production Date Now Used for Harvested Datasets in Addition to Distribution Date (`oai_dc` format) + +Fix the year displayed in citation for harvested dataset, especially for `oai_dc` format. + +For normal datasets, the date used is the "citation date" which is by default the publication date (the first release date) unless you [change it](https://guides.dataverse.org/en/5.12.1/api/native-api.html#set-citation-date-field-type-for-a-dataset). + +However, for a harvested dataset, the distribution date was used instead and this date is not always present in the harvested metadata. + +Now, the production date is used for harvested dataset in addition to distribution date when harvesting with the `oai_dc` format. + +### Publication Date Now Used for Harvested Dataset if Production Date is Not Set (`oai_dc` format) + +For exports and harvesting in `oai_dc` format, if "Production Date" is not set, "Publication Date" is now used instead. This change is reflected in the [Dataverse 4+ Metadata Crosswalk][] linked from the [Appendix][] of the User Guide. + +[Dataverse 4+ Metadata Crosswalk]: https://docs.google.com/spreadsheets/d/10Luzti7svVTVKTA-px27oq3RxCUM-QbiTkm8iMd5C54/edit#gid=1901625433&range=K7 +[Appendix]: https://guides.dataverse.org/en/5.12.1/user/appendix.html + +### Ability to Disable OAuth Sign Up While Allowing Existing Accounts to Log In + +A new option called `:AllowRemoteAuthSignUp` has been added providing a mechanism for disabling new account signups for specific OAuth2 authentication providers (Orcid, GitHub, Google etc.) while still allowing logins for already-existing accounts using this authentication method. + +See the [Installation Guide](https://guides.dataverse.org/en/5.12.1/installation/config.html#allowremoteauthsignup) for more information on the setting. + +## Major Use Cases and Infrastructure Enhancements + +Changes and fixes in this release include: + +- When harvesting datasets, I want the Production Date if I can't get the Distribution Date (PR #8732) +- When harvesting datasets, I want the Publication Date if I can't get the Production Date (PR #8733) +- As a sysadmin I'd like to disable (temporarily or permanently) sign ups from OAuth providers while allowing existing users to continue to log in from that provider (PR #9112) +- As a C/C++ developer I want to use Dataverse APIs (PR #9070) + +## New DB Settings + +The following DB settings have been added: + +- `:AllowRemoteAuthSignUp` + +See the [Database Settings](https://guides.dataverse.org/en/5.12.1/installation/config.html#database-settings) section of the Guides for more information. + +## Complete List of Changes + +For the complete list of code changes in this release, see the [5.12.1 Milestone](https://github.com/IQSS/dataverse/milestone/106?closed=1) in GitHub. + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.12.1/installation/). Please also contact us to get added to the [Dataverse Project Map](https://guides.dataverse.org/en/5.10/installation/config.html#putting-your-dataverse-installation-on-the-map-at-dataverse-org) if you have not done so already. + +## Upgrade Instructions + +Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc. + +0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.12.1. + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. + +```shell +export PAYARA=/usr/local/payara5 +``` + +(or `setenv PAYARA /usr/local/payara5` if you are using a `csh`-like shell) + +1\. Undeploy the previous version + +```shell + $PAYARA/bin/asadmin list-applications + $PAYARA/bin/asadmin undeploy dataverse<-version> +``` + +2\. Stop Payara + +```shell + service payara stop + rm -rf $PAYARA/glassfish/domains/domain1/generated +``` + +6\. Start Payara + +```shell + service payara start +``` + +7\. Deploy this version. + +```shell + $PAYARA/bin/asadmin deploy dataverse-5.12.1.war +``` + +8\. Restart payara + +```shell + service payara stop + service payara start +``` diff --git a/doc/release-notes/8732-date-in-citation-harvested-datasets.md b/doc/release-notes/8732-date-in-citation-harvested-datasets.md deleted file mode 100644 index 85f2d24a8a9..00000000000 --- a/doc/release-notes/8732-date-in-citation-harvested-datasets.md +++ /dev/null @@ -1,7 +0,0 @@ -Fix the year displayed in citation for harvested dataset, specialy for oai_dc format. - -For normal datasets, the date used is the "citation date" which is by default the publication date (the first release date) (https://guides.dataverse.org/en/latest/api/native-api.html?highlight=citationdate#set-citation-date-field-type-for-a-dataset). - -But for a harvested dataset, the distribution date is used instead and this date is not always present in the harvested metadata. With oai_dc format the date tag if used as production date. - -Now, the production date is used for harvested dataset in addition to distribution date. \ No newline at end of file diff --git a/doc/release-notes/8733-oai_dc-date.md b/doc/release-notes/8733-oai_dc-date.md deleted file mode 100644 index a2a09f361d3..00000000000 --- a/doc/release-notes/8733-oai_dc-date.md +++ /dev/null @@ -1,4 +0,0 @@ -For exports and harvesting in `oai_dc` format, if "Production Date" is not set, "Publication Date" is now used instead. This change is reflected in the [Dataverse 4+ Metadata Crosswalk][] linked from the [Appendix][] of the User Guide. - -[Dataverse 4+ Metadata Crosswalk]: https://docs.google.com/spreadsheets/d/10Luzti7svVTVKTA-px27oq3RxCUM-QbiTkm8iMd5C54/edit#gid=1901625433&range=K7 -[Appendix]: https://guides.dataverse.org/en/latest/user/appendix.html diff --git a/doc/release-notes/9111-oauth2-disabling-signups b/doc/release-notes/9111-oauth2-disabling-signups deleted file mode 100644 index 81f46128701..00000000000 --- a/doc/release-notes/9111-oauth2-disabling-signups +++ /dev/null @@ -1,3 +0,0 @@ -A new option :AllowRemoteAuthSignUp has been added providing a mechanism for disabling new account signups for specific OAuth2 authentication providers (Orcid, GitHub, Google etc.) while still allowing logins for already-existing accounts using this authentication method. - -See the config guide (https://guides.dataverse.org/en/latest/installation/config.html) for more information on the setting. From 1ebeebdf190afb9fc66430993aa09bd7b0066201 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 3 Nov 2022 15:53:33 -0400 Subject: [PATCH 137/232] bump version to 5.12.1 #9124 --- doc/sphinx-guides/source/conf.py | 4 ++-- doc/sphinx-guides/source/versions.rst | 3 ++- modules/dataverse-parent/pom.xml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 880ed561720..590eee4bd9d 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -66,9 +66,9 @@ # built documents. # # The short X.Y version. -version = '5.12' +version = '5.12.1' # The full version, including alpha/beta/rc tags. -release = '5.12' +release = '5.12.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index 1cbd785b5dd..e0a344de9a1 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -6,7 +6,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. -- 5.12 +- 5.12.1 +- `5.12 `__ - `5.11.1 `__ - `5.11 `__ - `5.10.1 `__ diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index ccc0a9a7f60..c1ba693da1b 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -129,7 +129,7 @@ - 5.12 + 5.12.1 11 UTF-8 From a9a51e709c642c72fa26b3e87ceca30cf35c1394 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 3 Nov 2022 16:14:12 -0400 Subject: [PATCH 138/232] explain create bug (500 error) #9103 --- doc/release-notes/5.12.1-release-notes.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/release-notes/5.12.1-release-notes.md b/doc/release-notes/5.12.1-release-notes.md index 8872fec060b..59a405f85ba 100644 --- a/doc/release-notes/5.12.1-release-notes.md +++ b/doc/release-notes/5.12.1-release-notes.md @@ -4,6 +4,12 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S ## Release Highlights +## Bug Fix for "Internal Server Error" When Creating a New Remote Account + +Unfortunately, as of 5.11 new remote users have seen "Internal Server Error" when creating an account or checking notifications. By "remote" user we mean any user that is not creating a builtin/local account. That is, remote users log in with institutional (Shibboleth), OAuth (ORCID, GitHub, or Google) or OIDC providers. + +This is a transient error that can be worked around by reloading the browser (or logging out and back in again) but it's obviously a very poor user experience and a bad first impression. This bug is the primary reason we are putting out this patch release. Other features and bug fixes are coming along for the ride. + ### Production Date Now Used for Harvested Datasets in Addition to Distribution Date (`oai_dc` format) Fix the year displayed in citation for harvested dataset, especially for `oai_dc` format. @@ -31,6 +37,7 @@ See the [Installation Guide](https://guides.dataverse.org/en/5.12.1/installation Changes and fixes in this release include: +- Users creating an account by logging in with Shibboleth, OAuth, or OIDC should not see errors. (Issue 9029, PR #9030) - When harvesting datasets, I want the Production Date if I can't get the Distribution Date (PR #8732) - When harvesting datasets, I want the Publication Date if I can't get the Production Date (PR #8733) - As a sysadmin I'd like to disable (temporarily or permanently) sign ups from OAuth providers while allowing existing users to continue to log in from that provider (PR #9112) From b86ac3898e2dfd1bd89675ad05b697d209974970 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 3 Nov 2022 16:23:13 -0400 Subject: [PATCH 139/232] reorder #9103 --- doc/release-notes/5.12.1-release-notes.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/release-notes/5.12.1-release-notes.md b/doc/release-notes/5.12.1-release-notes.md index 59a405f85ba..4701a969dc6 100644 --- a/doc/release-notes/5.12.1-release-notes.md +++ b/doc/release-notes/5.12.1-release-notes.md @@ -10,6 +10,12 @@ Unfortunately, as of 5.11 new remote users have seen "Internal Server Error" whe This is a transient error that can be worked around by reloading the browser (or logging out and back in again) but it's obviously a very poor user experience and a bad first impression. This bug is the primary reason we are putting out this patch release. Other features and bug fixes are coming along for the ride. +### Ability to Disable OAuth Sign Up While Allowing Existing Accounts to Log In + +A new option called `:AllowRemoteAuthSignUp` has been added providing a mechanism for disabling new account signups for specific OAuth2 authentication providers (Orcid, GitHub, Google etc.) while still allowing logins for already-existing accounts using this authentication method. + +See the [Installation Guide](https://guides.dataverse.org/en/5.12.1/installation/config.html#allowremoteauthsignup) for more information on the setting. + ### Production Date Now Used for Harvested Datasets in Addition to Distribution Date (`oai_dc` format) Fix the year displayed in citation for harvested dataset, especially for `oai_dc` format. @@ -27,12 +33,6 @@ For exports and harvesting in `oai_dc` format, if "Production Date" is not set, [Dataverse 4+ Metadata Crosswalk]: https://docs.google.com/spreadsheets/d/10Luzti7svVTVKTA-px27oq3RxCUM-QbiTkm8iMd5C54/edit#gid=1901625433&range=K7 [Appendix]: https://guides.dataverse.org/en/5.12.1/user/appendix.html -### Ability to Disable OAuth Sign Up While Allowing Existing Accounts to Log In - -A new option called `:AllowRemoteAuthSignUp` has been added providing a mechanism for disabling new account signups for specific OAuth2 authentication providers (Orcid, GitHub, Google etc.) while still allowing logins for already-existing accounts using this authentication method. - -See the [Installation Guide](https://guides.dataverse.org/en/5.12.1/installation/config.html#allowremoteauthsignup) for more information on the setting. - ## Major Use Cases and Infrastructure Enhancements Changes and fixes in this release include: From b2a9756fac4fff8b879e19d4d1269d6c2e8997d7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 3 Nov 2022 16:26:07 -0400 Subject: [PATCH 140/232] reword, clarify notifications (on create), heading #9103 --- doc/release-notes/5.12.1-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/5.12.1-release-notes.md b/doc/release-notes/5.12.1-release-notes.md index 4701a969dc6..a902d9b9173 100644 --- a/doc/release-notes/5.12.1-release-notes.md +++ b/doc/release-notes/5.12.1-release-notes.md @@ -4,9 +4,9 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S ## Release Highlights -## Bug Fix for "Internal Server Error" When Creating a New Remote Account +### Bug Fix for "Internal Server Error" When Creating a New Remote Account -Unfortunately, as of 5.11 new remote users have seen "Internal Server Error" when creating an account or checking notifications. By "remote" user we mean any user that is not creating a builtin/local account. That is, remote users log in with institutional (Shibboleth), OAuth (ORCID, GitHub, or Google) or OIDC providers. +Unfortunately, as of 5.11 new remote users have seen "Internal Server Error" when creating an account (or checking notifications just after creating an account). Remote users are those who log in with institutional (Shibboleth), OAuth (ORCID, GitHub, or Google) or OIDC providers. This is a transient error that can be worked around by reloading the browser (or logging out and back in again) but it's obviously a very poor user experience and a bad first impression. This bug is the primary reason we are putting out this patch release. Other features and bug fixes are coming along for the ride. From 089adf7db5cbf12e9b23aed63079c297413cfe2e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 4 Nov 2022 09:25:03 -0400 Subject: [PATCH 141/232] Update doc/release-notes/5.12.1-release-notes.md Co-authored-by: Oliver Bertuch --- doc/release-notes/5.12.1-release-notes.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/release-notes/5.12.1-release-notes.md b/doc/release-notes/5.12.1-release-notes.md index a902d9b9173..584f68477a4 100644 --- a/doc/release-notes/5.12.1-release-notes.md +++ b/doc/release-notes/5.12.1-release-notes.md @@ -106,4 +106,3 @@ export PAYARA=/usr/local/payara5 ```shell service payara stop service payara start -``` From bebd6e792dbb9dddec6479878e954cf04cbef187 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 4 Nov 2022 09:42:46 -0400 Subject: [PATCH 142/232] add note about Payara 6 #9103 --- doc/release-notes/5.12.1-release-notes.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/release-notes/5.12.1-release-notes.md b/doc/release-notes/5.12.1-release-notes.md index 584f68477a4..aa8896660f3 100644 --- a/doc/release-notes/5.12.1-release-notes.md +++ b/doc/release-notes/5.12.1-release-notes.md @@ -106,3 +106,10 @@ export PAYARA=/usr/local/payara5 ```shell service payara stop service payara start +``` + +## Upcoming Versions of Payara + +With the recent release of Payara 6 ([Payara 6.2022.1](https://github.com/payara/Payara/releases/tag/payara-server-6.2022.1) being the first version), the days of free-to-use Payara 5.x Platform Community versions [are numbered](https://blog.payara.fish/whats-new-in-the-november-2022-payara-platform-release). Specifically, Payara's blog post says, "Payara Platform Community 5.2022.4 has been released today as the penultimate Payara 5 Community release." + +Given the end of free-to-use Payara 5 versions, we plan to get the Dataverse software working on Payara 6 (#8305), which will require substantial efforts from the IQSS team and community members, as this also means shifting our app to be a [Jakarta EE 10](https://jakarta.ee/release/10/) application (upgrading from EE 8). We are currently working out the details and will share news as soon as we can. Rest assured we will do our best to provide you with a smooth transition. You can follow along in Issue #8305 and related pull requests and you are, of course, very welcome to participate by testing and otherwise contributing, as always. From 1c688f28c97bfeba6c13a5a1ca8fdd71743bfe40 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 4 Nov 2022 15:47:31 -0400 Subject: [PATCH 143/232] misplaced paren disabling direct download --- .../dataverse/api/DownloadInstanceWriter.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 01f627ea23b..2410da04072 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -217,37 +217,37 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] } } } - if (redirect_url_str!=null) { + } + if (redirect_url_str != null) { - logger.fine("Data Access API: redirect url: " + redirect_url_str); - URI redirect_uri; + logger.fine("Data Access API: redirect url: " + redirect_url_str); + URI redirect_uri; - try { - redirect_uri = new URI(redirect_url_str); - } catch (URISyntaxException ex) { - logger.info("Data Access API: failed to create redirect url (" + redirect_url_str + ")"); - redirect_uri = null; - } - if (redirect_uri != null) { - // increment the download count, if necessary: - if (di.getGbr() != null && !(isThumbnailDownload(di) || isPreprocessedMetadataDownload(di))) { - try { - logger.fine("writing guestbook response, for a download redirect."); - Command cmd = new CreateGuestbookResponseCommand(di.getDataverseRequestService().getDataverseRequest(), di.getGbr(), di.getGbr().getDataFile().getOwner()); - di.getCommand().submit(cmd); - MakeDataCountEntry entry = new MakeDataCountEntry(di.getRequestUriInfo(), di.getRequestHttpHeaders(), di.getDataverseRequestService(), di.getGbr().getDataFile()); - mdcLogService.logEntry(entry); - } catch (CommandException e) { - } + try { + redirect_uri = new URI(redirect_url_str); + } catch (URISyntaxException ex) { + logger.info("Data Access API: failed to create redirect url (" + redirect_url_str + ")"); + redirect_uri = null; + } + if (redirect_uri != null) { + // increment the download count, if necessary: + if (di.getGbr() != null && !(isThumbnailDownload(di) || isPreprocessedMetadataDownload(di))) { + try { + logger.fine("writing guestbook response, for a download redirect."); + Command cmd = new CreateGuestbookResponseCommand(di.getDataverseRequestService().getDataverseRequest(), di.getGbr(), di.getGbr().getDataFile().getOwner()); + di.getCommand().submit(cmd); + MakeDataCountEntry entry = new MakeDataCountEntry(di.getRequestUriInfo(), di.getRequestHttpHeaders(), di.getDataverseRequestService(), di.getGbr().getDataFile()); + mdcLogService.logEntry(entry); + } catch (CommandException e) { } - - // finally, issue the redirect: - Response response = Response.seeOther(redirect_uri).build(); - logger.fine("Issuing redirect to the file location."); - throw new RedirectionException(response); } - throw new ServiceUnavailableException(); + + // finally, issue the redirect: + Response response = Response.seeOther(redirect_uri).build(); + logger.fine("Issuing redirect to the file location."); + throw new RedirectionException(response); } + throw new ServiceUnavailableException(); } if (di.getConversionParam() != null) { From e16c8863b17f0d1f142725d82fe659a93fd1707e Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 7 Nov 2022 10:09:59 +0100 Subject: [PATCH 144/232] rename: postLoad -> initialize --- .../iq/dataverse/authorization/AuthenticationServiceBean.java | 2 +- .../iq/dataverse/authorization/users/AuthenticatedUser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index 6c401223cd5..d92ed78681b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -648,7 +648,7 @@ public AuthenticatedUser createAuthenticatedUser(UserRecordIdentifier userRecord actionLogSvc.log( new ActionLogRecord(ActionLogRecord.ActionType.Auth, "createUser") .setInfo(authenticatedUser.getIdentifier())); - authenticatedUser.postLoad(); + authenticatedUser.initialize(); return authenticatedUser; } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index 7299350b774..9fdfce2f1a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -148,7 +148,7 @@ void prePersist() { } @PostLoad - public void postLoad() { + public void initialize() { mutedNotificationsSet = Type.tokenizeToSet(mutedNotifications); mutedEmailsSet = Type.tokenizeToSet(mutedEmails); } From b1d94c86197a1d2ccfd998b5ba47700af9d95c04 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 7 Nov 2022 11:00:57 +0100 Subject: [PATCH 145/232] sortOrder column made not nullable --- src/main/java/edu/harvard/iq/dataverse/license/License.java | 2 +- .../resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/license/License.java b/src/main/java/edu/harvard/iq/dataverse/license/License.java index 0c8465e88e4..3073291a9d5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/license/License.java +++ b/src/main/java/edu/harvard/iq/dataverse/license/License.java @@ -76,7 +76,7 @@ public class License { @Column(nullable = false) private boolean isDefault; - @Column(nullable = true) + @Column(nullable = false) private Long sortOrder; @OneToMany(mappedBy="license") diff --git a/src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql index 43631ebd165..a449c85cf16 100644 --- a/src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql +++ b/src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql @@ -1,5 +1,5 @@ ALTER TABLE license -ADD COLUMN IF NOT EXISTS sortorder BIGINT; +ADD COLUMN IF NOT EXISTS sortorder BIGINT NOT NULL DEFAULT(0); CREATE INDEX IF NOT EXISTS license_sortorder_id ON license (sortorder, id); \ No newline at end of file From 4b0d36596835333bf5c528e3659b8a5bbef5ed60 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 7 Nov 2022 14:10:25 -0500 Subject: [PATCH 146/232] rename getEditVersion to getOrCreateEditVersion #8930 --- .../edu/harvard/iq/dataverse/Dataset.java | 16 ++++++++------ .../edu/harvard/iq/dataverse/DatasetPage.java | 12 +++++----- .../iq/dataverse/EditDatafilesPage.java | 8 +++---- .../edu/harvard/iq/dataverse/FilePage.java | 20 ++++++++--------- .../edu/harvard/iq/dataverse/api/Access.java | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 20 ++++++++--------- .../edu/harvard/iq/dataverse/api/Files.java | 2 +- .../CollectionDepositManagerImpl.java | 2 +- .../api/datadeposit/ContainerManagerImpl.java | 2 +- .../datadeposit/MediaResourceManagerImpl.java | 4 ++-- .../filesystem/FileRecordJobListener.java | 2 +- .../datasetutility/AddReplaceFileHelper.java | 4 ++-- .../command/impl/CreateNewDatasetCommand.java | 2 +- .../CuratePublishedDatasetVersionCommand.java | 12 +++++----- .../impl/GetDraftDatasetVersionCommand.java | 2 +- .../impl/PersistProvFreeFormCommand.java | 2 +- .../command/impl/RestrictFileCommand.java | 2 +- .../impl/ReturnDatasetToAuthorCommand.java | 4 ++-- .../impl/SetCurationStatusCommand.java | 2 +- .../impl/SubmitDatasetForReviewCommand.java | 2 +- .../impl/UpdateDatasetVersionCommand.java | 22 +++++++++---------- .../impl/CreateDatasetVersionCommandTest.java | 4 ++-- .../command/impl/RestrictFileCommandTest.java | 4 ++-- .../iq/dataverse/ingest/IngestUtilTest.java | 10 ++++----- .../iq/dataverse/util/FileUtilTest.java | 2 +- 25 files changed, 83 insertions(+), 81 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index a4f82d41bac..e91221ce36c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -391,19 +391,21 @@ private DatasetVersion createNewDatasetVersion(Template template, FileMetadata f /** * The "edit version" is the most recent *draft* of a dataset, and if the - * latest version of a dataset is published, a new draft will be created. - * + * latest version of a dataset is published, a new draft will be created. If + * you don't want to create a new version, you should be using + * getLatestVersion. + * * @return The edit version {@code this}. */ - public DatasetVersion getEditVersion() { - return getEditVersion(null, null); + public DatasetVersion getOrCreateEditVersion() { + return getOrCreateEditVersion(null, null); } - public DatasetVersion getEditVersion(FileMetadata fm) { - return getEditVersion(null, fm); + public DatasetVersion getOrCreateEditVersion(FileMetadata fm) { + return getOrCreateEditVersion(null, fm); } - public DatasetVersion getEditVersion(Template template, FileMetadata fm) { + public DatasetVersion getOrCreateEditVersion(Template template, FileMetadata fm) { DatasetVersion latestVersion = this.getLatestVersion(); if (!latestVersion.isWorkingCopy() || template != null) { // if the latest version is released or archived, create a new version for editing diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 0a8db69bf5b..6e71f6c5042 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2067,7 +2067,7 @@ private String init(boolean initFull) { } //Initalize with the default if there is one dataset.setTemplate(selectedTemplate); - workingVersion = dataset.getEditVersion(selectedTemplate, null); + workingVersion = dataset.getOrCreateEditVersion(selectedTemplate, null); updateDatasetFieldInputLevels(); } else { workingVersion = dataset.getCreateVersion(licenseServiceBean.getDefault()); @@ -2401,7 +2401,7 @@ private void resetVersionUI() { AuthenticatedUser au = (AuthenticatedUser) session.getUser(); //On create set pre-populated fields - for (DatasetField dsf : dataset.getEditVersion().getDatasetFields()) { + for (DatasetField dsf : dataset.getOrCreateEditVersion().getDatasetFields()) { if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.depositor) && dsf.isEmpty()) { dsf.getDatasetFieldValues().get(0).setValue(au.getLastName() + ", " + au.getFirstName()); } @@ -2458,7 +2458,7 @@ private void refreshSelectedFiles(List filesToRefresh){ } String termsOfAccess = workingVersion.getTermsOfUseAndAccess().getTermsOfAccess(); boolean requestAccess = workingVersion.getTermsOfUseAndAccess().isFileAccessRequest(); - workingVersion = dataset.getEditVersion(); + workingVersion = dataset.getOrCreateEditVersion(); workingVersion.getTermsOfUseAndAccess().setTermsOfAccess(termsOfAccess); workingVersion.getTermsOfUseAndAccess().setFileAccessRequest(requestAccess); List newSelectedFiles = new ArrayList<>(); @@ -2521,7 +2521,7 @@ public void edit(EditMode editMode) { if (this.readOnly) { dataset = datasetService.find(dataset.getId()); } - workingVersion = dataset.getEditVersion(); + workingVersion = dataset.getOrCreateEditVersion(); clone = workingVersion.cloneDatasetVersion(); if (editMode.equals(EditMode.METADATA)) { datasetVersionUI = datasetVersionUI.initDatasetVersionUI(workingVersion, true); @@ -3452,7 +3452,7 @@ private void deleteFiles(List filesToDelete) { if (markedForDelete.getId() != null) { // This FileMetadata has an id, i.e., it exists in the database. // We are going to remove this filemetadata from the version: - dataset.getEditVersion().getFileMetadatas().remove(markedForDelete); + dataset.getOrCreateEditVersion().getFileMetadatas().remove(markedForDelete); // But the actual delete will be handled inside the UpdateDatasetCommand // (called later on). The list "filesToBeDeleted" is passed to the // command as a parameter: @@ -3678,7 +3678,7 @@ public String save() { // have been created in the dataset. dataset = datasetService.find(dataset.getId()); - List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, null, true); + List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getOrCreateEditVersion(), newFiles, null, true); newFiles.clear(); // and another update command: diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 6cf294ffd6d..fc8df8681af 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -539,7 +539,7 @@ public String init() { return permissionsWrapper.notFound(); } - workingVersion = dataset.getEditVersion(); + workingVersion = dataset.getOrCreateEditVersion(); //TODO: review if we we need this check; // as getEditVersion should either return the exisiting draft or create a new one @@ -890,7 +890,7 @@ private void deleteFiles(List filesForDelete) { // ToDo - FileMetadataUtil.removeFileMetadataFromList should handle these two // removes so they could be put after this if clause and the else clause could // be removed. - dataset.getEditVersion().getFileMetadatas().remove(markedForDelete); + dataset.getOrCreateEditVersion().getFileMetadatas().remove(markedForDelete); fileMetadatas.remove(markedForDelete); filesToBeDeleted.add(markedForDelete); @@ -907,7 +907,7 @@ private void deleteFiles(List filesForDelete) { // 1. delete the filemetadata from the local display list: FileMetadataUtil.removeFileMetadataFromList(fileMetadatas, markedForDelete); // 2. delete the filemetadata from the version: - FileMetadataUtil.removeFileMetadataFromList(dataset.getEditVersion().getFileMetadatas(), markedForDelete); + FileMetadataUtil.removeFileMetadataFromList(dataset.getOrCreateEditVersion().getFileMetadatas(), markedForDelete); } if (markedForDelete.getDataFile().getId() == null) { @@ -1201,7 +1201,7 @@ public String save() { */ } - workingVersion = dataset.getEditVersion(); + workingVersion = dataset.getOrCreateEditVersion(); logger.fine("working version id: " + workingVersion.getId()); if (FileEditMode.EDIT == mode && Referrer.FILE == referrer) { diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 7f2c6dfca5c..85eb79d2ddc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -365,7 +365,7 @@ public String saveProvFreeform(String freeformTextInput, DataFile dataFileFromPo file.setProvEntityName(dataFileFromPopup.getProvEntityName()); //passing this value into the file being saved here is pretty hacky. Command cmd; - for (FileMetadata fmw : editDataset.getEditVersion().getFileMetadatas()) { + for (FileMetadata fmw : editDataset.getOrCreateEditVersion().getFileMetadatas()) { if (fmw.getDataFile().equals(this.fileMetadata.getDataFile())) { cmd = new PersistProvFreeFormCommand(dvRequestService.getDataverseRequest(), file, freeformTextInput); commandEngine.submit(cmd); @@ -381,15 +381,15 @@ public String restrictFile(boolean restricted) throws CommandException{ String fileNames = null; editDataset = this.file.getOwner(); if (restricted) { // get values from access popup - editDataset.getEditVersion().getTermsOfUseAndAccess().setTermsOfAccess(termsOfAccess); - editDataset.getEditVersion().getTermsOfUseAndAccess().setFileAccessRequest(fileAccessRequest); + editDataset.getOrCreateEditVersion().getTermsOfUseAndAccess().setTermsOfAccess(termsOfAccess); + editDataset.getOrCreateEditVersion().getTermsOfUseAndAccess().setFileAccessRequest(fileAccessRequest); } //using this method to update the terms for datasets that are out of compliance // with Terms of Access requirement - may get her with a file that is already restricted // we'll allow it try { Command cmd; - for (FileMetadata fmw : editDataset.getEditVersion().getFileMetadatas()) { + for (FileMetadata fmw : editDataset.getOrCreateEditVersion().getFileMetadatas()) { if (fmw.getDataFile().equals(this.fileMetadata.getDataFile())) { fileNames += fmw.getLabel(); cmd = new RestrictFileCommand(fmw.getDataFile(), dvRequestService.getDataverseRequest(), restricted); @@ -424,7 +424,7 @@ public String deleteFile() { FileMetadata markedForDelete = null; - for (FileMetadata fmd : editDataset.getEditVersion().getFileMetadatas()) { + for (FileMetadata fmd : editDataset.getOrCreateEditVersion().getFileMetadatas()) { if (fmd.getDataFile().getId().equals(fileId)) { markedForDelete = fmd; @@ -435,17 +435,17 @@ public String deleteFile() { // the file already exists as part of this dataset // so all we remove is the file from the fileMetadatas (for display) // and let the delete be handled in the command (by adding it to the filesToBeDeleted list - editDataset.getEditVersion().getFileMetadatas().remove(markedForDelete); + editDataset.getOrCreateEditVersion().getFileMetadatas().remove(markedForDelete); filesToBeDeleted.add(markedForDelete); } else { List filesToKeep = new ArrayList<>(); - for (FileMetadata fmo : editDataset.getEditVersion().getFileMetadatas()) { + for (FileMetadata fmo : editDataset.getOrCreateEditVersion().getFileMetadatas()) { if (!fmo.getDataFile().getId().equals(this.getFile().getId())) { filesToKeep.add(fmo); } } - editDataset.getEditVersion().setFileMetadatas(filesToKeep); + editDataset.getOrCreateEditVersion().setFileMetadatas(filesToKeep); } fileDeleteInProgress = true; @@ -612,7 +612,7 @@ public void setTermsMet(boolean termsMet) { public String save() { // Validate - Set constraintViolations = editDataset.getEditVersion().validate(); + Set constraintViolations = editDataset.getOrCreateEditVersion().validate(); if (!constraintViolations.isEmpty()) { //JsfHelper.addFlashMessage(JH.localize("dataset.message.validationError")); fileDeleteInProgress = false; @@ -629,7 +629,7 @@ public String save() { if (!filesToBeDeleted.isEmpty()) { // We want to delete the file (there's always only one file with this page) - editDataset.getEditVersion().getFileMetadatas().remove(filesToBeDeleted.get(0)); + editDataset.getOrCreateEditVersion().getFileMetadatas().remove(filesToBeDeleted.get(0)); deleteFileId = filesToBeDeleted.get(0).getDataFile().getId(); deleteStorageLocation = datafileService.getPhysicalFileToDelete(filesToBeDeleted.get(0).getDataFile()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index abeedf23b59..75aa57a0d2b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1384,7 +1384,7 @@ public Response allowAccessRequest(@PathParam("id") String datasetToAllowAccessI return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.fileAccess.failure.noUser", args)); } - dataset.getEditVersion().getTermsOfUseAndAccess().setFileAccessRequest(allowRequest); + dataset.getOrCreateEditVersion().getTermsOfUseAndAccess().setFileAccessRequest(allowRequest); try { engineSvc.submit(new UpdateDatasetVersionCommand(dataset, dataverseRequest)); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index aff543e643c..59bf81a4b8d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -630,7 +630,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, DatasetVersion managedVersion; if (updateDraft) { - final DatasetVersion editVersion = ds.getEditVersion(); + final DatasetVersion editVersion = ds.getOrCreateEditVersion(); editVersion.setDatasetFields(incomingVersion.getDatasetFields()); editVersion.setTermsOfUseAndAccess(incomingVersion.getTermsOfUseAndAccess()); editVersion.getTermsOfUseAndAccess().setDatasetVersion(editVersion); @@ -639,7 +639,7 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, return error(Status.CONFLICT, BundleUtil.getStringFromBundle("dataset.message.toua.invalid")); } Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); - managedVersion = managedDataset.getEditVersion(); + managedVersion = managedDataset.getOrCreateEditVersion(); } else { boolean hasValidTerms = TermsOfUseAndAccessValidator.isTOUAValid(incomingVersion.getTermsOfUseAndAccess(), null); if (!hasValidTerms) { @@ -698,7 +698,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); - DatasetVersion dsv = ds.getEditVersion(); + DatasetVersion dsv = ds.getOrCreateEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.updateDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, datasetFieldSvc, !replaceTerms, false, licenseSvc); dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv); @@ -709,7 +709,7 @@ public Response updateVersionMetadata(String jsonLDBody, @PathParam("id") String DatasetVersion managedVersion; if (updateDraft) { Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); - managedVersion = managedDataset.getEditVersion(); + managedVersion = managedDataset.getOrCreateEditVersion(); } else { managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); } @@ -731,14 +731,14 @@ public Response deleteMetadata(String jsonLDBody, @PathParam("id") String id) { try { Dataset ds = findDatasetOrDie(id); DataverseRequest req = createDataverseRequest(findUserOrDie()); - DatasetVersion dsv = ds.getEditVersion(); + DatasetVersion dsv = ds.getOrCreateEditVersion(); boolean updateDraft = ds.getLatestVersion().isDraft(); dsv = JSONLDUtil.deleteDatasetVersionMDFromJsonLD(dsv, jsonLDBody, metadataBlockService, licenseSvc); dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv); DatasetVersion managedVersion; if (updateDraft) { Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); - managedVersion = managedDataset.getEditVersion(); + managedVersion = managedDataset.getOrCreateEditVersion(); } else { managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); } @@ -769,7 +769,7 @@ private Response processDatasetFieldDataDelete(String jsonBody, String id, Datav Dataset ds = findDatasetOrDie(id); JsonObject json = Json.createReader(rdr).readObject(); - DatasetVersion dsv = ds.getEditVersion(); + DatasetVersion dsv = ds.getOrCreateEditVersion(); dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv); List fields = new LinkedList<>(); DatasetField singleField = null; @@ -882,7 +882,7 @@ private Response processDatasetFieldDataDelete(String jsonBody, String id, Datav boolean updateDraft = ds.getLatestVersion().isDraft(); DatasetVersion managedVersion = updateDraft - ? execCommand(new UpdateDatasetVersionCommand(ds, req)).getEditVersion() + ? execCommand(new UpdateDatasetVersionCommand(ds, req)).getOrCreateEditVersion() : execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); return ok(json(managedVersion)); @@ -932,7 +932,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque Dataset ds = findDatasetOrDie(id); JsonObject json = Json.createReader(rdr).readObject(); - DatasetVersion dsv = ds.getEditVersion(); + DatasetVersion dsv = ds.getOrCreateEditVersion(); dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv); List fields = new LinkedList<>(); DatasetField singleField = null; @@ -1037,7 +1037,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque DatasetVersion managedVersion; if (updateDraft) { - managedVersion = execCommand(new UpdateDatasetVersionCommand(ds, req)).getEditVersion(); + managedVersion = execCommand(new UpdateDatasetVersionCommand(ds, req)).getOrCreateEditVersion(); } else { managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, dsv)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 9dc0c3be524..4cf27064290 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -388,7 +388,7 @@ public Response updateFileMetadata(@FormDataParam("jsonData") String jsonData, } try { - DatasetVersion editVersion = df.getOwner().getEditVersion(); + DatasetVersion editVersion = df.getOwner().getOrCreateEditVersion(); //We get the new fileMetadata from the new version //This is because after generating the draft with getEditVersion, diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java index b6d75276ae1..6543d771ebe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionDepositManagerImpl.java @@ -110,7 +110,7 @@ public DepositReceipt createNew(String collectionUri, Deposit deposit, AuthCrede throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "user " + user.getDisplayInfo().getTitle() + " is not authorized to create a dataset in this dataverse."); } - DatasetVersion newDatasetVersion = dataset.getEditVersion(); + DatasetVersion newDatasetVersion = dataset.getOrCreateEditVersion(); String foreignFormat = SwordUtil.DCTERMS; try { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ContainerManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ContainerManagerImpl.java index dc178a9a740..8fb55a8eaf6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ContainerManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ContainerManagerImpl.java @@ -137,7 +137,7 @@ public DepositReceipt replaceMetadata(String uri, Deposit deposit, AuthCredentia if (!permissionService.isUserAllowedOn(user, updateDatasetCommand, dataset)) { throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "User " + user.getDisplayInfo().getTitle() + " is not authorized to modify dataverse " + dvThatOwnsDataset.getAlias()); } - DatasetVersion datasetVersion = dataset.getEditVersion(); + DatasetVersion datasetVersion = dataset.getOrCreateEditVersion(); // erase all metadata before creating populating dataset version List emptyDatasetFields = new ArrayList<>(); datasetVersion.setDatasetFields(emptyDatasetFields); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index 928ffd4a129..5491024c73c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -250,7 +250,7 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au // Make sure that the upload type is not rsync - handled above for dual mode // ------------------------------------- - if (dataset.getEditVersion().isHasPackageFile()) { + if (dataset.getOrCreateEditVersion().isHasPackageFile()) { throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, BundleUtil.getStringFromBundle("file.api.alreadyHasPackageFile")); } @@ -276,7 +276,7 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au } String uploadedZipFilename = deposit.getFilename(); - DatasetVersion editVersion = dataset.getEditVersion(); + DatasetVersion editVersion = dataset.getOrCreateEditVersion(); if (deposit.getInputStream() == null) { throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Deposit input stream was null."); diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java index 6b82a665c17..3ae8ce9b883 100644 --- a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java +++ b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java @@ -190,7 +190,7 @@ public void beforeJob() throws Exception { // if mode = REPLACE, remove all filemetadata from the dataset version and start fresh if (mode.equalsIgnoreCase(ImportMode.REPLACE.name())) { try { - DatasetVersion workingVersion = dataset.getEditVersion(); + DatasetVersion workingVersion = dataset.getOrCreateEditVersion(); List fileMetadataList = workingVersion.getFileMetadatas(); jobLogger.log(Level.INFO, "Removing any existing file metadata since mode = REPLACE"); for (FileMetadata fmd : fileMetadataList) { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 8e7922fd83b..febbb249a91 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -1200,7 +1200,7 @@ private boolean step_030_createNewFilesViaIngest(){ } // Load the working version of the Dataset - workingVersion = dataset.getEditVersion(); + workingVersion = dataset.getOrCreateEditVersion(); clone = workingVersion.cloneDatasetVersion(); try { CreateDataFileResult result = FileUtil.createDataFiles(workingVersion, @@ -1805,7 +1805,7 @@ private void setNewlyAddedFiles(List datafiles){ newlyAddedFileMetadatas = new ArrayList<>(); // Loop of uglinesss...but expect 1 to 4 files in final file list - List latestFileMetadatas = dataset.getEditVersion().getFileMetadatas(); + List latestFileMetadatas = dataset.getOrCreateEditVersion().getFileMetadatas(); for (DataFile newlyAddedFile : finalFileList){ diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java index 534e07feaae..1efaf14c755 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java @@ -81,7 +81,7 @@ protected void additionalParameterTests(CommandContext ctxt) throws CommandExcep @Override protected DatasetVersion getVersionToPersist( Dataset theDataset ) { - return theDataset.getEditVersion(); + return theDataset.getOrCreateEditVersion(); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index 772b6205b02..66ba00bcf55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -56,7 +56,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { DatasetVersion updateVersion = getDataset().getLatestVersionForCopy(); // Copy metadata from draft version to latest published version - updateVersion.setDatasetFields(getDataset().getEditVersion().initDatasetFields()); + updateVersion.setDatasetFields(getDataset().getOrCreateEditVersion().initDatasetFields()); validateOrDie(updateVersion, isValidateLenient()); @@ -68,14 +68,14 @@ public Dataset execute(CommandContext ctxt) throws CommandException { TermsOfUseAndAccess oldTerms = updateVersion.getTermsOfUseAndAccess(); - TermsOfUseAndAccess newTerms = getDataset().getEditVersion().getTermsOfUseAndAccess(); + TermsOfUseAndAccess newTerms = getDataset().getOrCreateEditVersion().getTermsOfUseAndAccess(); newTerms.setDatasetVersion(updateVersion); updateVersion.setTermsOfUseAndAccess(newTerms); //Put old terms on version that will be deleted.... - getDataset().getEditVersion().setTermsOfUseAndAccess(oldTerms); + getDataset().getOrCreateEditVersion().setTermsOfUseAndAccess(oldTerms); //Also set the fileaccessrequest boolean on the dataset to match the new terms getDataset().setFileAccessRequest(updateVersion.getTermsOfUseAndAccess().isFileAccessRequest()); - List newComments = getDataset().getEditVersion().getWorkflowComments(); + List newComments = getDataset().getOrCreateEditVersion().getWorkflowComments(); if (newComments!=null && newComments.size() >0) { for(WorkflowComment wfc: newComments) { wfc.setDatasetVersion(updateVersion); @@ -91,7 +91,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Look for file metadata changes and update published metadata if needed List pubFmds = updateVersion.getFileMetadatas(); int pubFileCount = pubFmds.size(); - int newFileCount = tempDataset.getEditVersion().getFileMetadatas().size(); + int newFileCount = tempDataset.getOrCreateEditVersion().getFileMetadatas().size(); /* The policy for this command is that it should only be used when the change is a 'minor update' with no file changes. * Nominally we could call .isMinorUpdate() for that but we're making the same checks as we go through the update here. */ @@ -131,7 +131,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.em().remove(mergedFmd); // including removing metadata from the list on the datafile draftFmd.getDataFile().getFileMetadatas().remove(draftFmd); - tempDataset.getEditVersion().getFileMetadatas().remove(draftFmd); + tempDataset.getOrCreateEditVersion().getFileMetadatas().remove(draftFmd); // And any references in the list held by categories for (DataFileCategory cat : tempDataset.getCategories()) { cat.getFileMetadatas().remove(draftFmd); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftDatasetVersionCommand.java index 88b5a75ea22..7e32b19e576 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDraftDatasetVersionCommand.java @@ -24,7 +24,7 @@ public GetDraftDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffect @Override public DatasetVersion execute(CommandContext ctxt) throws CommandException { - return ds.getEditVersion(); + return ds.getOrCreateEditVersion(); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PersistProvFreeFormCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PersistProvFreeFormCommand.java index aa06967675f..a258c36d6ea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PersistProvFreeFormCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PersistProvFreeFormCommand.java @@ -36,7 +36,7 @@ public DataFile execute(CommandContext ctxt) throws CommandException { } else { Dataset dataset = dataFile.getOwner(); - DatasetVersion workingVersion = dataset.getEditVersion(); + DatasetVersion workingVersion = dataset.getOrCreateEditVersion(); if (workingVersion.isDraft()) { if (dataset.isReleased()){ diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommand.java index 16fa40cd8a7..38cbeaf3d66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommand.java @@ -63,7 +63,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { } else { Dataset dataset = file.getOwner(); - DatasetVersion workingVersion = dataset.getEditVersion(); + DatasetVersion workingVersion = dataset.getOrCreateEditVersion(); // We need the FileMetadata for the file in the draft dataset version and the // file we have may still reference the fmd from the prior released version FileMetadata draftFmd = file.getFileMetadata(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java index 169f6d790d3..ba0348f57d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ReturnDatasetToAuthorCommand.java @@ -37,11 +37,11 @@ public Dataset execute(CommandContext ctxt) throws CommandException { throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.reject.datasetNotInReview"), this); } - dataset.getEditVersion().setLastUpdateTime(getTimestamp()); + dataset.getOrCreateEditVersion().setLastUpdateTime(getTimestamp()); dataset.setModificationTime(getTimestamp()); ctxt.engine().submit( new RemoveLockCommand(getRequest(), getDataset(), DatasetLock.Reason.InReview) ); - WorkflowComment workflowComment = new WorkflowComment(dataset.getEditVersion(), WorkflowComment.Type.RETURN_TO_AUTHOR, comment, (AuthenticatedUser) this.getUser()); + WorkflowComment workflowComment = new WorkflowComment(dataset.getOrCreateEditVersion(), WorkflowComment.Type.RETURN_TO_AUTHOR, comment, (AuthenticatedUser) this.getUser()); ctxt.datasets().addWorkflowComment(workflowComment); updateDatasetUser(ctxt); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCurationStatusCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCurationStatusCommand.java index c3a62a35bb3..72f0ef335fb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCurationStatusCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCurationStatusCommand.java @@ -77,7 +77,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { public Dataset save(CommandContext ctxt) throws CommandException { - getDataset().getEditVersion().setLastUpdateTime(getTimestamp()); + getDataset().getOrCreateEditVersion().setLastUpdateTime(getTimestamp()); getDataset().setModificationTime(getTimestamp()); Dataset savedDataset = ctxt.em().merge(getDataset()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommand.java index e38f5bae8e0..130030798ab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommand.java @@ -51,7 +51,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { private Dataset save(CommandContext ctxt) throws CommandException { - getDataset().getEditVersion().setLastUpdateTime(getTimestamp()); + getDataset().getOrCreateEditVersion().setLastUpdateTime(getTimestamp()); getDataset().setModificationTime(getTimestamp()); Dataset savedDataset = ctxt.em().merge(getDataset()); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 227c54c598f..33f64f23076 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -64,7 +64,7 @@ public UpdateDatasetVersionCommand(Dataset theDataset, DataverseRequest aRequest this.filesToDelete = new ArrayList<>(); this.clone = null; this.fmVarMet = null; - for (FileMetadata fmd : theDataset.getEditVersion().getFileMetadatas()) { + for (FileMetadata fmd : theDataset.getOrCreateEditVersion().getFileMetadatas()) { if (fmd.getDataFile().equals(fileToDelete)) { filesToDelete.add(fmd); break; @@ -114,10 +114,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { logger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", getDataset().getId()); } - getDataset().getEditVersion(fmVarMet).setDatasetFields(getDataset().getEditVersion(fmVarMet).initDatasetFields()); - validateOrDie(getDataset().getEditVersion(fmVarMet), isValidateLenient()); + getDataset().getOrCreateEditVersion(fmVarMet).setDatasetFields(getDataset().getOrCreateEditVersion(fmVarMet).initDatasetFields()); + validateOrDie(getDataset().getOrCreateEditVersion(fmVarMet), isValidateLenient()); - final DatasetVersion editVersion = getDataset().getEditVersion(fmVarMet); + final DatasetVersion editVersion = getDataset().getOrCreateEditVersion(fmVarMet); DatasetFieldUtil.tidyUpFields(editVersion.getDatasetFields(), true); @@ -204,10 +204,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // If the datasetversion doesn't match, we have the fmd from a published version // and we need to remove the one for the newly created draft instead, so we find // it here - logger.fine("Edit ver: " + theDataset.getEditVersion().getId()); + logger.fine("Edit ver: " + theDataset.getOrCreateEditVersion().getId()); logger.fine("fmd ver: " + fmd.getDatasetVersion().getId()); - if (!theDataset.getEditVersion().equals(fmd.getDatasetVersion())) { - fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getEditVersion()); + if (!theDataset.getOrCreateEditVersion().equals(fmd.getDatasetVersion())) { + fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getOrCreateEditVersion()); } } fmd = ctxt.em().merge(fmd); @@ -229,21 +229,21 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // In either case, to fully remove the fmd, we have to remove any other possible // references // From the datasetversion - FileMetadataUtil.removeFileMetadataFromList(theDataset.getEditVersion().getFileMetadatas(), fmd); + FileMetadataUtil.removeFileMetadataFromList(theDataset.getOrCreateEditVersion().getFileMetadatas(), fmd); // and from the list associated with each category for (DataFileCategory cat : theDataset.getCategories()) { FileMetadataUtil.removeFileMetadataFromList(cat.getFileMetadatas(), fmd); } } - for(FileMetadata fmd: theDataset.getEditVersion().getFileMetadatas()) { + for(FileMetadata fmd: theDataset.getOrCreateEditVersion().getFileMetadatas()) { logger.fine("FMD: " + fmd.getId() + " for file: " + fmd.getDataFile().getId() + "is in final draft version"); } if (recalculateUNF) { - ctxt.ingest().recalculateDatasetVersionUNF(theDataset.getEditVersion()); + ctxt.ingest().recalculateDatasetVersionUNF(theDataset.getOrCreateEditVersion()); } - theDataset.getEditVersion().setLastUpdateTime(getTimestamp()); + theDataset.getOrCreateEditVersion().setLastUpdateTime(getTimestamp()); theDataset.setModificationTime(getTimestamp()); savedDataset = ctxt.em().merge(theDataset); diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommandTest.java index 30c5048fa8f..dd8901a05dc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDatasetVersionCommandTest.java @@ -33,7 +33,7 @@ public void testSimpleVersionAddition() throws Exception { Dataset ds = makeDataset(); // Populate the Initial version - DatasetVersion dsvInitial = ds.getEditVersion(); + DatasetVersion dsvInitial = ds.getOrCreateEditVersion(); dsvInitial.setCreateTime( dateFmt.parse("20001012") ); dsvInitial.setLastUpdateTime( dsvInitial.getLastUpdateTime() ); dsvInitial.setId( MocksFactory.nextId() ); @@ -62,7 +62,7 @@ public void testSimpleVersionAddition() throws Exception { assertEquals( dsvCreationDate, dsvNew.getLastUpdateTime() ); assertEquals( dsvCreationDate.getTime(), ds.getModificationTime().getTime() ); assertEquals( ds, dsvNew.getDataset() ); - assertEquals( dsvNew, ds.getEditVersion() ); + assertEquals( dsvNew, ds.getOrCreateEditVersion() ); Map> expected = new HashMap<>(); expected.put(ds, Collections.singleton(Permission.AddDataset)); assertEquals(expected, testEngine.getReqiredPermissionsForObjects() ); diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommandTest.java index 1e8b8fb3106..7b663389a3a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/RestrictFileCommandTest.java @@ -108,7 +108,7 @@ public void testRestrictPublishedFile() throws Exception{ //asserts assertTrue(!file.isRestricted()); boolean fileFound = false; - for (FileMetadata fmw : dataset.getEditVersion().getFileMetadatas()) { + for (FileMetadata fmw : dataset.getOrCreateEditVersion().getFileMetadatas()) { if (file.equals(fmw.getDataFile())) { fileFound=true; //If it worked fmw is for the draft version and file.getFileMetadata() is for the published version @@ -193,7 +193,7 @@ public void testUnrestrictPublishedFile() throws Exception{ //asserts assertTrue(file.isRestricted()); boolean fileFound = false; - for (FileMetadata fmw : dataset.getEditVersion().getFileMetadatas()) { + for (FileMetadata fmw : dataset.getOrCreateEditVersion().getFileMetadatas()) { if (file.equals(fmw.getDataFile())) { fileFound = true; assertTrue(!fmw.isRestricted()); diff --git a/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java index 8e4b81ec921..ca68af4090c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/ingest/IngestUtilTest.java @@ -42,7 +42,7 @@ public void testCheckForDuplicateFileNamesNoDirectories() throws Exception { Dataset dataset = makeDataset(); // create dataset version - DatasetVersion datasetVersion = dataset.getEditVersion(); + DatasetVersion datasetVersion = dataset.getOrCreateEditVersion(); datasetVersion.setCreateTime(dateFmt.parse("20001012")); datasetVersion.setLastUpdateTime(datasetVersion.getLastUpdateTime()); datasetVersion.setId(MocksFactory.nextId()); @@ -146,7 +146,7 @@ public void testCheckForDuplicateFileNamesWithEmptyDirectoryLabels() throws Exce Dataset dataset = makeDataset(); // create dataset version - DatasetVersion datasetVersion = dataset.getEditVersion(); + DatasetVersion datasetVersion = dataset.getOrCreateEditVersion(); datasetVersion.setCreateTime(dateFmt.parse("20001012")); datasetVersion.setLastUpdateTime(datasetVersion.getLastUpdateTime()); datasetVersion.setId(MocksFactory.nextId()); @@ -251,7 +251,7 @@ public void testCheckForDuplicateFileNamesWithDirectories() throws Exception { Dataset dataset = makeDataset(); // create dataset version - DatasetVersion datasetVersion = dataset.getEditVersion(); + DatasetVersion datasetVersion = dataset.getOrCreateEditVersion(); datasetVersion.setCreateTime(dateFmt.parse("20001012")); datasetVersion.setLastUpdateTime(datasetVersion.getLastUpdateTime()); datasetVersion.setId(MocksFactory.nextId()); @@ -389,7 +389,7 @@ public void testCheckForDuplicateFileNamesTabular() throws Exception { Dataset dataset = makeDataset(); // create dataset version - DatasetVersion datasetVersion = dataset.getEditVersion(); + DatasetVersion datasetVersion = dataset.getOrCreateEditVersion(); datasetVersion.setCreateTime(dateFmt.parse("20001012")); datasetVersion.setLastUpdateTime(datasetVersion.getLastUpdateTime()); datasetVersion.setId(MocksFactory.nextId()); @@ -475,7 +475,7 @@ public void testCheckForDuplicateFileNamesWhenReplacing() throws Exception { Dataset dataset = makeDataset(); // create dataset version - DatasetVersion datasetVersion = dataset.getEditVersion(); + DatasetVersion datasetVersion = dataset.getOrCreateEditVersion(); datasetVersion.setCreateTime(dateFmt.parse("20001012")); datasetVersion.setLastUpdateTime(datasetVersion.getLastUpdateTime()); datasetVersion.setId(MocksFactory.nextId()); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index 01fb8aad6cf..f3d9d5eda46 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -128,7 +128,7 @@ public void testIsDownloadPopupRequiredNull() { @Test public void testIsDownloadPopupRequiredDraft() { Dataset dataset = new Dataset(); - DatasetVersion dsv1 = dataset.getEditVersion(); + DatasetVersion dsv1 = dataset.getOrCreateEditVersion(); assertEquals(DatasetVersion.VersionState.DRAFT, dsv1.getVersionState()); assertEquals(false, FileUtil.isDownloadPopupRequired(dsv1)); } From 81254adc4b3a2d0f9584b083143c2a4ec17fe980 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 8 Nov 2022 15:39:03 +0100 Subject: [PATCH 147/232] bugfix in test: licenseId was not parsed correctly --- src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java index 50d3c5b34ea..30fc603a998 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java @@ -92,7 +92,7 @@ public void testLicenses(){ body = getLicensesResponse.getBody().asString(); status = JsonPath.from(body).getString("status"); //Last added licens; with the highest id - long licenseId = JsonPath.from(body).getList("data[*].id").stream().max((x, y) -> Long.compare(x, y)).get(); + long licenseId = (long) JsonPath.from(body).getList("data.id").stream().max((x, y) -> Integer.compare(x, y)).get(); //Assumes the first license is active, which should be true on a test server long activeLicenseId = JsonPath.from(body).getLong("data[0].id"); assertEquals("OK", status); From 35540f95834f1b59f06635a2f5ff33e6dbce5fd2 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 8 Nov 2022 15:50:43 +0100 Subject: [PATCH 148/232] typo fix --- src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java index 30fc603a998..d6bfdb96777 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/LicensesIT.java @@ -91,7 +91,7 @@ public void testLicenses(){ getLicensesResponse.prettyPrint(); body = getLicensesResponse.getBody().asString(); status = JsonPath.from(body).getString("status"); - //Last added licens; with the highest id + //Last added license; with the highest id long licenseId = (long) JsonPath.from(body).getList("data.id").stream().max((x, y) -> Integer.compare(x, y)).get(); //Assumes the first license is active, which should be true on a test server long activeLicenseId = JsonPath.from(body).getLong("data[0].id"); From 7dce7d72a8a70eea7279f540c32fbb84bb3a0319 Mon Sep 17 00:00:00 2001 From: chenganj Date: Tue, 8 Nov 2022 12:39:46 -0500 Subject: [PATCH 149/232] refactored --- .../iq/dataverse/dataset/DatasetUtil.java | 18 ++++++++++++------ src/main/webapp/dataset-license-terms.xhtml | 6 +++--- .../webapp/datasetLicenseInfoFragment.xhtml | 4 ++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index fecfdc2bcfb..f1785a42098 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -39,6 +39,7 @@ import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.EnumUtils; public class DatasetUtil { @@ -547,7 +548,7 @@ public static License getLicense(DatasetVersion dsv) { public static String getLicenseName(DatasetVersion dsv) { License license = DatasetUtil.getLicense(dsv); - return license != null ? getLocalizedLicenseDetails(license.getName(),".name") + return license != null ? getLocalizedLicenseDetails(license,"NAME") : BundleUtil.getStringFromBundle("license.custom"); } @@ -573,15 +574,21 @@ public static String getLicenseIcon(DatasetVersion dsv) { public static String getLicenseDescription(DatasetVersion dsv) { License license = DatasetUtil.getLicense(dsv); - return license != null ? getLocalizedLicenseDetails(license.getName(),".description") : BundleUtil.getStringFromBundle("license.custom.description"); + return license != null ? getLocalizedLicenseDetails(license,"DESCRIPTION") : BundleUtil.getStringFromBundle("license.custom.description"); } - public static String getLocalizedLicenseDetails(String licenseName,String keyPart) { - String key = "license." + licenseName.toLowerCase().replace(" ", "_") + keyPart; + public enum LicenseOption { + NAME, DESCRIPTION + }; + public static String getLocalizedLicenseDetails(License license,String keyPart) { + String licenseName = license.getName(); String localizedLicenseValue = "" ; try { - localizedLicenseValue = BundleUtil.getStringFromPropertyFile(key, "License"); + if (EnumUtils.isValidEnum(LicenseOption.class, keyPart ) ){ + String key = "license." + licenseName.toLowerCase().replace(" ", "_") + "." + keyPart.toLowerCase(); + localizedLicenseValue = BundleUtil.getStringFromPropertyFile(key, "License"); + } } catch (Exception e) { localizedLicenseValue = licenseName; @@ -591,7 +598,6 @@ public static String getLocalizedLicenseDetails(String licenseName,String keyPar localizedLicenseValue = licenseName ; } return localizedLicenseValue; - } public static String getLocaleExternalStatus(String status) { diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 760f39d7170..8b5c86b9c1c 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -46,7 +46,7 @@

+ var="license" itemLabel="#{DatasetUtil:getLocalizedLicenseDetails(license, 'NAME')}" itemValue="#{license}"/> @@ -55,8 +55,8 @@

- - #{DatasetUtil:getLocalizedLicenseDetails(termsOfUseAndAccess.license.name,'.name')} + + #{DatasetUtil:getLocalizedLicenseDetails(termsOfUseAndAccess.license,'NAME')}

diff --git a/src/main/webapp/datasetLicenseInfoFragment.xhtml b/src/main/webapp/datasetLicenseInfoFragment.xhtml index e7a393a8ae7..257f6b3b12f 100644 --- a/src/main/webapp/datasetLicenseInfoFragment.xhtml +++ b/src/main/webapp/datasetLicenseInfoFragment.xhtml @@ -30,12 +30,12 @@ xmlns:jsf="http://xmlns.jcp.org/jsf">
+ jsf:rendered="#{!empty DatasetUtil:getLocalizedLicenseDetails(DatasetPage.workingVersion.termsOfUseAndAccess.license,'DESCRIPTION')} }">
- +
From c5bc60755f22955fcfa935085123d81d438f4423 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 8 Nov 2022 14:58:53 -0500 Subject: [PATCH 150/232] detect NetCDF and HDF5 files based on content #9117 --- doc/release-notes/9117-file-type-detection.md | 1 + modules/dataverse-parent/pom.xml | 5 +++ pom.xml | 8 +++- .../harvard/iq/dataverse/util/FileUtil.java | 33 +++++++++++++++++ .../iq/dataverse/util/FileUtilTest.java | 35 ++++++++++++++++++ src/test/resources/hdf/hdf4/hdf4test | Bin 0 -> 30275 bytes src/test/resources/hdf/hdf5/vlen_string_dset | Bin 0 -> 6304 bytes src/test/resources/netcdf/madis-raob.nc | Bin 0 -> 150612 bytes 8 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/9117-file-type-detection.md create mode 100644 src/test/resources/hdf/hdf4/hdf4test create mode 100644 src/test/resources/hdf/hdf5/vlen_string_dset create mode 100644 src/test/resources/netcdf/madis-raob.nc diff --git a/doc/release-notes/9117-file-type-detection.md b/doc/release-notes/9117-file-type-detection.md new file mode 100644 index 00000000000..7901b478acc --- /dev/null +++ b/doc/release-notes/9117-file-type-detection.md @@ -0,0 +1 @@ +NetCDF and HDF5 files are now detected based on their content rather than just their file extension. diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index c1ba693da1b..e36a78b11be 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -299,6 +299,11 @@ true + + unidata-all + Unidata All + https://artifacts.unidata.ucar.edu/repository/unidata-all/ + dvn.private Local repository for hosting jars not available from network repositories. diff --git a/pom.xml b/pom.xml index c6459cfc55c..8b6f98c5896 100644 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,7 @@ 0.8.7 5.2.1 2.4.1 + 5.5.3 org.junit.jupiter diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 339de904f9e..dc4f8b97f9a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -108,6 +108,8 @@ import java.util.Arrays; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; /** * a 4.0 implementation of the DVN FileUtil; @@ -467,6 +469,11 @@ public static String determineFileType(File f, String fileName) throws IOExcepti fileType = "application/fits"; } } + + // step 3: Check if NetCDF or HDF5 + if (fileType == null) { + fileType = checkNetcdfOrHdf5(f); + } // step 3: check the mime type of this file with Jhove if (fileType == null){ @@ -669,6 +676,32 @@ private static boolean isGraphMLFile(File file) { return isGraphML; } + public static String checkNetcdfOrHdf5(File file) { + try ( NetcdfFile netcdfFile = NetcdfFiles.open(file.getAbsolutePath())) { + if (netcdfFile == null) { + // Can't open as a NetCDF or HDF5 file. + return null; + } + String type = netcdfFile.getFileTypeId(); + if (type == null) { + return null; + } + switch (type) { + case "NETCDF": + return "application/netcdf"; + case "NetCDF-4": + return "application/netcdf"; + case "HDF5": + return "application/x-hdf5"; + default: + break; + } + } catch (IOException ex) { + return null; + } + return null; + } + // from MD5Checksum.java public static String calculateChecksum(String datafile, ChecksumType checksumType) { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index 01fb8aad6cf..e710236e446 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -373,4 +373,39 @@ public void testIsThumbnailSupported() throws Exception { assertFalse(FileUtil.isThumbnailSupported(filewBogusContentType)); } } + + @Test + public void testNetcdfFile() throws IOException { + // We got madis-raob.nc from https://www.unidata.ucar.edu/software/netcdf/examples/files.html + String path = "src/test/resources/netcdf/"; + String pathAndFile = path + "madis-raob.nc"; + File file = new File(pathAndFile); + String contentType = FileUtil.determineFileType(file, pathAndFile); + assertEquals("application/netcdf", contentType); + } + + @Test + public void testHdf5File() throws IOException { + // We got vlen_string_dset.h5 from https://github.com/h5py/h5py/blob/3.7.0/h5py/tests/data_files/vlen_string_dset.h5 + // and named in "vlen_string_dset" with no file extension for this test. + String path = "src/test/resources/hdf/hdf5/"; + String pathAndFile = path + "vlen_string_dset"; + File file = new File(pathAndFile); + String contentType = FileUtil.determineFileType(file, pathAndFile); + assertEquals("application/x-hdf5", contentType); + } + + @Test + public void testHdf4File() throws IOException { + // We got test.hdf from https://people.sc.fsu.edu/~jburkardt/data/hdf/hdf.html + // and named in "hdf4test" with no file extension for this test. + // HDF4 is the old format, the previous generation before HDF5. + // We can't detect it based on its content. + String path = "src/test/resources/hdf/hdf4/"; + String pathAndFile = path + "hdf4test"; + File file = new File(pathAndFile); + String contentType = FileUtil.determineFileType(file, pathAndFile); + assertEquals("application/octet-stream", contentType); + } + } diff --git a/src/test/resources/hdf/hdf4/hdf4test b/src/test/resources/hdf/hdf4/hdf4test new file mode 100644 index 0000000000000000000000000000000000000000..4674fdde19487d5c28b44f54562fbe5118284e0d GIT binary patch literal 30275 zcmeI532+ou8pprf0b@it1YvblR8$N}IE3IzGLs;|5Rn8#K_Mgq2@DxXCJ;a|;g(C} zlFNX|jUXx_9$>r>FT@*9sC8@C+TGf+T+3p2E!p4K(;0HiWCsbTq-W}v|35FUyI+6r z&Fh}mmG@%9nuo|i(buICBGP0uZfMkYnIuy19zHy3o8dpicZ=E~lSM{1!Q03)1%EBR z9eLEJ;X9M(UVN#MzFqFaFE#RSGyHBd&s5U?YUa6LWK0-dkEK4!$g^F_@JaY+($(AX z-8gPJUiYu*>iL_IJ`KOeOjp0(OrMVb!Aw{GlbJq4#FdC|%YM}N({vdvGx6k;@X&pR zpN(%7dRRh9e*_=HInKgw#ivsq{_gr-^RplIKjJ%+K1XD%#z)JkJ^`=ge*izqOjqwV z)92zhn(69ynCbKI-`&wkW@hS%~h5^=}iwfyQ^;I;e@;rp8D>IazVi}9SNkzYOA82Ojr518rd z51Hw-i*e!jmh4}B8#8?w{x-apM|~E)JLk0=uVvQrQcqcp^I{+4G;Ufi596Q2_oJLE z@IT`t*bja@=Wm?XN_;oGo|pQY@p@jX@E$W=J?+>?UyXmtOjrNBnZ5@9otduwxS38H zDT>2)VE^g|X*%Vvz*A2~`PFNDE~Q=~@H_EZ{5t*}nd$0ZH`6!Z|7)h>JsKa)zY*U9ujN;tW2SGy6JKNh z>M5&HFPrh7;WAa?ytd(KAJUz4^=}*L zJ7hav>oF~Kza&xR7vYad3+X9xaWD1u{pwp^hHpmtP54eCA))v`inJjn-332h7>&=zv=(v1A-GtK-Z_2l^2J8g*_ZJF)~&Z4m+oJF>T9z8l&br|)kUMJ{c=!in3wRT5n()h z7v)Kl)Pk{ta$NCo@kVAXj)c(HM)`6?4J1pAzZ*H(m(e8}rPqCHPfqX7E6VQ?!wO1E z{TwdLI9wAk{v3H~hl(-|7aibmlP38&ocHX`<8b`B_Tk)xPWpOY-k60OF$?#_EMj10 zdUlpXrFc9=)dNyFtkT%Z_z@z}8fJc04|g_P6|Yx_5e7MrvkC zzq%(A(jq!UFSEugio2uc)!|N}xttfZgZH+RZy6oBkNb*fX)5PStX#m={zAEkoXy!& zxE7Hk=hUdLUs}yaPgI&^)2cnyY7Tm9f1$1nV(WUUIq0qZ9Xl)EUF4}fL8- zP&M1C*~yxXrdHclhnj6$PtC?iOKBw+(}%T@OUT)lJ}1H`RRdLR*vm&xobXldDOYVu z&|CWpZ!pIWT(!X*JMbqs!(BMamhS(v+tR4vmeg>|U}~5Eo-jTg{;xMphxIOSxwz7l zIQ@L?Tid*?LpyYB9qp?_>u9gv7ixa)xkh)k4adAz zu9NG@-H|GdFp65I1>{w40+B{}^|mF_SKa_kpwV^`sqPZ4=x3@4RK7a9jPQG3{=44# z`;N%ux=pAnO{fMmp{{bi(TCJ%LcU%^?;m0$PC7}vBuJukmM(13RhxzP{OhzMkF(el zmzy`zD7m-b;l^EG^vRZ#ot~YMR=2-P&r7ngrFRbmEv)nkTK@r}xdx zlX1Hg!|+5qxk_$eAnHbT7418tNJnb#>NmLBx1WudgxIH?jpNilhS@Tz-Bnh#i_rZ> zyYP2?NeXa<(Msx7CX$RM^CMaPI;ezWA}IpRzy^HlN=b(PFdW7~8O(u2umaY>7T5)Q zp%MKy9EbnFPw=ydLx48u2!${R2X5jVkwC+AL_-s33gZ0W^aP z;UZ`bEr8oxM=Q7(T0=gV<3^nzqa0Xq;3M=GR2Z@2~eKsxk= zTOkAbK_=V={o!^P09lX?17Q%{0fXUAxC@5BP`DfJfnhKla$p4HLLN9_B#eT5D1gy0 z23#-}3c(HIU_2B7x8e>Dl)waFh}=;MlVCDTfqUUTma#qFd2z=r(jax*dHSeH`6^ z?m%~>Qb8+`_S2Hk`1L7zpRMfak6(dW?T(C5+T z(HGDc&==7c(S7JXbU(TueF=RDtwbx)m(iEeSI}3`SJ79|*U;C{*U{I}1Ly(t4fGB4 zAbJpe6MYkX3w;ZH8+{vn2Ym;97kw8!gdRfQL*GLWqleM=(f830&=1fL(GSs&(2vlM z(T~wj&`;1$(NEDM=n?cY^fUBx^mFtJ^b7P$^h@+B^egmh^lS7edbCWemlXjk0#*d9 z2v`xYB490ts0V@Jl1gr>H5wId)MZk)H6#**(Rs^gF zSP`%yU`4=+fE58N0#*d92v`xYB492`H9)W111pX&d z_@*14R1hZFg@XgW2Z8M*v5qo}Isz`V`K zDI;Z+Rgmjt?-oWS%Y0nd2$Xx8>*L*~(oy!Cqdub3#tBC#T%QOqrhi za;GiVT~b)?DVh`~Qf|H)ZQ#$b6E6#?`8Gak@y2Z6`cW(4bUiZR96RY#JDvJIYW}`5 zcmmD#Su@i5o8(kaJGEk2xvdCT5%@htpla^OhSub!-Y7KmerxZqD%Q2r8_Vx{Z49Yo zMHUT2nUwN#qS{9Miaj|=XYD!8`KBEB%gw}q^OO?I(|Ml2EcxbA0cT|)fGKC|d4hM{ zf%_Wrc|uq)lkwLx&^)a2WW)DA6Rx?8fWJwigE@9R1I1y z;a-`_0(Og7pl-1&VM)1VY`2`Z^?X5_%IUY1U*Y*e1r_guhM>pS%&T)tJ_?)Z%)UQry6jh@04UGw*YB!2rFXn#u~N3t3Hbt!!b9MivnWxM0Bl E0NB#lt^fc4 literal 0 HcmV?d00001 diff --git a/src/test/resources/hdf/hdf5/vlen_string_dset b/src/test/resources/hdf/hdf5/vlen_string_dset new file mode 100644 index 0000000000000000000000000000000000000000..dd20547f8e9a5d0597c76763b2618131c96033cd GIT binary patch literal 6304 zcmeHI%}N7749@JfI#k4;lX|KzAo>Q1ySfLl#rEK_B2{~_uv@8*pf~Xedh}6zD?6Da zW$`FT1-}#~nIw~){rL9lYmuF;xAt1Z<~=oNGlsT&sm*pIV|%L)G;r!))PE!rIN~3X z9;Ypx|Glq&mFE;XZN7OOM~4lGyd~iO6#_+m(&ZPv*0(m6ek#uljI4JQL*rv%g&%rH zE0MplFhhmO&dC#$mf^b*OGBM2yZaQG&i9l~VQ!Or+$V{oz4oVWpwvE=LAq{-em0)5 z_I}qVdXD>LExpT5#$U8vmfGpEZi43m0U+?#5m*fuoT*r_M%MQ%9nD&uHLhM~X+J;h zTtxj;!51x+bD>PtbU2&@Z+`=Po~;^}<#>8PCQp_hRqg8O_Ff00e*l5C8%|00{gt0&|>?K0E*b literal 0 HcmV?d00001 diff --git a/src/test/resources/netcdf/madis-raob.nc b/src/test/resources/netcdf/madis-raob.nc new file mode 100644 index 0000000000000000000000000000000000000000..d0cae0d077d12c7dd1279f63274997847c5a627a GIT binary patch literal 150612 zcmeHQZEzjcd0yGrl5A{jQf%{;ED*o}$w(H)HU%X4%CcnqA?vy}rD@DccO`AOU%4L^ zqBLkGGy};@!%RCAoRYYZ8JG#3Ap;Dgp+x~^LOV^GeoRZ!q;;pA3_tk6q(7Qwn7Yq< z_U_f$bI-l^thkb{#e2u=-Mf3v+4nr>Iq!4M*>|sE1A{G=Wzn_{+r`+{=G^fk#T1VG zPg}}KA97E-NqpUctv&Be9uH37`xB*eC#7?TaIQ6VGMy+DbP>L{rHQWvr`>cRlbhM+rI8XJ8ynR%hOKQplS<)w7min9yJ67Fz_bM)bX^N%X=A70m{2=Faz`)WRK6}3vMcJA9BX*T|21k*6~Dgz$v(t zkJ7g7?80YPS2vE=dx;ZH&U40+t}u|Ux0dk1p|zplrQA>ozl_%WO!_!3QJQs0r*J5f zMk>FKRK^CBamXoj(9X9K=_b6SJL2ZNOoFmm)D~#}jhnFFK-c!C6GtZ|@@|2y&R@R- z#}^L_oJwZWlc&;X41A}1bR3J>K+3TbZhk!HWr3NU$|T(6?Y50e`C=;N0L9ZzvM4G? zW#rrmH|M6ugEDUIKRG-Sw~xEyQ)%xDMc0mJ-EnWi8%OhI()Nfme!ny6+S`xEM?Pck z+0(hRyK|KI(vGsq1()pk#JwG{j(Eq3j(a+ebsXrhVa`5WvepG@Pu)vjh3y-TWXMJ* zn@xJI4~LYF*x{$!JmTc@ZbH;f^iZPtd)!mbvw6@#L)Eh9SUGKnQoh~YVUHCH_Jm&qs4?m0aYNwLV~6fHTrT6% z;dH^xWiy}{?kW%$*XC_IP)@sbI6a;$CXmic+s^p7n=RN$FE7|MpI$VHXoFDtocnoq zd_Mi=W3f2d7a|?*L^$8<6bqS@FbGX3+(zCW&!h{Em-cV&I^$Ees2>>zKjN1q5r)cH z2RwEPnh3b9OG4>RiI%2xH=b~kUcxKPh^86I*RmN2Agg8%mFD>=M_=w-jPKjOyzi=$puWN&w<~vWh&zCxrJ)OxV;1ediv;c~3 z?U!>Tv(GE!#huW`t-d%FufE|W;ZG3Jo$yKD^ zN*p*uZ5;O|(?Sr^1#p76M0~giM4%3ezTH1BD!y~B#piH((LF_Z-Q8VE*t0O^v>x%PaSdYijfj zU0%6wTvMZO=<>>aV_Rk4C}#;Uk^2S4A(g(=?-v#M*NZca=BMHxtuej%7xW-EH-c|o5@p;ro1=A zi<20w&>&yGGG|s%emj4L1v`J*t~RK z5>?9Eq2z^0zWQUlQC5GcgdqQ|Rq}f@oWd}G#w)e*@2Zj?QwuS8sF(k?QvT}{`==y{ zkjyUEdv__{%}Ty}sXJ6LyPTigBH^}E%`c?FDN99CNJWmMLN=pZ&fZcv*DB>qlVs$D zWQ6jyQ@a)O7%?JaDetZlQ^1+bn0+V4x!$Bl*yG#Hpp8#3iIN~jv6LS=Ax&1Q>xw@c z5jrHL|LA>aJ?ilL*w&>nbPZ;@R55&L{wn6}rv|-b63)A1bZI^(($F!sS0BN?-5(N& z$ueD}_G!l!%=>j^Q5rf=9~{3mg0ddPG3J{2!l(;m6XF`Bp|Wb)bRHk1fjyU;wOSs0 z7sst$`gAaV4enF34CJ8?%D~q0^aI$h%jVqEiY=6;W;ye=h;kl6`gYjjFB4F1zutB0 zNM@!>39yS$uQOL6{W>on zDr6xz&r&(Up5ZhWQSi;jhxln~&n)U?BYit@XTBK0uc+mmrE;cI8T5)^si@3TvS+op zkF-#M@5JRxPaZyMi>`!K1>_uab2y8#G;^V=Ph^tClxz3fnF*WnWs76UU`2u)6F%#- zeXr%q-^8_mqmSTNUdtgn=(84EfL*)1sDkst*1*o;1Zat|Z^>4=b}P6~{ctV%G}d=; zZ7k?FYGKWK*{{8X6NRzooK_rP%5*Et^09JYwKH3;5UbG z)y&p8q@%Xlk2OEa6UkP}1XzQ0Hr3WODHozorB<=^&37zo|Gjf`UFDSy*SC>QUjLJA zr8d&F|AzQXI+LY|Vc>NuF}*eqzn)HDxzob}oxQz#@9wV7?#^zztLt|9x1;MbWTN12 z8`>9ETdg@a8I&m7QLP_d7S9yZ2`@cqkGeF4cN<`jiW&8$#r&LfrV*@mdY@&q&m(o+xU5KuT}S0Enl~B-_~?7br|)*p%Hdh_x6Gc-p{2=$3cn)T_XMt+ z*mn-;O7@+^p%HHN?Gr`SwC@n^Wn$ldmFzn_ZQia|&*OL4dB95*Q?{73N=;kSHp0=w zw)aZ5(bFRw8ew%lXZ&lG4qzy;+&~n@57cg*&H}cR?TykxK~SquNc7!4jv`B zSizj{5XsULwn~4z9*fEa)F2N>LibT^Cv8~OJ8M}3-@|{En5x==e*E+YNY^TC9~huK zL4ONuujU6kwYCdfQ5*P~(9<3&89E{6Q)NG5ZJ_5f_sCCX^uF}+17aL)_ZJE|4-YKx zkRt9+`2BLp=51EXH-@lp6LrBNAdc0#P+RDQ8(}APK(D_bWL7wQ?JSEY3en3y-d9(uZ^kKW5YB5`OmKhy68<1+H0MgnYDWL|m5p37zk$a-N2h zy7a-?p;}I`CU|A#T6h@&6lu{tcULKUPwD(kb=SE=MT#bh%;48{%KGsV6jd-hSp1c1t-?t? z=V+YLDr}FY^coHR+X|Z@B<_;yl5_L?_Y2aJxjRj=C&trk69Sc0) z1%I-pkcd1MxM9GZwns7u%d!XDEIbFj)DfCIS6`o(xuG2sz$R;7mi{*x6MXwD(zS{@ zL>m+Aty_m0g9F@CT?1(FI{mkQg?qG#dPN%>?1@q@Wssm-yItw?e|fzxil0~WErQ;^ zK;zwj15xxo5(gCF*KuGa&-K-IY@)TE7yS7F`DX-Hf(udB?{ACZ2NXe9*Yahb&3fLn zdj7&3II?Csx}N)y2m@XaA+DR=tDIP0ihDzxkBi&tpR?12*vs_4iI%@e5%c z5zj{S_SaHzRl^ry9sk8gQKvRhr)WC9)2vR4g6p5ZvWCw;=Qn-sUh6%p_60oGxd=SK zbNCm}N_>c->4tnz)LhR8)^mUD$CL}T-Y@znTo7aYl5dEr?_Fm8A*Af84`1v1a4@op zf9B`&%Db6mI0a_(%zp`<&wpnGcUv=^k33Jm%8wbyx_Ft#MAO?v%=ImZXcpD0q3IOK zS%ouFJUjf|%X3KADr}FY>9OBkKCI=0?PxeIuMzXHMbh$D zXuKM5A&Qn8aDjDv5qPjjI{u13#wY(AeV*P?-!GI!haSDF8m{_mo}V}E{QQ-3Xuq}D zT=3dOIR=+vzG}t5Un|98$}eAtRd(s!3zZvc45oIy-8>ea-f8(RjYSYgOmz@~PPWr0 zr~|E0>EcP{Sa@Xi-!AZNUzS+-C#{z6Jx{j!b@9c#R1fUR^)ZQu*Vac-bKSZs& F z%jNOG0_R@Dh)H{#m{{JKCCTdN$ZFv75;(N^mJNxeK>{!`GX&0D*Za* z{Oghx<2m_Ri&n4AW9G=uy?Y;QM9dJ?A(CDzF>_j~tFNyu_If%r`=_rF=6-!5Yqk;x zrLB`9+uGP?)@{9RFLR6>@%>+Ed|Kv5L|r%Gha&A-ez3MzYWp(B#Su4tF2?(T?^uBL zE3t7}|Dni#P5;i?-?Z9)4sHb6pCH|&Lu2jVQ6=sI1yvmcsPp!^V*oP9EJlua|36tI zM;sB=CDIt6T8x~Q`08K)_f*FNjf|1QSm1x(!9CjjdPN=!REv?))=e1{=;m`gCeV)w zv+K0U#a~|z=03_d4ce8{vtd$5oKOqEnZIR4-}EF z<-b|un^xn`ZvscwPSKAg`@anotp5>jYOTlL5PEO2(?$UrZTHJmxgozNsh!`OzyIFS zWv$vW3tRC%Fx4TX?Li%AuUn(2^8S06?kDw2^NXqLhpbgwUnHCTI#uX;P_JrhvsK=6 z*49grb?sb^b$wZMow&XALzxGPt~ZhgimYpSz`DNVx=tLpyhG-IGG`|a)LXBw@{YUK zA1Jb3F?W}JH|u&+>iYU~)~YK$3tp_9rm!`mv+A{k@Y(q^Q^58Yd3LawDIxpa+&Q5dv}evyt^Q`e?;vzXn)r2=56~u-;&#Y z0lFQIbJzC&nwqWpW}bgHt#0rACvXLE?!iycm!F-}@3&rzt(tCMDaE-xzf zJw}bC)atc)e7m*DTDAXW+z0XPbo-EA2e#ALL${v3LvMSn*VPvF`S?Ys%iCnDUzZBK z4(dbgp{tJ^uP?QQRG$jD*3PqduDvXCZNX&`9}%>_qSqh2{VU{J+x|SyUh;XiX#02N zzOU#twSB#H^PB7XvyfcZ_06o;O{v$`S?C0<$so=>vR;2O9Oo{{L|L*ccu_aD%{;!H zcs}?jY)5>1WWDaLYP%$H6*Fw$r2KuBM*n6EbbIhg>briOBI~viw=cDcR4+xkb#Y9r z+j`wz<`{Ru`9GEU5mDET_@PLqxH?4{ms!ow!R*mmOk0eaSt*!IZ!e|!BpNP<~85KzYf zjf`!_7~nt>^+Ier)hqHCpjzC%)IL((l)(Vqm;ZKK)BMdB;yAs(QRYHKy^qEPMfj_5 zfwjL<`{idWi}0R<@O85?4;G;9yCeC$u*lc)fLTRcB3ap1!Ji>%{UhhpI*t*D{lYOIC!xjO7IjDw@G>oAQo5C<1o!~J;A zcvr?D zeJKXk@B0hT?apfcSn_A}G+Xu0JpXRm`SVynb!qGu)R%a^)VfV^4v2@Nx>V?P zP@ih?aJLxLhjqKTb^Cp}{T1DAMEkRDH?MAgD7XCpzTiW?i$2u$_2S{=c-X)ntNyH> zZma&8b-QVG`ylB6*3C&Ph$7_}_P|fjUbk+3ogjt$4tciv#2Wg+bKncs&?9U5rvx2>HRymN z)1le8$_aIirU7f{chACp@%N@8>bd{B8LIs%$Cq5o@21!UtfMYS%l+qH_3FerPQUNa z#B1qy|FO)0h&pb@0oL$lZvSt|?Y{sG4?j-{e6#A?YU{V^t9f4DwDa=2>HX}M{YU!a z*z<=GU<4QeMt~7u1Q-EEfDvE>7y(9r5nu!u0Y-okU<4QeMt~7u1Q-EEfDvE>7y(9r z5nu!u0Y-okU<4QeMt~7u1Q-EEfDvE>7y(9r5nu!u0Y-okU<4QeMt~7u1Q-EEfDvE> z7y(9r5nu!u0Y-okU<4QeMt~7u1Q-EEfDvE>7y(9r5nu!u0Y-okU<4QeMt~7u1Q-EE zfDvE>7y(9r5nu!u0Y-okU<4QeMt~7u1Q-EEfDvE>7y(9r5nu!u0Y-okU<4QeMt~7u z1Q-EEfDvE>7y(9r5nu!u0Y-okU<4QeMt~7u1Q-EEfDvE>7y(9r5nu!u0Y-okU<4Qe zMt~7u1Q-EEfDvE>7y(9r5nu!u0Y-okU<4QeMt~7u1Q-EEfDvE>7y(9r5nu!u0Y-ok zU<4QeMt~7u1Q-EEfDvE>7y(9r5nu!u0Y+d2A#k2Qi~u9R2rvSS03*N%FanGKBftnS z0*nA7u&fARnW1F`Ej4ge7y(9r5nu!u0Y-okU<4QeMt~7u1Q>w}6@eC7ARyzHbD?0l z@88{NpKy|1!Yj3N-20G| z&AaxfQ*i5;v8U5M>7^5Pe9Colb<(QMw-;D>DYtXT^(LnZk)c+LGyZ<(Gr#;G=wFt# z`u+V!`s2o#O9}Tq{V(hK@A{9Sethr$(cjNJ;P1ciQ~&y%*j+;TGsyGoU``-dDhzh?^$bb<2}}({f;$w+pIMhpRxvj z?{Q;)6qi1V2e}U4qijm{rTxl0TpAuS&Wtm0AokT?OvJwW)S1|06BDt=E)Owf4O~*lKj&yS8q# zYLydQ>-)=%M)uux@@=E?f^vD^d_J)6x1T!GH}lk)-d{{i^!D`h^uE;7)0f9~4s*Uu z*w8=4z7P8@p06>7n~%fXCM%~+#M;nd%-OQoXsxJN+CN;{2j}SeAI%!i*J9EDNf~mi zMyObXP078)UmL~seOS3ODeFzcbl!9zeRHKT5Qx$LBsQ#B_rF)#pT(ZG7qM-!EJ|xSUt@n1ZX+6Jn(+4fhUw<|tkovv z&AoB{|BTW-Y?iLiG~HRN#pJ%It`RP8oHw(t|7aNZYBu!snw3{AKc6?lIVw)#`P%(+ z*1%YwH2}>RxC7f-9Iu|WVn4%}=q$EB!IsDNIcx{9?ZdVc8`i;N?Rf6?Z`l6E+*g_U zG)`ysEVCJLk{+)Hhva&i;e)YxFb9|e%mL;AbAUO(9AFMG2bcpFHV4G}6v4X`kKw(FFaP9B>><2M z@etmlC}MN5jm=uI6WHQ7{w?h9#rqY%i!^w@BKA8t7x7(-W_*6jHZ}IyyA+>AhqN`z z6DxYQ{UyUqO{;!ymO8N=i~u9B0ugA$_^S8Ky+$#ReTN@2O4s}IKQv4C->;dab8j|F z_ub8A>AwEgX6c@IW7Y)UJAQf2sJy<`f9^9W&ooW%%U?9IuYcpKkDHX)_r;BKCh1Jm z^l$!8v-`f1Gs3U;st1kIncBzcF6_Apw;#73bAUO(9AFMG2bcrQ0phJJ7-zFv)JhOynce;_qt(XqVEZ8INxuVeyidfey;+X_-!vb*M+UG zw9zp(ex}^mXUZ2cp217u(%x#44?WXv)Yc?bVlx;4Mqni(&}hBq@^}o%9AFMG2bcrQ z0pFb9|e z%mL;AbKv91feU#ay&Sf$#5o?Fb9|e%z>54f!OCP zD|XrZ@4v-jVjdIx#+G(c)9hG?zx{^4_ZH0UOMlMmOQ;NO+E1-I^{@Whu3FcFYyDR~ zG7=m&dM|#zuWyC5QxV+DuB5AZ9}8W`+t7_ Date: Thu, 10 Nov 2022 14:03:05 +0100 Subject: [PATCH 151/232] renamed flyway script and fixed default value --- ...orting_licenses.sql => V5.13.0.1__8671-sorting_licenses.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/main/resources/db/migration/{V5.12.0.1__8671-sorting_licenses.sql => V5.13.0.1__8671-sorting_licenses.sql} (60%) diff --git a/src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql similarity index 60% rename from src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql rename to src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql index a449c85cf16..6fe3f1142c2 100644 --- a/src/main/resources/db/migration/V5.12.0.1__8671-sorting_licenses.sql +++ b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql @@ -1,5 +1,5 @@ ALTER TABLE license -ADD COLUMN IF NOT EXISTS sortorder BIGINT NOT NULL DEFAULT(0); +ADD COLUMN IF NOT EXISTS sortorder BIGINT NOT NULL DEFAULT 0; CREATE INDEX IF NOT EXISTS license_sortorder_id ON license (sortorder, id); \ No newline at end of file From 022955da1b3af8b139f3764898f067169259dd3d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 10 Nov 2022 12:33:18 -0500 Subject: [PATCH 152/232] fix typos --- doc/sphinx-guides/source/api/external-tools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index c5b1c43745e..09293ea8a64 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -106,11 +106,11 @@ Terminology allowedApiCalls An array of objects defining callbacks the tool is allowed to make to the Dataverse API. If the dataset or file being accessed is not public, the callback URLs will be signed to allow the tool access for a defined time. - allowdApiCalls name A name the tool will use to identify this callback URL + allowedApiCalls name A name the tool will use to identify this callback URL allowedApiCalls urlTemplate The relative URL for the callback using the reserved words to indicate where values should by dynamically substituted - allowdApiCalls httpMethod Which HTTP method the specified callback uses + allowedApiCalls httpMethod Which HTTP method the specified callback uses allowedApiCalls timeOut For non-public datasets and datafiles, how long the signed URLs given to the tool should be valid for. From b357cb2f0b2c4cceb1b67913d8d2a6342703eab5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 10 Nov 2022 12:34:30 -0500 Subject: [PATCH 153/232] add SITE_URL for running single tests FWIW: more recent Mockito has a mockStatic method but that doesn't appear to be available --- .../externaltools/ExternalToolHandlerTest.java | 13 +++++++++---- .../harvard/iq/dataverse/util/UrlTokenUtilTest.java | 12 +++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index 70c835839bb..cdefc844c03 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -7,12 +7,11 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonUtil; import javax.json.Json; import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -21,6 +20,7 @@ import java.util.List; import org.junit.Test; +import org.mockito.Mockito; public class ExternalToolHandlerTest { @@ -206,8 +206,8 @@ public void testGetToolUrlWithOptionalQueryParameters() { } @Test - public void testGetToolUrlWithallowedApiCalls() { - + public void testGetToolUrlWithAllowedApiCalls() { + String oldVal = System.setProperty(SystemConfig.SITE_URL, "https://librascholar.org"); System.out.println("allowedApiCalls test"); Dataset ds = new Dataset(); ds.setId(1L); @@ -233,5 +233,10 @@ public void testGetToolUrlWithallowedApiCalls() { assertTrue(signedUrl.contains("&method=GET")); assertTrue(signedUrl.contains("&token=")); System.out.println(JsonUtil.prettyPrint(jo)); + if(oldVal==null) { + System.clearProperty(SystemConfig.SITE_URL); + } else { + System.setProperty(SystemConfig.SITE_URL, oldVal); + } } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java index ffc6b813045..68e1a9291ee 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java @@ -7,21 +7,18 @@ import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.when; - import java.util.ArrayList; import java.util.List; import org.junit.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.Mockito; public class UrlTokenUtilTest { @Test public void testGetToolUrlWithOptionalQueryParameters() { + String oldVal = System.setProperty(SystemConfig.SITE_URL, "https://librascholar.org"); DataFile dataFile = new DataFile(); dataFile.setId(42l); FileMetadata fmd = new FileMetadata(); @@ -46,5 +43,10 @@ public void testGetToolUrlWithOptionalQueryParameters() { URLTokenUtil urlTokenUtil2 = new URLTokenUtil(ds, apiToken, "en"); assertEquals("https://librascholar.org/api/datasets/50?key=" + apiToken.getTokenString(), urlTokenUtil2.replaceTokensWithValues("{siteUrl}/api/datasets/{datasetId}?key={apiToken}")); assertEquals("https://librascholar.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2ABCDEF&key=" + apiToken.getTokenString(), urlTokenUtil2.replaceTokensWithValues("{siteUrl}/api/datasets/:persistentId/?persistentId={datasetPid}&key={apiToken}")); + if(oldVal==null) { + System.clearProperty(SystemConfig.SITE_URL); + } else { + System.setProperty(SystemConfig.SITE_URL, oldVal); + } } } From d59fbfc3f1a8c1ca84ade61405c0720415b67b6e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 10 Nov 2022 13:51:18 -0500 Subject: [PATCH 154/232] rename sql script post 5.12.1 release #7715 --- ...ls-for-tools.sql => V5.12.1.1__7715-signed-urls-for-tools.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.12.0.1__7715-signed-urls-for-tools.sql => V5.12.1.1__7715-signed-urls-for-tools.sql} (100%) diff --git a/src/main/resources/db/migration/V5.12.0.1__7715-signed-urls-for-tools.sql b/src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql similarity index 100% rename from src/main/resources/db/migration/V5.12.0.1__7715-signed-urls-for-tools.sql rename to src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql From de8386bd7d267b2105c7d4f0f50f5e7fa3481641 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 10 Nov 2022 15:16:19 -0500 Subject: [PATCH 155/232] typos and minor wording and formatting fixes #7715 --- .../root/external-tools/fabulousFileTool.json | 2 +- .../source/api/external-tools.rst | 32 ++++++++++--------- .../iq/dataverse/api/AbstractApiBean.java | 6 ++-- .../dataverse/externaltools/ExternalTool.java | 4 +-- .../iq/dataverse/util/UrlSignerUtil.java | 2 +- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json index 83594929a96..1c132576099 100644 --- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json +++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json @@ -1,6 +1,6 @@ { "displayName": "Fabulous File Tool", - "description": "A non-existent tool that is Fabulous Fun for Files!", + "description": "A non-existent tool that is fabulous fun for files!", "toolName": "fabulous", "scope": "file", "types": [ diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index c5b1c43745e..a8f83590871 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -92,9 +92,9 @@ Terminology contentType File level tools operate on a specific **file type** (content type or MIME type such as "application/pdf") and this must be specified. Dataset level tools do not use contentType. - toolParameters **httpMethod**, **Query parameters**, and **allowedApiCalls** are supported and described below. + toolParameters **httpMethod**, **queryParameters**, and **allowedApiCalls** are supported and described below. - httpMethod Either **GET** or **POST**. + httpMethod Either ``GET`` or ``POST``. queryParameters **Key/value combinations** that can be appended to the toolUrl. For example, once substitution takes place (described below) the user may be redirected to ``https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org``. @@ -106,13 +106,13 @@ Terminology allowedApiCalls An array of objects defining callbacks the tool is allowed to make to the Dataverse API. If the dataset or file being accessed is not public, the callback URLs will be signed to allow the tool access for a defined time. - allowdApiCalls name A name the tool will use to identify this callback URL + allowedApiCalls name A name the tool will use to identify this callback URL such as ``retrieveDataFile``. - allowedApiCalls urlTemplate The relative URL for the callback using the reserved words to indicate where values should by dynamically substituted + allowedApiCalls urlTemplate The relative URL for the callback using reserved words to indicate where values should by dynamically substituted such as ``/api/v1/datasets/{datasetId}``. - allowdApiCalls httpMethod Which HTTP method the specified callback uses + allowedApiCalls httpMethod Which HTTP method the specified callback uses such as ``GET`` or ``POST``. - allowedApiCalls timeOut For non-public datasets and datafiles, how long the signed URLs given to the tool should be valid for. + allowedApiCalls timeOut For non-public datasets and datafiles, how many minutes the signed URLs given to the tool should be valid for. Must be an integer. toolName A **name** of an external tool that is used to differentiate between external tools and also used in bundle.properties for localization in the Dataverse installation web interface. For example, the toolName for Data Explorer is ``explorer``. For the Data Curation Tool the toolName is ``dct``. This is an optional parameter in the manifest JSON file. =========================== ========== @@ -146,17 +146,19 @@ Reserved Words Authorization Options +++++++++++++++++++++ -When called for Datasets or DataFiles that are not public, i.e. in a draft dataset or for a restricted file, external tools are allowed access via the user's credentials. This is accomplished by one of two mechanisms: +When called for datasets or data files that are not public (i.e. in a draft dataset or for a restricted file), external tools are allowed access via the user's credentials. This is accomplished by one of two mechanisms: * Signed URLs (more secure, recommended) - Configured via the allowedApiCalls section of the manifest. The tool will be provided with signed URLs allowing the specified access to the given dataset or datafile for the specified amount of time. The tool will not be able to access any other datasets or files the user may have access to and will not be able to make calls other than those specified. - For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest. - For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool. - -* ApiToken (deprecated, less secure, not recommended) - Configured via the queryParameters by including an {apiToken} value. When this is present Dataverse will send the user's apiToken to the tool. With the user's apiToken, the tool can perform any action via the Dataverse api that the user could. External tools configured via this method should be assessed for their trustworthiness. - For tools invoked via GET, this will be done via a query parameter in the request URL which could be cached in the browser's history. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool. - For tools invoked via POST, Dataverse will send a JSON body including the apiToken. + + - Configured via the ``allowedApiCalls`` section of the manifest. The tool will be provided with signed URLs allowing the specified access to the given dataset or datafile for the specified amount of time. The tool will not be able to access any other datasets or files the user may have access to and will not be able to make calls other than those specified. + - For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest. + - For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool. + +* API Token (deprecated, less secure, not recommended) + + - Configured via the ``queryParameters`` by including an ``{apiToken}`` value. When this is present Dataverse will send the user's apiToken to the tool. With the user's API token, the tool can perform any action via the Dataverse API that the user could. External tools configured via this method should be assessed for their trustworthiness. + - For tools invoked via GET, this will be done via a query parameter in the request URL which could be cached in the browser's history. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool. + - For tools invoked via POST, Dataverse will send a JSON body including the apiToken. Internationalization of Your External Tool ++++++++++++++++++++++++++++++++++++++++++ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index a6e1f4d9ef1..a7e09000ade 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -432,10 +432,10 @@ private AuthenticatedUser findAuthenticatedUserOrDie( String key, String wfid ) private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { AuthenticatedUser authUser = null; - // The signUrl contains a param telling which user this is supposed to be for. + // The signedUrl contains a param telling which user this is supposed to be for. // We don't trust this. So we lookup that user, and get their API key, and use - // that as a secret in validation the signedURL. If the signature can't be - // validating with their key, the user (or their API key) has been changed and + // that as a secret in validating the signedURL. If the signature can't be + // validated with their key, the user (or their API key) has been changed and // we reject the request. // ToDo - add null checks/ verify that calling methods catch things. String user = httpRequest.getParameter("user"); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java index 97838b45cc5..1789b7a90c3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalTool.java @@ -97,8 +97,8 @@ public class ExternalTool implements Serializable { /** * Set of API calls the tool would like to be able to use (e,.g. for retrieving - * data through the Dataverse REST api). Used to build signedUrls for POST - * headers, as in DPCreator + * data through the Dataverse REST API). Used to build signedUrls for POST + * headers, as in DP Creator */ @Column(nullable = true, columnDefinition = "TEXT") private String allowedApiCalls; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java index ee3dd127196..3e15dc2b75c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -71,7 +71,7 @@ public static String signUrl(String baseUrl, Integer timeout, String user, Strin * This method will only return true if the URL and parameters except the * "token" are unchanged from the original/match the values sent to this method, * and the "token" parameter matches what this method recalculates using the - * shared key THe method also assures that the "until" timestamp is after the + * shared key. The method also assures that the "until" timestamp is after the * current time. * * @param signedUrl - the signed URL as received from Dataverse From 5c5468cd6e0b6d2de34665f568092afb692d35ce Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 10 Nov 2022 16:12:42 -0500 Subject: [PATCH 156/232] use same term as in ext tools --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index c9ce12fec98..b3fa5d132de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2281,7 +2281,7 @@ public Response getSignedUrl(JsonObject urlInfo) throws WrappedResponse { String baseUrl = urlInfo.getString("url"); int timeout = urlInfo.getInt("timeout", 10); - String method = urlInfo.getString("method", "GET"); + String method = urlInfo.getString("httpMethod", "GET"); String signedUrl = UrlSignerUtil.signUrl(baseUrl, timeout, userId, method, key); From 673987b69e27fd1f1d0e56851839b99b1d18ec65 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 10 Nov 2022 16:45:08 -0500 Subject: [PATCH 157/232] doc for requestSignedUrl --- doc/sphinx-guides/source/api/native-api.rst | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6d68d648cb3..1b5a31dfea0 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4072,4 +4072,28 @@ The fully expanded example above (without environment variables) looks like this curl -X DELETE https://demo.dataverse.org/api/admin/template/24 - +Request Signed URL +~~~~~~~~~~~~~~~~~~ + +Dataverse has the ability to create signed URLs for it's API calls. +A signature, which is valid only for the specific API call and only for a specified duration, allows the call to proceed with the authentication of the specified user. +It is intended as an alternative to the use of an API key (which is valid for a long time period and can be used with any API call). +Signed URLs were developed to support External Tools but may be useful in other scenarios where Dataverse or a third-party tool needs to delegate limited access to another user or tool. +This API call allows a Dataverse superUser to generate a signed URL for such scenarios. +The JSON input parameter required is an object with the following keys: + +- ``url`` - the exact URL to sign, including api version number and all query parameters +- ``timeout`` - how long in minutes the signature should be valid for, default is 10 minutes +- ``httpMethod`` - which HTTP method is required, default is GET +- ``user`` - the user identifier for the account associated with this signature, the default is the superuser making the call. The API call will succeed/fail based on whether the specified user has the required permissions. + +A curl example using allowing access to a dataset's metadata + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export JSON={"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeout":5,"user":"alberteinstein"} + + curl -H 'X-Dataverse-key:$API_KEY' -d $JSON $SERVER_URL/api/admin/requestSignedUrl + From bf875423cc29eacd92298118afd6ba7c3464b8ef Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 10 Nov 2022 18:00:08 -0500 Subject: [PATCH 158/232] Updates per reviews --- doc/sphinx-guides/source/api/native-api.rst | 40 ++++++++++++++++++- .../iq/dataverse/api/AbstractApiBean.java | 4 +- .../edu/harvard/iq/dataverse/api/Admin.java | 7 ++-- .../externaltools/ExternalToolHandler.java | 27 +++++++------ .../iq/dataverse/util/UrlSignerUtil.java | 22 +++++----- .../ExternalToolHandlerTest.java | 8 +--- .../iq/dataverse/util/UrlTokenUtilTest.java | 11 ++--- 7 files changed, 77 insertions(+), 42 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 1b5a31dfea0..de727592694 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2029,6 +2029,24 @@ Archiving is an optional feature that may be configured for a Dataverse installa curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/:persistentId/$VERSION/archivalStatus?persistentId=$PERSISTENT_IDENTIFIER" +Get External Tool Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API call is intended as a callback that can be used by :doc:`/installation/external-tools` to retrieve signed Urls necessary for their interaction with Dataverse. +It can be called directly as well. + +The response is a JSON object described in the :doc:`/api/external-tools` section of the API guide. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV + export VERSION=1.0 + export TOOL_ID=1 + + + curl -H "X-Dataverse-key: $API_TOKEN" -H "Accept:application/json" "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/toolparams/$TOOL_ID?persistentId=$PERSISTENT_IDENTIFIER" Files ----- @@ -2689,6 +2707,24 @@ Note the optional "limit" parameter. Without it, the API will attempt to populat By default, the admin API calls are blocked and can only be called from localhost. See more details in :ref:`:BlockedApiEndpoints <:BlockedApiEndpoints>` and :ref:`:BlockedApiPolicy <:BlockedApiPolicy>` settings in :doc:`/installation/config`. +Get External Tool Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API call is intended as a callback that can be used by :doc:`/installation/external-tools` to retrieve signed Urls necessary for their interaction with Dataverse. +It can be called directly as well. + +The response is a JSON object described in the :doc:`/api/external-tools` section of the API guide. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export FILE_ID=3 + export FILEMETADATA_ID=1 + export TOOL_ID=1 + + curl -H "X-Dataverse-key: $API_TOKEN" -H "Accept:application/json" "$SERVER_URL/api/files/$FILE_ID/metadata/$FILEMETADATA_ID/toolparams/$TOOL_ID + Users Token Management ---------------------- @@ -4083,7 +4119,7 @@ This API call allows a Dataverse superUser to generate a signed URL for such sce The JSON input parameter required is an object with the following keys: - ``url`` - the exact URL to sign, including api version number and all query parameters -- ``timeout`` - how long in minutes the signature should be valid for, default is 10 minutes +- ``timeOut`` - how long in minutes the signature should be valid for, default is 10 minutes - ``httpMethod`` - which HTTP method is required, default is GET - ``user`` - the user identifier for the account associated with this signature, the default is the superuser making the call. The API call will succeed/fail based on whether the specified user has the required permissions. @@ -4093,7 +4129,7 @@ A curl example using allowing access to a dataset's metadata export SERVER_URL=https://demo.dataverse.org export API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export JSON={"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeout":5,"user":"alberteinstein"} + export JSON={"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeOut":5,"user":"alberteinstein"} curl -H 'X-Dataverse-key:$API_KEY' -d $JSON $SERVER_URL/api/admin/requestSignedUrl diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index a7e09000ade..a33635fe0d1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -363,7 +363,7 @@ protected AuthenticatedUser findUserByApiToken( String apiKey ) { protected User findUserOrDie() throws WrappedResponse { final String requestApiKey = getRequestApiKey(); final String requestWFKey = getRequestWorkflowInvocationID(); - if (requestApiKey == null && requestWFKey == null && getRequestParameter("token")==null) { + if (requestApiKey == null && requestWFKey == null && getRequestParameter(UrlSignerUtil.SIGNED_URL_TOKEN)==null) { return GuestUser.get(); } PrivateUrlUser privateUrlUser = privateUrlSvc.getPrivateUrlUserFromToken(requestApiKey); @@ -420,7 +420,7 @@ private AuthenticatedUser findAuthenticatedUserOrDie( String key, String wfid ) } else { throw new WrappedResponse(badWFKey(wfid)); } - } else if (getRequestParameter("token") != null) { + } else if (getRequestParameter(UrlSignerUtil.SIGNED_URL_TOKEN) != null) { AuthenticatedUser authUser = getAuthenticatedUserFromSignedUrl(); if (authUser != null) { return authUser; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index b3fa5d132de..b435f23f52c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -92,6 +92,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.DeleteRoleCommand; import edu.harvard.iq.dataverse.engine.command.impl.DeleteTemplateCommand; import edu.harvard.iq.dataverse.engine.command.impl.RegisterDvObjectCommand; +import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.userdata.UserListMaker; @@ -2280,12 +2281,12 @@ public Response getSignedUrl(JsonObject urlInfo) throws WrappedResponse { } String baseUrl = urlInfo.getString("url"); - int timeout = urlInfo.getInt("timeout", 10); - String method = urlInfo.getString("httpMethod", "GET"); + int timeout = urlInfo.getInt(ExternalToolHandler.TIMEOUT, 10); + String method = urlInfo.getString(ExternalToolHandler.HTTP_METHOD, "GET"); String signedUrl = UrlSignerUtil.signUrl(baseUrl, timeout, userId, method, key); - return ok(Json.createObjectBuilder().add("signedUrl", signedUrl)); + return ok(Json.createObjectBuilder().add(ExternalToolHandler.SIGNED_URL, signedUrl)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index c9da22081b9..f55439d23a9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -43,6 +43,13 @@ public class ExternalToolHandler extends URLTokenUtil { private final ExternalTool externalTool; private String requestMethod; + + public static final String HTTP_METHOD="httpMethod"; + public static final String TIMEOUT="timeOut"; + public static final String SIGNED_URL="signedUrl"; + public static final String NAME="name"; + public static final String URL_TEMPLATE="urlTemplate"; + /** * File level tool @@ -71,17 +78,13 @@ public ExternalToolHandler(ExternalTool externalTool, Dataset dataset, ApiToken this.externalTool = externalTool; } - // TODO: rename to handleRequest() to someday handle sending headers as well as - // query parameters. public String handleRequest() { return handleRequest(false); } - // TODO: rename to handleRequest() to someday handle sending headers as well as - // query parameters. public String handleRequest(boolean preview) { JsonObject toolParameters = JsonUtil.getJsonObject(externalTool.getToolParameters()); - JsonString method = toolParameters.getJsonString("httpMethod"); + JsonString method = toolParameters.getJsonString(HTTP_METHOD); requestMethod = method != null ? method.getString() : HttpMethod.GET; JsonObject params = getParams(toolParameters); logger.fine("Found params: " + JsonUtil.prettyPrint(params)); @@ -172,11 +175,11 @@ public JsonObjectBuilder createPostBody(JsonObject params) { JsonArrayBuilder apisBuilder = Json.createArrayBuilder(); apiArray.getValuesAs(JsonObject.class).forEach(((apiObj) -> { - logger.info(JsonUtil.prettyPrint(apiObj)); - String name = apiObj.getJsonString("name").getString(); - String httpmethod = apiObj.getJsonString("httpMethod").getString(); - int timeout = apiObj.getInt("timeOut"); - String urlTemplate = apiObj.getJsonString("urlTemplate").getString(); + logger.fine(JsonUtil.prettyPrint(apiObj)); + String name = apiObj.getJsonString(NAME).getString(); + String httpmethod = apiObj.getJsonString(HTTP_METHOD).getString(); + int timeout = apiObj.getInt(TIMEOUT); + String urlTemplate = apiObj.getJsonString(URL_TEMPLATE).getString(); logger.fine("URL Template: " + urlTemplate); urlTemplate = SystemConfig.getDataverseSiteUrlStatic() + urlTemplate; String apiPath = replaceTokensWithValues(urlTemplate); @@ -189,8 +192,8 @@ public JsonObjectBuilder createPostBody(JsonObject params) { System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); } logger.fine("Signed URL: " + url); - apisBuilder.add(Json.createObjectBuilder().add("name", name).add("httpMethod", httpmethod) - .add("signedUrl", url).add("timeOut", timeout)); + apisBuilder.add(Json.createObjectBuilder().add(NAME, name).add(HTTP_METHOD, httpmethod) + .add(SIGNED_URL, url).add(TIMEOUT, timeout)); })); bodyBuilder.add("signedUrls", apisBuilder); return bodyBuilder; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java index 3e15dc2b75c..29c4e8a6fb9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/UrlSignerUtil.java @@ -20,6 +20,10 @@ public class UrlSignerUtil { private static final Logger logger = Logger.getLogger(UrlSignerUtil.class.getName()); + public static final String SIGNED_URL_TOKEN="token"; + public static final String SIGNED_URL_METHOD="method"; + public static final String SIGNED_URL_USER="user"; + public static final String SIGNED_URL_UNTIL="until"; /** * * @param baseUrl - the URL to sign - cannot contain query params @@ -45,18 +49,18 @@ public static String signUrl(String baseUrl, Integer timeout, String user, Strin LocalDateTime validTime = LocalDateTime.now(); validTime = validTime.plusMinutes(timeout); validTime.toString(); - signedUrlBuilder.append(firstParam ? "?" : "&").append("until=").append(validTime); + signedUrlBuilder.append(firstParam ? "?" : "&").append(SIGNED_URL_UNTIL + "=").append(validTime); firstParam = false; } if (user != null) { - signedUrlBuilder.append(firstParam ? "?" : "&").append("user=").append(user); + signedUrlBuilder.append(firstParam ? "?" : "&").append(SIGNED_URL_USER + "=").append(user); firstParam = false; } if (method != null) { - signedUrlBuilder.append(firstParam ? "?" : "&").append("method=").append(method); + signedUrlBuilder.append(firstParam ? "?" : "&").append(SIGNED_URL_METHOD + "=").append(method); firstParam=false; } - signedUrlBuilder.append(firstParam ? "?" : "&").append("token="); + signedUrlBuilder.append(firstParam ? "?" : "&").append(SIGNED_URL_TOKEN + "="); logger.fine("String to sign: " + signedUrlBuilder.toString() + ""); String signedUrl = signedUrlBuilder.toString(); signedUrl= signedUrl + (DigestUtils.sha512Hex(signedUrl + key)); @@ -98,19 +102,19 @@ public static boolean isValidUrl(String signedUrl, String user, String method, S String allowedMethod = null; String allowedUser = null; for (NameValuePair nvp : params) { - if (nvp.getName().equals("token")) { + if (nvp.getName().equals(SIGNED_URL_TOKEN)) { hash = nvp.getValue(); logger.fine("Hash: " + hash); } - if (nvp.getName().equals("until")) { + if (nvp.getName().equals(SIGNED_URL_UNTIL)) { dateString = nvp.getValue(); logger.fine("Until: " + dateString); } - if (nvp.getName().equals("method")) { + if (nvp.getName().equals(SIGNED_URL_METHOD)) { allowedMethod = nvp.getValue(); logger.fine("Method: " + allowedMethod); } - if (nvp.getName().equals("user")) { + if (nvp.getName().equals(SIGNED_URL_USER)) { allowedUser = nvp.getValue(); logger.fine("User: " + allowedUser); } @@ -154,7 +158,7 @@ public static boolean hasToken(String urlString) { URL url = new URL(urlString); List params = URLEncodedUtils.parse(url.getQuery(), Charset.forName("UTF-8")); for (NameValuePair nvp : params) { - if (nvp.getName().equals("token")) { + if (nvp.getName().equals(SIGNED_URL_TOKEN)) { return true; } } diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java index cdefc844c03..70393ebcb2b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonUtil; +import edu.harvard.iq.dataverse.util.testing.SystemProperty; import javax.json.Json; import javax.json.JsonObject; @@ -206,8 +207,8 @@ public void testGetToolUrlWithOptionalQueryParameters() { } @Test + @SystemProperty(key = SystemConfig.SITE_URL, value = "https://librascholar.org") public void testGetToolUrlWithAllowedApiCalls() { - String oldVal = System.setProperty(SystemConfig.SITE_URL, "https://librascholar.org"); System.out.println("allowedApiCalls test"); Dataset ds = new Dataset(); ds.setId(1L); @@ -233,10 +234,5 @@ public void testGetToolUrlWithAllowedApiCalls() { assertTrue(signedUrl.contains("&method=GET")); assertTrue(signedUrl.contains("&token=")); System.out.println(JsonUtil.prettyPrint(jo)); - if(oldVal==null) { - System.clearProperty(SystemConfig.SITE_URL); - } else { - System.setProperty(SystemConfig.SITE_URL, oldVal); - } } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java index 68e1a9291ee..5e5c14ed063 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java @@ -6,19 +6,19 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.util.testing.SystemProperty; + import static org.junit.Assert.assertEquals; import java.util.ArrayList; import java.util.List; import org.junit.Test; -import org.mockito.Mockito; public class UrlTokenUtilTest { @Test + @SystemProperty(key = SystemConfig.SITE_URL, value = "https://librascholar.org") public void testGetToolUrlWithOptionalQueryParameters() { - - String oldVal = System.setProperty(SystemConfig.SITE_URL, "https://librascholar.org"); DataFile dataFile = new DataFile(); dataFile.setId(42l); FileMetadata fmd = new FileMetadata(); @@ -43,10 +43,5 @@ public void testGetToolUrlWithOptionalQueryParameters() { URLTokenUtil urlTokenUtil2 = new URLTokenUtil(ds, apiToken, "en"); assertEquals("https://librascholar.org/api/datasets/50?key=" + apiToken.getTokenString(), urlTokenUtil2.replaceTokensWithValues("{siteUrl}/api/datasets/{datasetId}?key={apiToken}")); assertEquals("https://librascholar.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2ABCDEF&key=" + apiToken.getTokenString(), urlTokenUtil2.replaceTokensWithValues("{siteUrl}/api/datasets/:persistentId/?persistentId={datasetPid}&key={apiToken}")); - if(oldVal==null) { - System.clearProperty(SystemConfig.SITE_URL); - } else { - System.setProperty(SystemConfig.SITE_URL, oldVal); - } } } From 4a50fca92c95f1ad5f84f98e9a0cd5ed82a88c86 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 11 Nov 2022 19:25:55 +0100 Subject: [PATCH 159/232] docs(storage): add some detailed notes about temporary upload file storage #6656 --- .../source/installation/config.rst | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 813fa9b139b..b225594ec3b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -700,6 +700,26 @@ Once you have configured a trusted remote store, you can point your users to the =========================================== ================== ========================================================================== =================== +.. _temporary-file-storage: + +Temporary Upload File Storage ++++++++++++++++++++++++++++++ + +When uploading files via the API or Web UI, you need to be aware that multiple steps are involved to enable +features like ingest processing, transfer to a permanent storage, checking for duplicates, unzipping etc. + +All of these processes are triggered after finishing transfers over the wire and moving the data into a temporary +(configurable) location on disk at :ref:`${dataverse.files.directory} `\ ``/temp``. + +Before being moved there, + +- JSF Web UI uploads are stored at :ref:`${dataverse.files.uploads} `, defaulting to + ``/usr/local/payara5/glassfish/domains/domain1/uploads`` folder in a standard installation. This place is + configurable and might be set to a separate disk volume, swiped regularly for leftovers. +- API uploads are stored at the system's temporary files location indicated by the Java system property + ``java.io.tmpdir``, defaulting to ``/tmp`` on Linux. If this location is backed by a `tmpfs `_ + on your machine, large file uploads via API will cause RAM and/or swap usage bursts. You might want to point this to + a different location, restrict maximum size of it and monitor for leftovers. .. _Branding Your Installation: @@ -1412,15 +1432,20 @@ Note that it's also possible to use the ``dataverse.fqdn`` as a variable, if you We are absolutely aware that it's confusing to have both ``dataverse.fqdn`` and ``dataverse.siteUrl``. https://github.com/IQSS/dataverse/issues/6636 is about resolving this confusion. +.. _dataverse.files.directory: + dataverse.files.directory +++++++++++++++++++++++++ This is how you configure the path Dataverse uses for temporary files. (File store specific dataverse.files.\.directory options set the permanent data storage locations.) +.. _dataverse.files.uploads: + dataverse.files.uploads +++++++++++++++++++++++ Configure a folder to store the incoming file stream during uploads (before transfering to `${dataverse.files.directory}/temp`). +Please also see :ref:`temporary-file-storage` for more details. You can use an absolute path or a relative, which is relative to the application server domain directory. Defaults to ``./uploads``, which resolves to ``/usr/local/payara5/glassfish/domains/domain1/uploads`` in a default From ac600bf97fead632bd94cefdfe64977b3b8f8822 Mon Sep 17 00:00:00 2001 From: Pietro Monticone <38562595+pitmonticone@users.noreply.github.com> Date: Sat, 12 Nov 2022 20:06:58 +0100 Subject: [PATCH 160/232] Fix a few typos --- doc/JAVADOC_GUIDE.md | 2 +- doc/mergeParty/readme.md | 4 ++-- doc/sphinx-guides/SphinxRSTCheatSheet.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/JAVADOC_GUIDE.md b/doc/JAVADOC_GUIDE.md index 8001abda248..997c40e1624 100644 --- a/doc/JAVADOC_GUIDE.md +++ b/doc/JAVADOC_GUIDE.md @@ -88,7 +88,7 @@ Here's a better approach: /** The dataverse we move the dataset from */ private Dataverse sourceDataverse; - /** The dataverse we movet the dataset to */ + /** The dataverse we move the dataset to */ private Dataverse destinationDataverse; diff --git a/doc/mergeParty/readme.md b/doc/mergeParty/readme.md index 061673fffa0..6f3af8511dc 100644 --- a/doc/mergeParty/readme.md +++ b/doc/mergeParty/readme.md @@ -73,10 +73,10 @@ Note that before we were asking `isGuest` and now we ask `isAuthenticated`, so t ## Other Added Things ### Settings bean -Settings (in `edu.harvard.iq.dataverse.settings`) are where the application stores its more complex, admin-editable configuration. Technically, its a persistent `Map`, that can be accessed via API (`edu.harvard.iq.dataverse.api.Admin`, on path `{server}/api/s/settings`). Currenly used for the signup mechanism. +Settings (in `edu.harvard.iq.dataverse.settings`) are where the application stores its more complex, admin-editable configuration. Technically, its a persistent `Map`, that can be accessed via API (`edu.harvard.iq.dataverse.api.Admin`, on path `{server}/api/s/settings`). Currently used for the signup mechanism. ### Admin API -Accessible under url `{server}/api/s/`, API calls to this bean should be editing confugurations, allowing full indexing and more. The idea behing putting all of them under the `/s/` path is that we can later block these calls using a filter. This way, we could, say, allow access from localhost only. Or, we could block this completely based on some environemnt variable. +Accessible under url `{server}/api/s/`, API calls to this bean should be editing configurations, allowing full indexing and more. The idea behind putting all of them under the `/s/` path is that we can later block these calls using a filter. This way, we could, say, allow access from localhost only. Or, we could block this completely based on some environment variable. ### `setup-all.sh` script A new script that sets up the users and the dataverses, sets the system up for built-in signup, and then indexes the dataverses using solr. Requires the [jq utility](http://stedolan.github.io/jq/). On Macs with [homebrew](http://brew.sh) installed, getting this utility is a `brew install jq` command away. diff --git a/doc/sphinx-guides/SphinxRSTCheatSheet.md b/doc/sphinx-guides/SphinxRSTCheatSheet.md index 1ccd293080c..300260cb5b1 100755 --- a/doc/sphinx-guides/SphinxRSTCheatSheet.md +++ b/doc/sphinx-guides/SphinxRSTCheatSheet.md @@ -10,7 +10,7 @@ RST Cheat Sheet for Sphinx v 1.2.2 | Bold text | **text** | | | Italics/emphasis | *text* | | | literal | ``literal`` | | -| Internal cross-reference link | See section 5.3.1 of Sphinx documentationand example below | See section 5.3.1 of Sphinx documentationand example below | +| Internal cross-reference link | See section 5.3.1 of Sphinx documentation and example below | See section 5.3.1 of Sphinx documentation and example below | | code block | .. code-block:: guess | Allows for code blocks to be displayed properly | For more cheats please visit the [RST cheat sheet google doc] (https://docs.google.com/document/d/105H3iwPwgnPqwuMJI7q-h6FLtXV_EUCiwq2P13lADgA/edit?usp=sharing) \ No newline at end of file From aa16553b255849ae5c1ef96aef0d06161b3dcc60 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 14 Nov 2022 14:33:28 -0500 Subject: [PATCH 161/232] Create 8838-cstr.md Adding release notes for addition of CSRT PID type. --- doc/release-notes/8838-cstr.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/8838-cstr.md diff --git a/doc/release-notes/8838-cstr.md b/doc/release-notes/8838-cstr.md new file mode 100644 index 00000000000..6434155a18c --- /dev/null +++ b/doc/release-notes/8838-cstr.md @@ -0,0 +1 @@ +A persistent identifier, CSRT, is added to the Related Publication field's ID Type field, and for datasets published with CSRT IDs, Dataverse will include them in the datasets' Schema.org metadata exports. \ No newline at end of file From 3ca13b339ff4cb4ad5b295cc868b20e71f4920df Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 14 Nov 2022 14:39:01 -0500 Subject: [PATCH 162/232] Update 8838-cstr.md Adding a heading --- doc/release-notes/8838-cstr.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/8838-cstr.md b/doc/release-notes/8838-cstr.md index 6434155a18c..c50cd97ccd1 100644 --- a/doc/release-notes/8838-cstr.md +++ b/doc/release-notes/8838-cstr.md @@ -1 +1,3 @@ +### CSRT PID Types Added to Related Publication ID Type field + A persistent identifier, CSRT, is added to the Related Publication field's ID Type field, and for datasets published with CSRT IDs, Dataverse will include them in the datasets' Schema.org metadata exports. \ No newline at end of file From 9cc059fc96d1eea8be73933bebc0dbb32172478d Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 14 Nov 2022 14:56:40 -0500 Subject: [PATCH 163/232] Update 8838-cstr.md Adding required steps for updating the Citation metadata block to include the new ID type --- doc/release-notes/8838-cstr.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/8838-cstr.md b/doc/release-notes/8838-cstr.md index c50cd97ccd1..3527dee3770 100644 --- a/doc/release-notes/8838-cstr.md +++ b/doc/release-notes/8838-cstr.md @@ -1,3 +1,12 @@ ### CSRT PID Types Added to Related Publication ID Type field -A persistent identifier, CSRT, is added to the Related Publication field's ID Type field, and for datasets published with CSRT IDs, Dataverse will include them in the datasets' Schema.org metadata exports. \ No newline at end of file +A persistent identifier, [CSRT](https://www.cstr.cn/search/specification/), is added to the Related Publication field's ID Type child field. For datasets published with CSRT IDs, Dataverse will also include them in the datasets' Schema.org metadata exports. + +The CSRT + +### Required Upgrade Steps + +Update the Citation metadata block: + +- `wget https://github.com/IQSS/dataverse/releases/download/v#.##/citation.tsv` +- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` \ No newline at end of file From f779acf5eba5f054dc5cfe2606c51002fd5fbfa1 Mon Sep 17 00:00:00 2001 From: Julian Gautier Date: Mon, 14 Nov 2022 15:10:07 -0500 Subject: [PATCH 164/232] Update 8838-cstr.md --- doc/release-notes/8838-cstr.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/8838-cstr.md b/doc/release-notes/8838-cstr.md index 3527dee3770..d6bcd33f412 100644 --- a/doc/release-notes/8838-cstr.md +++ b/doc/release-notes/8838-cstr.md @@ -9,4 +9,5 @@ The CSRT Update the Citation metadata block: - `wget https://github.com/IQSS/dataverse/releases/download/v#.##/citation.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` \ No newline at end of file +- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` +- Add the updated citation.properties file to the appropriate directory \ No newline at end of file From cda90ada60c58c99a9e9680821c5771139d3a0ac Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 15 Nov 2022 17:47:09 -0500 Subject: [PATCH 165/232] Changes per request. --- doc/release-notes/7715-signed-urls-for-external-tools.md | 2 +- .../db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/7715-signed-urls-for-external-tools.md b/doc/release-notes/7715-signed-urls-for-external-tools.md index 00b5cff24b3..c2d3859c053 100644 --- a/doc/release-notes/7715-signed-urls-for-external-tools.md +++ b/doc/release-notes/7715-signed-urls-for-external-tools.md @@ -1,3 +1,3 @@ # Improved Security for External Tools -This release adds support for configuring external tools to use signed URLs to access the Dataverse API. This eliminates the need for tools to have access to the user's apiToken in order to access draft or restricted datasets and datafiles. \ No newline at end of file +This release adds support for configuring external tools to use signed URLs to access the Dataverse API. This eliminates the need for tools to have access to the user's apiToken in order to access draft or restricted datasets and datafiles. Signed URLS can be transferred via POST or via a callback when triggering a tool via GET. \ No newline at end of file diff --git a/src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql b/src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql index b47529800d3..5e13de057dd 100644 --- a/src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql +++ b/src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql @@ -1 +1 @@ -ALTER TABLE externaltool ADD COLUMN IF NOT EXISTS allowedapicalls VARCHAR; +ALTER TABLE externaltool ADD COLUMN IF NOT EXISTS allowedapicalls TEXT; From 6b78dc562e9ab9ee6720e49615ee1c70180df389 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 16 Nov 2022 10:54:16 +0100 Subject: [PATCH 166/232] drop sortorder column in integration test --- .../resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql index 6fe3f1142c2..b5b76628e14 100644 --- a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql +++ b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql @@ -1,3 +1,4 @@ +ALTER TABLE license DROP COLUMN sortorder; ALTER TABLE license ADD COLUMN IF NOT EXISTS sortorder BIGINT NOT NULL DEFAULT 0; From ff392049595aaaf1a1d242cf8bfe3de13649a31a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 16 Nov 2022 10:55:37 +0100 Subject: [PATCH 167/232] revert de column drop --- .../resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql index b5b76628e14..6fe3f1142c2 100644 --- a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql +++ b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql @@ -1,4 +1,3 @@ -ALTER TABLE license DROP COLUMN sortorder; ALTER TABLE license ADD COLUMN IF NOT EXISTS sortorder BIGINT NOT NULL DEFAULT 0; From 08c7e88d8f12a6cc5ff40490f05db09cebd5de91 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 16 Nov 2022 10:59:03 +0100 Subject: [PATCH 168/232] moved drop sortorder column to the right file --- .../db/migration/V5.9.0.1__7440-configurable-license-list.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql index cb76b2270a4..a6b5f55f70c 100644 --- a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql +++ b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql @@ -1,3 +1,4 @@ +ALTER TABLE license DROP COLUMN sortorder; ALTER TABLE termsofuseandaccess ADD COLUMN IF NOT EXISTS license_id BIGINT; DO $$ From b50081d3aafcadc2fe211636bbbee2f63c800ffb Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 16 Nov 2022 10:59:31 +0100 Subject: [PATCH 169/232] revert de column drop --- .../db/migration/V5.9.0.1__7440-configurable-license-list.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql index a6b5f55f70c..cb76b2270a4 100644 --- a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql +++ b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql @@ -1,4 +1,3 @@ -ALTER TABLE license DROP COLUMN sortorder; ALTER TABLE termsofuseandaccess ADD COLUMN IF NOT EXISTS license_id BIGINT; DO $$ From cbe1092434a1ddb21da522db43d42682c4490d0d Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 16 Nov 2022 11:13:49 +0100 Subject: [PATCH 170/232] drop colomn sort order if exists --- .../resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql | 1 + .../db/migration/V5.9.0.1__7440-configurable-license-list.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql index 6fe3f1142c2..11338afc0f3 100644 --- a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql +++ b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql @@ -1,3 +1,4 @@ +ALTER TABLE license DROP COLUMN IF EXISTS sortorder; ALTER TABLE license ADD COLUMN IF NOT EXISTS sortorder BIGINT NOT NULL DEFAULT 0; diff --git a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql index cb76b2270a4..2772f4b656b 100644 --- a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql +++ b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql @@ -2,7 +2,7 @@ ALTER TABLE termsofuseandaccess ADD COLUMN IF NOT EXISTS license_id BIGINT; DO $$ BEGIN - + ALTER TABLE license DROP COLUMN IF EXISTS sortorder; BEGIN ALTER TABLE termsofuseandaccess ADD CONSTRAINT fk_termsofuseandcesss_license_id foreign key (license_id) REFERENCES license(id); EXCEPTION From 98d697411e3a3ade9ab9fe183543ba1f29952466 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 16 Nov 2022 11:15:29 +0100 Subject: [PATCH 171/232] revert dropping --- .../resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql | 1 - .../db/migration/V5.9.0.1__7440-configurable-license-list.sql | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql index 11338afc0f3..6fe3f1142c2 100644 --- a/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql +++ b/src/main/resources/db/migration/V5.13.0.1__8671-sorting_licenses.sql @@ -1,4 +1,3 @@ -ALTER TABLE license DROP COLUMN IF EXISTS sortorder; ALTER TABLE license ADD COLUMN IF NOT EXISTS sortorder BIGINT NOT NULL DEFAULT 0; diff --git a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql index 2772f4b656b..a8f7f41e2ef 100644 --- a/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql +++ b/src/main/resources/db/migration/V5.9.0.1__7440-configurable-license-list.sql @@ -2,7 +2,6 @@ ALTER TABLE termsofuseandaccess ADD COLUMN IF NOT EXISTS license_id BIGINT; DO $$ BEGIN - ALTER TABLE license DROP COLUMN IF EXISTS sortorder; BEGIN ALTER TABLE termsofuseandaccess ADD CONSTRAINT fk_termsofuseandcesss_license_id foreign key (license_id) REFERENCES license(id); EXCEPTION From d1a56800ce504e2c4ef42ebea4e27de8eb1ef7ef Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 16 Nov 2022 14:01:36 +0100 Subject: [PATCH 172/232] feat: make API signing secret a JvmSetting #7715 --- .../java/edu/harvard/iq/dataverse/api/AbstractApiBean.java | 3 ++- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 3 ++- .../iq/dataverse/externaltools/ExternalToolHandler.java | 5 +++-- .../java/edu/harvard/iq/dataverse/settings/JvmSettings.java | 4 ++++ .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 5 ----- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index a33635fe0d1..e919ecf786d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -46,6 +46,7 @@ import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; import edu.harvard.iq.dataverse.locality.StorageSiteServiceBean; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -440,7 +441,7 @@ private AuthenticatedUser getAuthenticatedUserFromSignedUrl() { // ToDo - add null checks/ verify that calling methods catch things. String user = httpRequest.getParameter("user"); AuthenticatedUser targetUser = authSvc.getAuthenticatedUser(user); - String key = System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + String key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + authSvc.findApiTokenByUser(targetUser).getTokenString(); String signedUrl = httpRequest.getRequestURL().toString() + "?" + httpRequest.getQueryString(); String method = httpRequest.getMethod(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index b435f23f52c..31a874f85c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -14,6 +14,7 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DataverseSession; import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.GlobalId; @@ -2277,7 +2278,7 @@ public Response getSignedUrl(JsonObject urlInfo) throws WrappedResponse { if (key == null) { return error(Response.Status.CONFLICT, "Do not have a valid user with apiToken"); } - key = System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + key; + key = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + key; } String baseUrl = urlInfo.getString("url"); diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index f55439d23a9..542c6e2cd26 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.URLTokenUtil; @@ -117,7 +118,7 @@ public String handleRequest(boolean preview) { } if (apiToken != null) { callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(), HttpMethod.GET, - System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + apiToken.getTokenString()); + JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString()); } paramsString= "?callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback)); if (getLocaleCode() != null) { @@ -189,7 +190,7 @@ public JsonObjectBuilder createPostBody(JsonObject params) { ApiToken apiToken = getApiToken(); if (apiToken != null) { url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(), httpmethod, - System.getProperty(SystemConfig.API_SIGNING_SECRET, "") + getApiToken().getTokenString()); + JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + getApiToken().getTokenString()); } logger.fine("Signed URL: " + url); apisBuilder.add(Json.createObjectBuilder().add(NAME, name).add(HTTP_METHOD, httpmethod) diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 223e4b86da9..e409607346b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -42,6 +42,10 @@ public enum JvmSettings { VERSION(PREFIX, "version"), BUILD(PREFIX, "build"), + // API SETTINGS + SCOPE_API(PREFIX, "api"), + API_SIGNING_SECRET(SCOPE_API, "signing-secret"), + ; private static final String SCOPE_SEPARATOR = "."; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 1fa3bd6a82b..80af2df081c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -122,11 +122,6 @@ public class SystemConfig { public final static String DEFAULTCURATIONLABELSET = "DEFAULT"; public final static String CURATIONLABELSDISABLED = "DISABLED"; - // A secret used in signing URLs - individual urls are signed using this and the - // intended user's apiKey, creating an aggregate key that is unique to the user - // but not known to the user (as their apiKey is) - public final static String API_SIGNING_SECRET = "dataverse.api-signing-secret"; - public String getVersion() { return getVersion(false); } From bb1865cf4e0ba19e36f96b6e6cf6c646fa6ae52c Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 16 Nov 2022 14:47:45 +0100 Subject: [PATCH 173/232] sortorder colomn definition fix in entity bean --- src/main/java/edu/harvard/iq/dataverse/license/License.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/license/License.java b/src/main/java/edu/harvard/iq/dataverse/license/License.java index 3073291a9d5..c6e2cdbc2e5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/license/License.java +++ b/src/main/java/edu/harvard/iq/dataverse/license/License.java @@ -76,7 +76,7 @@ public class License { @Column(nullable = false) private boolean isDefault; - @Column(nullable = false) + @Column(nullable = false, columnDefinition = "BIGINT NOT NULL DEFAULT 0") private Long sortOrder; @OneToMany(mappedBy="license") From af976f54865708ca5102cd2c73ab8d141b777fef Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 16 Nov 2022 17:11:01 -0500 Subject: [PATCH 174/232] some straightforward fixes/cleanup for the create harvest client api (#8290) --- .../harvard/iq/dataverse/api/HarvestingClients.java | 10 ++++++---- .../edu/harvard/iq/dataverse/util/json/JsonParser.java | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index d17e76c499a..d7b35a0357a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -32,7 +32,8 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; -@Stateless +// huh, why was this api @Stateless?? +//@Stateless @Path("harvest/clients") public class HarvestingClients extends AbstractApiBean { @@ -162,10 +163,10 @@ public Response createHarvestingClient(String jsonBody, @PathParam("nickName") S ownerDataverse.setHarvestingClientConfigs(new ArrayList<>()); } ownerDataverse.getHarvestingClientConfigs().add(harvestingClient); - + DataverseRequest req = createDataverseRequest(findUserOrDie()); - HarvestingClient managedHarvestingClient = execCommand( new CreateHarvestingClientCommand(req, harvestingClient)); - return created( "/harvest/clients/" + nickName, harvestingConfigAsJson(managedHarvestingClient)); + harvestingClient = execCommand(new CreateHarvestingClientCommand(req, harvestingClient)); + return created( "/harvest/clients/" + nickName, harvestingConfigAsJson(harvestingClient)); } catch (JsonParseException ex) { return error( Response.Status.BAD_REQUEST, "Error parsing harvesting client: " + ex.getMessage() ); @@ -296,6 +297,7 @@ public static JsonObjectBuilder harvestingConfigAsJson(HarvestingClient harvesti return jsonObjectBuilder().add("nickName", harvestingConfig.getName()). add("dataverseAlias", harvestingConfig.getDataverse().getAlias()). add("type", harvestingConfig.getHarvestType()). + add("style", harvestingConfig.getHarvestStyle()). add("harvestUrl", harvestingConfig.getHarvestingUrl()). add("archiveUrl", harvestingConfig.getArchiveUrl()). add("archiveDescription",harvestingConfig.getArchiveDescription()). diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 4ecdc73ae6e..54b16596ab4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -903,6 +903,7 @@ public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingC harvestingClient.setName(obj.getString("nickName",null)); harvestingClient.setHarvestType(obj.getString("type",null)); + harvestingClient.setHarvestStyle(obj.getString("style", "default")); harvestingClient.setHarvestingUrl(obj.getString("harvestUrl",null)); harvestingClient.setArchiveUrl(obj.getString("archiveUrl",null)); harvestingClient.setArchiveDescription(obj.getString("archiveDescription")); From 79f4c8549b08203db0e50cfd3fbd26e6552bbf51 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 17 Nov 2022 00:13:08 +0100 Subject: [PATCH 175/232] doc(signed-url): add configuration documentation #7715 --- .../source/api/external-tools.rst | 2 + doc/sphinx-guides/source/api/native-api.rst | 5 ++- .../source/installation/config.rst | 37 ++++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst index a8f83590871..4f6c9a8015c 100644 --- a/doc/sphinx-guides/source/api/external-tools.rst +++ b/doc/sphinx-guides/source/api/external-tools.rst @@ -143,6 +143,8 @@ Reserved Words ``{localeCode}`` optional The code for the language ("en" for English, "fr" for French, etc.) that user has selected from the language toggle in a Dataverse installation. See also :ref:`i18n`. =========================== ========== =========== +.. _api-exttools-auth: + Authorization Options +++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index de727592694..e48661929c7 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4106,7 +4106,8 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash curl -X DELETE https://demo.dataverse.org/api/admin/template/24 - + +.. _api-native-signed-url: Request Signed URL ~~~~~~~~~~~~~~~~~~ @@ -4133,3 +4134,5 @@ A curl example using allowing access to a dataset's metadata curl -H 'X-Dataverse-key:$API_KEY' -d $JSON $SERVER_URL/api/admin/requestSignedUrl +Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra +security. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index c61bf451eb7..8767ef14c6a 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -580,8 +580,7 @@ Optionally, you may provide static credentials for each S3 storage using MicroPr - ``dataverse.files..access-key`` for this storage's "access key ID" - ``dataverse.files..secret-key`` for this storage's "secret access key" -You may provide the values for these via any of the -`supported config sources `_. +You may provide the values for these via any `supported MicroProfile Config API source`_. **WARNING:** @@ -1670,6 +1669,36 @@ This setting is useful in cases such as running your Dataverse installation behi "HTTP_VIA", "REMOTE_ADDR" + +.. _dataverse.api.signature-secret: + +dataverse.api.signature-secret +++++++++++++++++++++++++++++++ + +Context: Dataverse has the ability to create "Signed URLs" for it's API calls. Using a signed URL makes it obsolete to +provide API tokens to tools, which carries the risk of leaking extremely sensitive information on exposure. Signed URLs +can be limited to certain allowed actions, which is much more secure. See :ref:`api-exttools-auth` and +:ref:`api-native-signed-url` for more details. The key to sign a URL is created from the secret API token of the +creating user plus a shared secret provided by an administrator. + +This setting will default to an empty string, but you should provide it for extra security. + +Here is an example how to set your shared secret with the secure method "password alias": + +.. code-block:: shell + + echo "AS_ADMIN_ALIASPASSWORD=change-me-super-secret" > /tmp/password.txt + asadmin create-password-alias --passwordfile /tmp/password.txt dataverse.api.signature-secret + rm /tmp/password.txt + +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable +``DATAVERSE_API_SIGNATURE_SECRET``. + +**WARNING:** For security, do not use the sources "environment variable" or "system property" (JVM option) in a +production context! Rely on password alias, secrets directory or cloud based sources instead! + + + .. _:ApplicationServerSettings: Application Server Settings @@ -3067,3 +3096,7 @@ The interval in seconds between Dataverse calls to Globus to check on upload pro +++++++++++++++++++++++++ A true/false option to add a Globus transfer option to the file download menu which is not yet fully supported in the dataverse-globus app. See :ref:`globus-support` for details. + + + +.. _supported MicroProfile Config API source: https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Config/Overview.html From 33a5cee41720dd3c84fd8a006d8b82f3f08d0b49 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 17 Nov 2022 11:01:45 -0500 Subject: [PATCH 176/232] doc entries for the previously undocumented harvesting clients api (#8290) --- doc/sphinx-guides/source/api/native-api.rst | 99 +++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6d68d648cb3..54bbe67ce1e 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3200,6 +3200,105 @@ The fully expanded example above (without the environment variables) looks like Only users with superuser permissions may delete harvesting sets. +Managing Harvesting Clients +--------------------------- + +The following API can be used to create and manage "Harvesting Clients". A Harvesting Client is a configuration entry that allows your Dataverse installation to harvest and index metadata from a specific remote location, either regularly, on a configured schedule, or on a one-off basis. For more information, see the :doc:`/admin/harvestclients` section of the Admin Guide. + +List All Congigured Harvesting Clients +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Shows all the Harvesting Clients configured:: + + GET http://$SERVER/api/harvest/clients/ + +Show a specific Harvesting Client +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Shows a Harvesting Client with a defined nickname:: + + GET http://$SERVER/api/harvest/clients/$nickname + +.. code-block:: bash + + curl "http://localhost:8080/api/harvest/clients/myclient" + + { + "status":"OK", + { + "data": { + "lastDatasetsFailed": "22", + "lastDatasetsDeleted": "0", + "metadataFormat": "oai_dc", + "archiveDescription": "This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data.", + "archiveUrl": "https://dataverse.foo.edu", + "harvestUrl": "https://dataverse.foo.edu/oai", + "style": "dataverse", + "type": "oai", + "dataverseAlias": "fooData", + "nickName": "myClient", + "set": "fooSet", + "schedule": "none", + "status": "inActive", + "lastHarvest": "Thu Oct 13 14:48:57 EDT 2022", + "lastResult": "SUCCESS", + "lastSuccessful": "Thu Oct 13 14:48:57 EDT 2022", + "lastNonEmpty": "Thu Oct 13 14:48:57 EDT 2022", + "lastDatasetsHarvested": "137" + } + } + + +Create a Harvesting Client +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To create a new harvesting client you must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: + +- nickName: Alpha-numeric may also contain -, _, or %, but no spaces. Must also be unique in the installation. Must match the nickName in the Path +- dataverseAlias: The alias of an existing collection where harvested datasets will be deposited. +- dataverseAlias: The alias of an existing collection where harvested datasets will be deposited. +- harvestUrl: The URL of the remote OAI archive +- archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://dataverse.harvard.edu/ vs. https://dataverse.harvard.edu/oai. +- metadataFormat: A supported metadata format. For example, "oai_dc" or "ddi". + +The following optional fields are supported: + +- archiveDescription: What the name suggests. If not supplied, will default to "This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data." +- set: The OAI set on the remote server. If not supplied, will default to none, i.e., "harvest everything". +- schedule: Harvesting schedule. Defaults to "none". +- style: Defaults to "default" - a generic OAI archive. (Make sure to use "dataverse" when configuring harvesting from another Dataverse installation). + +An example JSON file would look like this:: + + { + "nickName": "zenodo", + "dataverseAlias": "zenodoHarvested", + "type": "oai", + "harvestUrl": "https://zenodo.org/oai2d", + "archiveUrl": "https://zenodo.org", + "archiveDescription": "Moissonné depuis la collection LMOPS de l'entrepôt Zenodo. En cliquant sur ce jeu de données, vous serez redirigé vers Zenodo.", + "metadataFormat": "oai_dc", + "set": "user-lmops" + } + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=http://localhost:8080 + + curl -H X-Dataverse-key:$API_TOKEN -X POST "$SERVER_URL/api/harvest/clients/zenodo" --upload-file client.json + +The fully expanded example above (without the environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "client.json" + +Only users with superuser permissions may create or configure harvesting clients. + + PIDs ---- From c86c5137cc98d93ad3616fcf062cb222ae5240fc Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Thu, 17 Nov 2022 17:08:33 +0100 Subject: [PATCH 177/232] Fix link to example JSON dataset in API docs --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6d68d648cb3..ec3c9b5dc59 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -526,7 +526,7 @@ To create a dataset, you must supply a JSON file that contains at least the foll - Description Text - Subject -As a starting point, you can download :download:`dataset-finch1.json <../../../../scripts/search/tests/data/dataset-finch1.json>` and modify it to meet your needs. (:download:`dataset-create-new-all-default-fields.json <../../../../scripts/api/data/dataset-finch1_fr.json>` is a variant of this file that includes setting the metadata language (see :ref:`:MetadataLanguages`) to French (fr). In addition to this minimal example, you can download :download:`dataset-create-new-all-default-fields.json <../../../../scripts/api/data/dataset-create-new-all-default-fields.json>` which populates all of the metadata fields that ship with a Dataverse installation.) +As a starting point, you can download :download:`dataset-finch1.json <../../../../scripts/search/tests/data/dataset-finch1.json>` and modify it to meet your needs. (:download:`dataset-finch1_fr.json <../../../../scripts/api/data/dataset-finch1_fr.json>` is a variant of this file that includes setting the metadata language (see :ref:`:MetadataLanguages`) to French (fr). In addition to this minimal example, you can download :download:`dataset-create-new-all-default-fields.json <../../../../scripts/api/data/dataset-create-new-all-default-fields.json>` which populates all of the metadata fields that ship with a Dataverse installation.) The curl command below assumes you have kept the name "dataset-finch1.json" and that this file is in your current working directory. From eded574cf930552b55b578404c6ba03e969d9f00 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 17 Nov 2022 12:00:20 -0500 Subject: [PATCH 178/232] more doc. corrections #8290 --- doc/sphinx-guides/source/api/native-api.rst | 28 ++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 54bbe67ce1e..e33b502f43c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3258,7 +3258,7 @@ To create a new harvesting client you must supply a JSON file that describes the - dataverseAlias: The alias of an existing collection where harvested datasets will be deposited. - dataverseAlias: The alias of an existing collection where harvested datasets will be deposited. - harvestUrl: The URL of the remote OAI archive -- archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://dataverse.harvard.edu/ vs. https://dataverse.harvard.edu/oai. +- archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai. - metadataFormat: A supported metadata format. For example, "oai_dc" or "ddi". The following optional fields are supported: @@ -3296,8 +3296,34 @@ The fully expanded example above (without the environment variables) looks like curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "client.json" + { + "status": "OK", + "data": { + "metadataFormat": "oai_dc", + "archiveDescription": "Moissonné depuis la collection LMOPS de l'entrepôt Zenodo. En cliquant sur ce jeu de données, vous serez redirigé vers Zenodo.", + "archiveUrl": "https://zenodo.org", + "harvestUrl": "https://zenodo.org/oai2d", + "style": "default", + "type": "oai", + "dataverseAlias": "zenodoHarvested", + "nickName": "zenodo", + "set": "user-lmops", + "schedule": "none", + "status": "inActive", + "lastHarvest": "N/A", + "lastSuccessful": "N/A", + "lastNonEmpty": "N/A", + "lastDatasetsHarvested": "N/A", + "lastDatasetsDeleted": "N/A" + } + } + Only users with superuser permissions may create or configure harvesting clients. +Create a Harvesting Client +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to the API above, using the same JSON format, but run on an existing client and using the PUT method instead of POST. PIDs ---- From b2e3a40753c3b47ecbc2d851a15269a927a551b7 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 17 Nov 2022 12:04:44 -0500 Subject: [PATCH 179/232] correction (#8290) --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index e33b502f43c..e06ab5111e3 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3320,7 +3320,7 @@ The fully expanded example above (without the environment variables) looks like Only users with superuser permissions may create or configure harvesting clients. -Create a Harvesting Client +Modify a Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~ Similar to the API above, using the same JSON format, but run on an existing client and using the PUT method instead of POST. From 5445a85a3907062ff37145fdc0b05d5191b7804b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 17 Nov 2022 16:58:11 -0500 Subject: [PATCH 180/232] wordsmithing --- .../source/installation/config.rst | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8767ef14c6a..43ad63a850c 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1675,15 +1675,18 @@ This setting is useful in cases such as running your Dataverse installation behi dataverse.api.signature-secret ++++++++++++++++++++++++++++++ -Context: Dataverse has the ability to create "Signed URLs" for it's API calls. Using a signed URL makes it obsolete to -provide API tokens to tools, which carries the risk of leaking extremely sensitive information on exposure. Signed URLs -can be limited to certain allowed actions, which is much more secure. See :ref:`api-exttools-auth` and -:ref:`api-native-signed-url` for more details. The key to sign a URL is created from the secret API token of the -creating user plus a shared secret provided by an administrator. - -This setting will default to an empty string, but you should provide it for extra security. - -Here is an example how to set your shared secret with the secure method "password alias": +Context: Dataverse has the ability to create "Signed URLs" for it's API calls. Using a signed URLs is more secure than +providing API tokens, which are long-lived and give the holder all of the permissions of the user. In contrast, signed URLs +are time limited and only allow the action of the API call in the URL. See :ref:`api-exttools-auth` and +:ref:`api-native-signed-url` for more details. + +The key used to sign a URL is created from the API token of the creating user plus a signature-secret provided by an administrator. +**Using a signature-secret is highly recommended.** This setting defaults to an empty string. Using a non-empty +signature-secret makes it impossible for someone who knows an API token from forging signed URLs and provides extra security by +making the overall signing key longer. + +Since the signature-secret is sensitive, you should treat it like a password. Here is an example how to set your shared secret +with the secure method "password alias": .. code-block:: shell From 2ec5f238711efbe12eb8ee82e87216b774ba5f86 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Nov 2022 17:06:48 -0500 Subject: [PATCH 181/232] bug fix for #9056 --- src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index d64a1f7cce1..c563c9af6f8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -68,6 +68,7 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti case WORKFLOW_FAILURE: return BundleUtil.getStringFromBundle("notification.email.workflow.failure.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case STATUSUPDATED: + datasetDisplayName = ((DatasetVersion)objectOfNotification).getDataset().getDisplayName(); return BundleUtil.getStringFromBundle("notification.email.status.change.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case CREATEACC: return BundleUtil.getStringFromBundle("notification.email.create.account.subject", rootDvNameAsList); From 04b3a78f1a7b52685906ac450c7897944a7b8d47 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Nov 2022 17:12:50 -0500 Subject: [PATCH 182/232] fix others --- .../java/edu/harvard/iq/dataverse/util/MailUtil.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index c563c9af6f8..72980c3451a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -34,8 +34,12 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti List rootDvNameAsList = Arrays.asList(BrandingUtil.getInstallationBrandName()); String datasetDisplayName = ""; - if (objectOfNotification != null && (objectOfNotification instanceof Dataset) ) { - datasetDisplayName = ((Dataset)objectOfNotification).getDisplayName(); + if (objectOfNotification != null) { + if (objectOfNotification instanceof Dataset) { + datasetDisplayName = ((Dataset) objectOfNotification).getDisplayName(); + } else if (objectOfNotification instanceof DatasetVersion) { + datasetDisplayName = ((DatasetVersion) objectOfNotification).getDataset().getDisplayName(); + } } switch (userNotification.getType()) { @@ -68,7 +72,6 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti case WORKFLOW_FAILURE: return BundleUtil.getStringFromBundle("notification.email.workflow.failure.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case STATUSUPDATED: - datasetDisplayName = ((DatasetVersion)objectOfNotification).getDataset().getDisplayName(); return BundleUtil.getStringFromBundle("notification.email.status.change.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case CREATEACC: return BundleUtil.getStringFromBundle("notification.email.create.account.subject", rootDvNameAsList); From e986a89b52cea995a89b6ddc36100d272152c0a3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Nov 2022 17:41:07 -0500 Subject: [PATCH 183/232] Revert "fix others" This reverts commit 04b3a78f1a7b52685906ac450c7897944a7b8d47. --- .../java/edu/harvard/iq/dataverse/util/MailUtil.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index 72980c3451a..c563c9af6f8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -34,12 +34,8 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti List rootDvNameAsList = Arrays.asList(BrandingUtil.getInstallationBrandName()); String datasetDisplayName = ""; - if (objectOfNotification != null) { - if (objectOfNotification instanceof Dataset) { - datasetDisplayName = ((Dataset) objectOfNotification).getDisplayName(); - } else if (objectOfNotification instanceof DatasetVersion) { - datasetDisplayName = ((DatasetVersion) objectOfNotification).getDataset().getDisplayName(); - } + if (objectOfNotification != null && (objectOfNotification instanceof Dataset) ) { + datasetDisplayName = ((Dataset)objectOfNotification).getDisplayName(); } switch (userNotification.getType()) { @@ -72,6 +68,7 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti case WORKFLOW_FAILURE: return BundleUtil.getStringFromBundle("notification.email.workflow.failure.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case STATUSUPDATED: + datasetDisplayName = ((DatasetVersion)objectOfNotification).getDataset().getDisplayName(); return BundleUtil.getStringFromBundle("notification.email.status.change.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case CREATEACC: return BundleUtil.getStringFromBundle("notification.email.create.account.subject", rootDvNameAsList); From 803a6e94176a8114a96aa06e8c7369aab658731f Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Nov 2022 17:41:17 -0500 Subject: [PATCH 184/232] Revert "bug fix for #9056" This reverts commit 2ec5f238711efbe12eb8ee82e87216b774ba5f86. --- src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index c563c9af6f8..d64a1f7cce1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -68,7 +68,6 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti case WORKFLOW_FAILURE: return BundleUtil.getStringFromBundle("notification.email.workflow.failure.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case STATUSUPDATED: - datasetDisplayName = ((DatasetVersion)objectOfNotification).getDataset().getDisplayName(); return BundleUtil.getStringFromBundle("notification.email.status.change.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName)); case CREATEACC: return BundleUtil.getStringFromBundle("notification.email.create.account.subject", rootDvNameAsList); From 76202477b5d695be6e88ea532be0f3af1def55a0 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 17 Nov 2022 17:48:11 -0500 Subject: [PATCH 185/232] re-apply message fixes --- src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java index d64a1f7cce1..72980c3451a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java @@ -34,8 +34,12 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti List rootDvNameAsList = Arrays.asList(BrandingUtil.getInstallationBrandName()); String datasetDisplayName = ""; - if (objectOfNotification != null && (objectOfNotification instanceof Dataset) ) { - datasetDisplayName = ((Dataset)objectOfNotification).getDisplayName(); + if (objectOfNotification != null) { + if (objectOfNotification instanceof Dataset) { + datasetDisplayName = ((Dataset) objectOfNotification).getDisplayName(); + } else if (objectOfNotification instanceof DatasetVersion) { + datasetDisplayName = ((DatasetVersion) objectOfNotification).getDataset().getDisplayName(); + } } switch (userNotification.getType()) { From c704b002c810139ca87af697a314e2f155944f7a Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 17 Nov 2022 18:40:45 -0500 Subject: [PATCH 186/232] more fixes (DELETE api, etc.) #8290 --- doc/sphinx-guides/source/api/native-api.rst | 1 - .../iq/dataverse/api/HarvestingClients.java | 141 ++++++++++++++++-- .../iq/dataverse/util/json/JsonParser.java | 3 +- 3 files changed, 130 insertions(+), 15 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index e06ab5111e3..e7715725454 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3256,7 +3256,6 @@ To create a new harvesting client you must supply a JSON file that describes the - nickName: Alpha-numeric may also contain -, _, or %, but no spaces. Must also be unique in the installation. Must match the nickName in the Path - dataverseAlias: The alias of an existing collection where harvested datasets will be deposited. -- dataverseAlias: The alias of an existing collection where harvested datasets will be deposited. - harvestUrl: The URL of the remote OAI archive - archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai. - metadataFormat: A supported metadata format. For example, "oai_dc" or "ddi". diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index d7b35a0357a..5fb47e93f11 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -5,12 +5,14 @@ import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.impl.CreateHarvestingClientCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetHarvestingClientCommand; import edu.harvard.iq.dataverse.engine.command.impl.UpdateHarvestingClientCommand; import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean; import edu.harvard.iq.dataverse.harvest.client.HarvestingClientServiceBean; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import javax.json.JsonObjectBuilder; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; @@ -24,6 +26,7 @@ import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; +import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; @@ -38,8 +41,8 @@ public class HarvestingClients extends AbstractApiBean { - @EJB - DataverseServiceBean dataverseService; + //@EJB + //DataverseServiceBean dataverseService; @EJB HarvesterServiceBean harvesterService; @EJB @@ -112,6 +115,10 @@ public Response harvestingClient(@PathParam("nickName") String nickName, @QueryP return error(Response.Status.NOT_FOUND, "Harvesting client " + nickName + " not found."); } + // See the comment in the harvestingClients() (plural) above for the explanation + // of why we are looking up the client twice (tl;dr: to utilize the + // authorization logic in the command) + HarvestingClient retrievedHarvestingClient = null; try { @@ -144,20 +151,56 @@ public Response harvestingClient(@PathParam("nickName") String nickName, @QueryP @POST @Path("{nickName}") public Response createHarvestingClient(String jsonBody, @PathParam("nickName") String nickName, @QueryParam("key") String apiKey) throws IOException, JsonParseException { - + // Note that we don't check the user's authorization within the API + // method. Insetead, we will end up reporting a "not authorized" + // exception thrown by the Command, if this user has no permission + // to perform the action. + try ( StringReader rdr = new StringReader(jsonBody) ) { JsonObject json = Json.createReader(rdr).readObject(); + // Check that the client with this name doesn't exist yet: + // (we could simply let the command fail, but that does not result + // in a pretty report to the end user) + + HarvestingClient lookedUpClient = null; + try { + lookedUpClient = harvestingClientService.findByNickname(nickName); + } catch (Exception ex) { + logger.warning("Exception caught looking up harvesting client " + nickName + ": " + ex.getMessage()); + // let's hope that this was a fluke of some kind; we'll proceed + // with the attempt to create a new client and report an error + // if that fails too. + } + + if (lookedUpClient != null) { + return error(Response.Status.BAD_REQUEST, "Harvesting client " + nickName + " already exists"); + } + HarvestingClient harvestingClient = new HarvestingClient(); - // TODO: check that it doesn't exist yet... - harvestingClient.setName(nickName); + String dataverseAlias = jsonParser().parseHarvestingClient(json, harvestingClient); - Dataverse ownerDataverse = dataverseService.findByAlias(dataverseAlias); + if (dataverseAlias == null) { + return error(Response.Status.BAD_REQUEST, "dataverseAlias must be supplied"); + } + + // Check if the dataverseAlias supplied is valid, i.e. corresponds + // to an existing dataverse (collection): + Dataverse ownerDataverse = dataverseSvc.findByAlias(dataverseAlias); if (ownerDataverse == null) { return error(Response.Status.BAD_REQUEST, "No such dataverse: " + dataverseAlias); } + // The nickname supplied as part of the Rest path takes precedence: + harvestingClient.setName(nickName); + + if (StringUtil.isEmpty(harvestingClient.getArchiveUrl()) + || StringUtil.isEmpty(harvestingClient.getHarvestingUrl()) + || StringUtil.isEmpty(harvestingClient.getMetadataPrefix())) { + return error(Response.Status.BAD_REQUEST, "Required fields harvestUrl, archiveUrl and metadataFormat must be supplied"); + } + harvestingClient.setDataverse(ownerDataverse); if (ownerDataverse.getHarvestingClientConfigs() == null) { ownerDataverse.setHarvestingClientConfigs(new ArrayList<>()); @@ -199,15 +242,39 @@ public Response modifyHarvestingClient(String jsonBody, @PathParam("nickName") S DataverseRequest req = createDataverseRequest(findUserOrDie()); JsonObject json = Json.createReader(rdr).readObject(); - String newDataverseAlias = jsonParser().parseHarvestingClient(json, harvestingClient); + HarvestingClient newHarvestingClient = new HarvestingClient(); + String newDataverseAlias = jsonParser().parseHarvestingClient(json, newHarvestingClient); if (newDataverseAlias != null && !newDataverseAlias.equals("") && !newDataverseAlias.equals(ownerDataverseAlias)) { return error(Response.Status.BAD_REQUEST, "Bad \"dataverseAlias\" supplied. Harvesting client "+nickName+" belongs to the dataverse "+ownerDataverseAlias); } - HarvestingClient managedHarvestingClient = execCommand( new UpdateHarvestingClientCommand(req, harvestingClient)); - return created( "/datasets/" + nickName, harvestingConfigAsJson(managedHarvestingClient)); + + // Go through the supported editable fields and update the client accordingly: + + if (newHarvestingClient.getHarvestingUrl() != null) { + harvestingClient.setHarvestingUrl(newHarvestingClient.getHarvestingUrl()); + } + if (newHarvestingClient.getHarvestingSet() != null) { + harvestingClient.setHarvestingSet(newHarvestingClient.getHarvestingSet()); + } + if (newHarvestingClient.getMetadataPrefix() != null) { + harvestingClient.setMetadataPrefix(newHarvestingClient.getMetadataPrefix()); + } + if (newHarvestingClient.getArchiveUrl() != null) { + harvestingClient.setArchiveUrl(newHarvestingClient.getArchiveUrl()); + } + if (newHarvestingClient.getArchiveDescription() != null) { + harvestingClient.setArchiveDescription(newHarvestingClient.getArchiveDescription()); + } + if (newHarvestingClient.getHarvestStyle() != null) { + harvestingClient.setHarvestStyle(newHarvestingClient.getHarvestStyle()); + } + // TODO: Make schedule configurable via this API too. + + harvestingClient = execCommand( new UpdateHarvestingClientCommand(req, harvestingClient)); + return ok( "/harvest/clients/" + nickName, harvestingConfigAsJson(harvestingClient)); } catch (JsonParseException ex) { return error( Response.Status.BAD_REQUEST, "Error parsing harvesting client: " + ex.getMessage() ); @@ -219,9 +286,59 @@ public Response modifyHarvestingClient(String jsonBody, @PathParam("nickName") S } - // TODO: - // add a @DELETE method - // (there is already a DeleteHarvestingClient command) + @DELETE + @Path("{nickName}") + public Response deleteHarvestingClient(@PathParam("nickName") String nickName) throws IOException { + // Deleting a client can take a while (if there's a large amnount of + // harvested content associated with it). So instead of calling the command + // directly, we will be calling an async. service bean method. + // Without the command engine taking care of authorization, we'll need + // to check if the user has the right to do this explicitly: + + try { + User u = findUserOrDie(); + if ((!(u instanceof AuthenticatedUser) || !u.isSuperuser())) { + throw new WrappedResponse(error(Response.Status.UNAUTHORIZED, "Only superusers can delete harvesting clients.")); + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + + HarvestingClient harvestingClient = null; + + try { + harvestingClient = harvestingClientService.findByNickname(nickName); + } catch (Exception ex) { + logger.warning("Exception caught looking up harvesting client " + nickName + ": " + ex.getMessage()); + return error( Response.Status.BAD_REQUEST, "Internal error: failed to look up harvesting client " + nickName); + } + + if (harvestingClient == null) { + return error(Response.Status.NOT_FOUND, "Harvesting client " + nickName + " not found."); + } + + // Check if the client is in a state where it can be safely deleted: + + if (harvestingClient.isDeleteInProgress()) { + return error( Response.Status.BAD_REQUEST, "Harvesting client " + nickName + " is already being deleted (in progress)"); + } + + if (harvestingClient.isHarvestingNow()) { + return error( Response.Status.BAD_REQUEST, "It is not safe to delete client " + nickName + " while a harvesting job is in progress"); + } + + // Finally, delete it (asynchronously): + + try { + harvestingClientService.deleteClient(harvestingClient.getId()); + } catch (Exception ex) { + return error( Response.Status.BAD_REQUEST, "Internal error: failed to delete harvesting client " + nickName); + } + + + return ok("Harvesting Client " + nickName + ": delete in progress"); + } + // Methods for managing harvesting runs (jobs): diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 54b16596ab4..3e5c096aff0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -902,11 +902,10 @@ public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingC String dataverseAlias = obj.getString("dataverseAlias",null); harvestingClient.setName(obj.getString("nickName",null)); - harvestingClient.setHarvestType(obj.getString("type",null)); harvestingClient.setHarvestStyle(obj.getString("style", "default")); harvestingClient.setHarvestingUrl(obj.getString("harvestUrl",null)); harvestingClient.setArchiveUrl(obj.getString("archiveUrl",null)); - harvestingClient.setArchiveDescription(obj.getString("archiveDescription")); + harvestingClient.setArchiveDescription(obj.getString("archiveDescription", BundleUtil.getStringFromBundle("harvestclients.viewEditDialog.archiveDescription.default.generic"))); harvestingClient.setMetadataPrefix(obj.getString("metadataFormat",null)); harvestingClient.setHarvestingSet(obj.getString("set",null)); From fda574ac022cd5554874ba640d915cebe0dd96cc Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 18 Nov 2022 10:52:16 -0500 Subject: [PATCH 187/232] More small fixes (#8290) --- .../edu/harvard/iq/dataverse/api/HarvestingClients.java | 9 ++++++--- .../iq/dataverse/harvest/client/HarvestingClient.java | 4 +++- .../edu/harvard/iq/dataverse/util/json/JsonParser.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index 5fb47e93f11..d4c096efdbc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.engine.command.impl.UpdateHarvestingClientCommand; import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean; import edu.harvard.iq.dataverse.harvest.client.HarvestingClientServiceBean; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.json.JsonParseException; import javax.json.JsonObjectBuilder; @@ -22,7 +23,6 @@ import java.util.List; import java.util.logging.Logger; import javax.ejb.EJB; -import javax.ejb.Stateless; import javax.json.Json; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; @@ -35,8 +35,6 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; -// huh, why was this api @Stateless?? -//@Stateless @Path("harvest/clients") public class HarvestingClients extends AbstractApiBean { @@ -195,6 +193,11 @@ public Response createHarvestingClient(String jsonBody, @PathParam("nickName") S // The nickname supplied as part of the Rest path takes precedence: harvestingClient.setName(nickName); + // Populate the description field, if none is supplied: + if (harvestingClient.getArchiveDescription() == null) { + harvestingClient.setArchiveDescription(BundleUtil.getStringFromBundle("harvestclients.viewEditDialog.archiveDescription.default.generic")); + } + if (StringUtil.isEmpty(harvestingClient.getArchiveUrl()) || StringUtil.isEmpty(harvestingClient.getHarvestingUrl()) || StringUtil.isEmpty(harvestingClient.getMetadataPrefix())) { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java index 32365e17852..aeb010fad6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java @@ -188,7 +188,9 @@ public String getHarvestingUrl() { } public void setHarvestingUrl(String harvestingUrl) { - this.harvestingUrl = harvestingUrl.trim(); + if (harvestingUrl != null) { + this.harvestingUrl = harvestingUrl.trim(); + } } private String archiveUrl; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 3e5c096aff0..905479c4e0d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -905,7 +905,7 @@ public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingC harvestingClient.setHarvestStyle(obj.getString("style", "default")); harvestingClient.setHarvestingUrl(obj.getString("harvestUrl",null)); harvestingClient.setArchiveUrl(obj.getString("archiveUrl",null)); - harvestingClient.setArchiveDescription(obj.getString("archiveDescription", BundleUtil.getStringFromBundle("harvestclients.viewEditDialog.archiveDescription.default.generic"))); + harvestingClient.setArchiveDescription(obj.getString("archiveDescription", null)); harvestingClient.setMetadataPrefix(obj.getString("metadataFormat",null)); harvestingClient.setHarvestingSet(obj.getString("set",null)); From ae28d0a0a17bdaf9097ed3b7b1f1923071ddb649 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 18 Nov 2022 10:55:49 -0500 Subject: [PATCH 188/232] documented delete client api too (#8290) --- doc/sphinx-guides/source/api/native-api.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index e7715725454..64d08696294 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3287,13 +3287,13 @@ An example JSON file would look like this:: export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=http://localhost:8080 - curl -H X-Dataverse-key:$API_TOKEN -X POST "$SERVER_URL/api/harvest/clients/zenodo" --upload-file client.json + curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-Type: application/json" "$SERVER_URL/api/harvest/clients/zenodo" --upload-file client.json The fully expanded example above (without the environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "client.json" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-Type: application/json" "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "client.json" { "status": "OK", @@ -3324,6 +3324,18 @@ Modify a Harvesting Client Similar to the API above, using the same JSON format, but run on an existing client and using the PUT method instead of POST. +Delete a Harvesting Client +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Self-explanatory: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "http://localhost:8080/api/harvest/clients/$nickName" + +Only users with superuser permissions may delete harvesting clients. + + PIDs ---- From 837c4912c86a5d35c57e916620f509fb8ad032f1 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 18 Nov 2022 11:19:45 -0500 Subject: [PATCH 189/232] toned down an .info logger (#8290) --- .../java/edu/harvard/iq/dataverse/api/HarvestingClients.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index d4c096efdbc..370f6ea5d98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -39,8 +39,6 @@ public class HarvestingClients extends AbstractApiBean { - //@EJB - //DataverseServiceBean dataverseService; @EJB HarvesterServiceBean harvesterService; @EJB @@ -124,7 +122,7 @@ public Response harvestingClient(@PathParam("nickName") String nickName, @QueryP // exception, that already has a proper HTTP response in it. retrievedHarvestingClient = execCommand(new GetHarvestingClientCommand(createDataverseRequest(findUserOrDie()), harvestingClient)); - logger.info("retrieved Harvesting Client " + retrievedHarvestingClient.getName() + " with the GetHarvestingClient command."); + logger.fine("retrieved Harvesting Client " + retrievedHarvestingClient.getName() + " with the GetHarvestingClient command."); } catch (WrappedResponse wr) { return wr.getResponse(); } catch (Exception ex) { From 364e347e4a9342bdc02962f14e937369a67969b0 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 21 Nov 2022 09:14:26 -0500 Subject: [PATCH 190/232] test invalid lat/long (too large) #8239 --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 7 +++++++ .../edu/harvard/iq/dataverse/search/SearchUtilTest.java | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index fc3b911c0a5..61a55a88a3b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -1259,6 +1259,13 @@ public void testGeospatialSearchInvalid() { .statusCode(BAD_REQUEST.getStatusCode()) .body("message", CoreMatchers.equalTo("Must contain a single comma to separate latitude and longitude.")); + Response pointLatLongTooLarge = UtilIT.search("*", null, "&geo_point=999,999&geo_radius=5"); + pointLatLongTooLarge.prettyPrint(); + pointLatLongTooLarge.then().assertThat() + // "Search Syntax Error: Error from server at http://localhost:8983/solr/collection1: + // Can't parse point '999.0,999.0' because: Bad X value 999.0 is not in boundary Rect(minX=-180.0,maxX=180.0,minY=-90.0,maxY=90.0)" + .statusCode(BAD_REQUEST.getStatusCode()); + Response junkRadius = UtilIT.search("*", null, "&geo_point=40,60&geo_radius=junk"); junkRadius.prettyPrint(); junkRadius.then().assertThat() diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java index 33f50c9a4c0..6e2fb762c3b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SearchUtilTest.java @@ -111,6 +111,8 @@ public void testGetGeoPoint() { assertThrows(NumberFormatException.class, () -> { SearchUtil.getGeoRadius("somejunk,morejunk"); }, "Must be numbers."); + // invalid but let it go, it's handled by Solr, which throws an informative exception + assertEquals("999.0,-999.0", SearchUtil.getGeoPoint("999,-999")); } @Test From ffc5539c1d52dea9d6fd5acd4b3fb6c98e22e119 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 21 Nov 2022 17:50:21 -0500 Subject: [PATCH 191/232] update flyway numbering --- ...ls-for-tools.sql => V5.13.0.2__7715-signed-urls-for-tools.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.12.1.1__7715-signed-urls-for-tools.sql => V5.13.0.2__7715-signed-urls-for-tools.sql} (100%) diff --git a/src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql b/src/main/resources/db/migration/V5.13.0.2__7715-signed-urls-for-tools.sql similarity index 100% rename from src/main/resources/db/migration/V5.12.1.1__7715-signed-urls-for-tools.sql rename to src/main/resources/db/migration/V5.13.0.2__7715-signed-urls-for-tools.sql From 03188e72c9abeab1254aca8d9170eb4271cc4d4a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 22 Nov 2022 11:47:14 -0500 Subject: [PATCH 192/232] fix netcdf "classic" detection, beef up release note #9117 Also fix test so it doesn't rely on the file extension ".nc". --- doc/release-notes/9117-file-type-detection.md | 4 ++++ .../edu/harvard/iq/dataverse/util/FileUtil.java | 13 ++++++++++++- .../harvard/iq/dataverse/util/FileUtilTest.java | 3 ++- .../resources/netcdf/{madis-raob.nc => madis-raob} | Bin 4 files changed, 18 insertions(+), 2 deletions(-) rename src/test/resources/netcdf/{madis-raob.nc => madis-raob} (100%) diff --git a/doc/release-notes/9117-file-type-detection.md b/doc/release-notes/9117-file-type-detection.md index 7901b478acc..462eaace8ed 100644 --- a/doc/release-notes/9117-file-type-detection.md +++ b/doc/release-notes/9117-file-type-detection.md @@ -1 +1,5 @@ NetCDF and HDF5 files are now detected based on their content rather than just their file extension. + +Both "classic" NetCDF 3 files and more modern NetCDF 4 files are detected based on content. + +Detection for HDF4 files is only done through the file extension ".hdf", as before. diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index dc4f8b97f9a..257bc166ea0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -687,7 +687,7 @@ public static String checkNetcdfOrHdf5(File file) { return null; } switch (type) { - case "NETCDF": + case "NetCDF": return "application/netcdf"; case "NetCDF-4": return "application/netcdf"; @@ -697,6 +697,17 @@ public static String checkNetcdfOrHdf5(File file) { break; } } catch (IOException ex) { + /** + * When an HDF4 file is passed, it won't be detected. Instead, we've + * seen exceptions like this: + * + * ucar.nc2.internal.iosp.hdf4.H4header makeDimension WARNING: + * **dimension length=0 for TagVGroup= *refno=124 tag= VG (1965) + * Vgroup length=28 class= Dim0.0 name= ixx using data 123 + * + * java.lang.IllegalArgumentException: Dimension length =0 must be > + * 0 + */ return null; } return null; diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java index e710236e446..5fafb2be479 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java @@ -377,8 +377,9 @@ public void testIsThumbnailSupported() throws Exception { @Test public void testNetcdfFile() throws IOException { // We got madis-raob.nc from https://www.unidata.ucar.edu/software/netcdf/examples/files.html + // and named it "madis-raob" with no file extension for this test. String path = "src/test/resources/netcdf/"; - String pathAndFile = path + "madis-raob.nc"; + String pathAndFile = path + "madis-raob"; File file = new File(pathAndFile); String contentType = FileUtil.determineFileType(file, pathAndFile); assertEquals("application/netcdf", contentType); diff --git a/src/test/resources/netcdf/madis-raob.nc b/src/test/resources/netcdf/madis-raob similarity index 100% rename from src/test/resources/netcdf/madis-raob.nc rename to src/test/resources/netcdf/madis-raob From c8864f38c221c7f6eec2540fde8092c02e3eec19 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 22 Nov 2022 14:47:25 -0500 Subject: [PATCH 193/232] replace "leftovers" with "stale uploads" #6656 --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index b225594ec3b..beb63f17bfd 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -715,11 +715,11 @@ Before being moved there, - JSF Web UI uploads are stored at :ref:`${dataverse.files.uploads} `, defaulting to ``/usr/local/payara5/glassfish/domains/domain1/uploads`` folder in a standard installation. This place is - configurable and might be set to a separate disk volume, swiped regularly for leftovers. + configurable and might be set to a separate disk volume where stale uploads are purged periodically. - API uploads are stored at the system's temporary files location indicated by the Java system property ``java.io.tmpdir``, defaulting to ``/tmp`` on Linux. If this location is backed by a `tmpfs `_ on your machine, large file uploads via API will cause RAM and/or swap usage bursts. You might want to point this to - a different location, restrict maximum size of it and monitor for leftovers. + a different location, restrict maximum size of it, and monitor for stale uploads. .. _Branding Your Installation: From afdae3e533300ad574be747c997ee0c25f1e3a3d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 22 Nov 2022 14:48:43 -0500 Subject: [PATCH 194/232] add release note #6656 --- doc/release-notes/6656-file-uploads.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/6656-file-uploads.md diff --git a/doc/release-notes/6656-file-uploads.md b/doc/release-notes/6656-file-uploads.md new file mode 100644 index 00000000000..a2430a5d0a8 --- /dev/null +++ b/doc/release-notes/6656-file-uploads.md @@ -0,0 +1 @@ +new JVM option: dataverse.files.uploads From 76eca878258c68240b34f11e76e2af4f7e62b988 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 22 Nov 2022 18:50:55 -0500 Subject: [PATCH 195/232] new framework for stopping harvest jobs (#7940) --- .../harvest/client/ClientHarvestRun.java | 17 +++++-- .../harvest/client/HarvesterServiceBean.java | 46 +++++++++---------- .../client/HarvestingClientServiceBean.java | 40 ++++++++++++---- .../harvest/client/StopHarvestException.java | 17 +++++++ src/main/java/propertyFiles/Bundle.properties | 2 +- src/main/webapp/dashboard.xhtml | 2 +- 6 files changed, 87 insertions(+), 37 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/harvest/client/StopHarvestException.java diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java index 0dc94f835e9..50d06807a13 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java @@ -40,12 +40,13 @@ public void setId(Long id) { this.id = id; } - public enum RunResultType { SUCCESS, FAILURE, INPROGRESS }; + public enum RunResultType { SUCCESS, FAILURE, INPROGRESS, INTERRUPTED }; private static String RESULT_LABEL_SUCCESS = "SUCCESS"; private static String RESULT_LABEL_FAILURE = "FAILED"; private static String RESULT_LABEL_INPROGRESS = "IN PROGRESS"; private static String RESULT_DELETE_IN_PROGRESS = "DELETE IN PROGRESS"; + private static String RESULT_LABEL_INTERRUPTED = "INTERRUPTED"; @ManyToOne @JoinColumn(nullable = false) @@ -76,6 +77,8 @@ public String getResultLabel() { return RESULT_LABEL_FAILURE; } else if (isInProgress()) { return RESULT_LABEL_INPROGRESS; + } else if (isInterrupted()) { + return RESULT_LABEL_INTERRUPTED; } return null; } @@ -84,8 +87,8 @@ public String getDetailedResultLabel() { if (harvestingClient != null && harvestingClient.isDeleteInProgress()) { return RESULT_DELETE_IN_PROGRESS; } - if (isSuccess()) { - String resultLabel = RESULT_LABEL_SUCCESS; + if (isSuccess() || isInterrupted()) { + String resultLabel = getResultLabel(); resultLabel = resultLabel.concat("; "+harvestedDatasetCount+" harvested, "); resultLabel = resultLabel.concat(deletedDatasetCount+" deleted, "); @@ -128,6 +131,14 @@ public void setInProgress() { harvestResult = RunResultType.INPROGRESS; } + public boolean isInterrupted() { + return RunResultType.INTERRUPTED == harvestResult; + } + + public void setInterrupted() { + harvestResult = RunResultType.INTERRUPTED; + } + // Time of this harvest attempt: @Temporal(value = TemporalType.TIMESTAMP) private Date startTime; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index e7156dfe9aa..0ed95177e43 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -85,6 +85,7 @@ public class HarvesterServiceBean { public static final String HARVEST_RESULT_FAILED="failed"; public static final String DATAVERSE_PROPRIETARY_METADATA_FORMAT="dataverse_json"; public static final String DATAVERSE_PROPRIETARY_METADATA_API="/api/datasets/export?exporter="+DATAVERSE_PROPRIETARY_METADATA_FORMAT+"&persistentId="; + public static final String DATAVERSE_HARVEST_STOP_FILE="/var/run/stopharvest_"; public HarvesterServiceBean() { @@ -144,7 +145,6 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId Dataverse harvestingDataverse = harvestingClientConfig.getDataverse(); - MutableBoolean harvestErrorOccurred = new MutableBoolean(false); String logTimestamp = logFormatter.format(new Date()); Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingDataverse.getAlias() + logTimestamp); String logFileName = "../logs" + File.separator + "harvest_" + harvestingClientConfig.getName() + "_" + logTimestamp + ".log"; @@ -155,20 +155,14 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId PrintWriter importCleanupLog = new PrintWriter(new FileWriter( "../logs/harvest_cleanup_" + harvestingClientConfig.getName() + "_" + logTimestamp+".txt")); - List harvestedDatasetIds = null; - - List harvestedDatasetIdsThisBatch = new ArrayList(); - + List harvestedDatasetIds = new ArrayList(); List failedIdentifiers = new ArrayList(); List deletedIdentifiers = new ArrayList(); Date harvestStartTime = new Date(); try { - boolean harvestingNow = harvestingClientConfig.isHarvestingNow(); - - if (harvestingNow) { - harvestErrorOccurred.setValue(true); + if (harvestingClientConfig.isHarvestingNow()) { hdLogger.log(Level.SEVERE, "Cannot begin harvesting, Dataverse " + harvestingDataverse.getName() + " is currently being harvested."); } else { @@ -177,7 +171,7 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId if (harvestingClientConfig.isOai()) { - harvestedDatasetIds = harvestOAI(dataverseRequest, harvestingClientConfig, hdLogger, importCleanupLog, harvestErrorOccurred, failedIdentifiers, deletedIdentifiers, harvestedDatasetIdsThisBatch); + harvestOAI(dataverseRequest, harvestingClientConfig, hdLogger, importCleanupLog, failedIdentifiers, deletedIdentifiers, harvestedDatasetIds); } else { throw new IOException("Unsupported harvest type"); @@ -187,8 +181,11 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size()); } + } catch (StopHarvestException she) { + hdLogger.log(Level.INFO, "HARVEST INTERRUPTED BY EXTERNAL REQUEST"); + harvestingClientService.setPartiallyCompleted(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size()); } catch (Throwable e) { - harvestErrorOccurred.setValue(true); + // Any other exception should be treated as a complete failure String message = "Exception processing harvest, server= " + harvestingClientConfig.getHarvestingUrl() + ",format=" + harvestingClientConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage(); hdLogger.log(Level.SEVERE, message); logException(e, hdLogger); @@ -215,12 +212,11 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId * @param harvestErrorOccurred have we encountered any errors during harvest? * @param failedIdentifiers Study Identifiers for failed "GetRecord" requests */ - private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, PrintWriter importCleanupLog, MutableBoolean harvestErrorOccurred, List failedIdentifiers, List deletedIdentifiers, List harvestedDatasetIdsThisBatch) - throws IOException, ParserConfigurationException, SAXException, TransformerException { + private void harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, PrintWriter importCleanupLog, List failedIdentifiers, List deletedIdentifiers, List harvestedDatasetIds) + throws IOException, ParserConfigurationException, SAXException, TransformerException, StopHarvestException { logBeginOaiHarvest(hdLogger, harvestingClient); - List harvestedDatasetIds = new ArrayList(); OaiHandler oaiHandler; HttpClient httpClient = null; @@ -243,6 +239,10 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien try { for (Iterator
idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) { + // Before each iteration, check if this harvesting job needs to be aborted: + if (checkIfStoppingJob(harvestingClient, harvestedDatasetIds.size())) { + throw new StopHarvestException("Harvesting stopped by external request"); + } Header h = idIter.next(); String identifier = h.getIdentifier(); @@ -265,18 +265,11 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien if (datasetId != null) { harvestedDatasetIds.add(datasetId); - - if ( harvestedDatasetIdsThisBatch == null ) { - harvestedDatasetIdsThisBatch = new ArrayList(); - } - harvestedDatasetIdsThisBatch.add(datasetId); - } if (getRecordErrorOccurred.booleanValue() == true) { failedIdentifiers.add(identifier); - harvestErrorOccurred.setValue(true); - //temporary: + //can be uncommented out for testing failure handling: //throw new IOException("Exception occured, stopping harvest"); } } @@ -286,8 +279,6 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien logCompletedOaiHarvest(hdLogger, harvestingClient); - return harvestedDatasetIds; - } private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, List deletedIdentifiers, Date dateStamp, HttpClient httpClient) { @@ -410,6 +401,13 @@ private void deleteHarvestedDatasetIfExists(String persistentIdentifier, Dataver } hdLogger.info("No dataset found for " + persistentIdentifier + ", skipping delete. "); } + + private boolean checkIfStoppingJob(HarvestingClient harvestingClient, int howmany) { + Long pid = ProcessHandle.current().pid(); + String stopFileName = DATAVERSE_HARVEST_STOP_FILE + harvestingClient.getName() + "." + pid; + + return new File(stopFileName).isFile(); + } private void logBeginOaiHarvest(Logger hdLogger, HarvestingClient harvestingClient) { hdLogger.log(Level.INFO, "BEGIN HARVEST, oaiUrl=" diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java index 0af73550190..bcfc01cea99 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java @@ -204,22 +204,46 @@ public void setHarvestFailure(Long hcId, Date currentTime) { currentRun.setFailed(); currentRun.setFinishTime(currentTime); } - } + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + public void setPartiallyCompleted(Long hcId, Date finishTime, int harvestedCount, int failedCount, int deletedCount) { + recordHarvestJobStatus(hcId, finishTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.INTERRUPTED); + } + + public void recordHarvestJobStatus(Long hcId, Date finishTime, int harvestedCount, int failedCount, int deletedCount, ClientHarvestRun.RunResultType result) { + HarvestingClient harvestingClient = em.find(HarvestingClient.class, hcId); + if (harvestingClient == null) { + return; + } + em.refresh(harvestingClient); + + ClientHarvestRun currentRun = harvestingClient.getLastRun(); + + if (currentRun != null && currentRun.isInProgress()) { + + currentRun.setResult(result); + currentRun.setFinishTime(finishTime); + currentRun.setHarvestedDatasetCount(Long.valueOf(harvestedCount)); + currentRun.setFailedDatasetCount(Long.valueOf(failedCount)); + currentRun.setDeletedDatasetCount(Long.valueOf(deletedCount)); + } + } public Long getNumberOfHarvestedDatasetByClients(List clients) { - String dvs = null; + String clientIds = null; for (HarvestingClient client: clients) { - if (dvs == null) { - dvs = client.getDataverse().getId().toString(); + if (clientIds == null) { + clientIds = client.getId().toString(); } else { - dvs = dvs.concat(","+client.getDataverse().getId().toString()); + clientIds = clientIds.concat(","+client.getId().toString()); } } try { - return (Long) em.createNativeQuery("SELECT count(d.id) FROM dataset d, " - + " dvobject o WHERE d.id = o.id AND o.owner_id in (" - + dvs + ")").getSingleResult(); + return (Long) em.createNativeQuery("SELECT count(d.id) FROM dataset d " + + " WHERE d.harvestingclient_id in (" + + clientIds + ")").getSingleResult(); } catch (Exception ex) { logger.info("Warning: exception trying to count harvested datasets by clients: " + ex.getMessage()); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/StopHarvestException.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/StopHarvestException.java new file mode 100644 index 00000000000..dffa2dd0385 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/StopHarvestException.java @@ -0,0 +1,17 @@ +package edu.harvard.iq.dataverse.harvest.client; + +/** + * + * @author landreev + */ + +public class StopHarvestException extends Exception { + public StopHarvestException(String message) { + super(message); + } + + public StopHarvestException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b19e80020ba..f7b46c308f5 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -520,7 +520,7 @@ harvestclients.btn.add=Add Client harvestclients.tab.header.name=Nickname harvestclients.tab.header.url=URL harvestclients.tab.header.lastrun=Last Run -harvestclients.tab.header.lastresults=Last Results +harvestclients.tab.header.lastresults=Last Result harvestclients.tab.header.action=Actions harvestclients.tab.header.action.btn.run=Run Harvesting harvestclients.tab.header.action.btn.edit=Edit diff --git a/src/main/webapp/dashboard.xhtml b/src/main/webapp/dashboard.xhtml index c5b6a507a92..5a72b52937b 100644 --- a/src/main/webapp/dashboard.xhtml +++ b/src/main/webapp/dashboard.xhtml @@ -42,7 +42,7 @@ #{dashboardPage.numberOfHarvestedDatasets}

- +

From fbf7190f193fb5d65dc6c100d7bb6c2d62dd1b0e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 23 Nov 2022 09:45:40 -0500 Subject: [PATCH 196/232] Improved error handling per QA --- doc/sphinx-guides/source/api/native-api.rst | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 3 +++ src/main/java/edu/harvard/iq/dataverse/api/Files.java | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 91bfe723602..af9836e9ce2 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2711,7 +2711,7 @@ Get External Tool Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This API call is intended as a callback that can be used by :doc:`/installation/external-tools` to retrieve signed Urls necessary for their interaction with Dataverse. -It can be called directly as well. +It can be called directly as well. (Note that the required FILEMETADATA_ID is the "id" returned in the JSON response from the /api/files/$FILE_ID/metadata call.) The response is a JSON object described in the :doc:`/api/external-tools` section of the API guide. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index d08660aeadf..cb097e076f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3600,6 +3600,9 @@ public Response getExternalToolDVParams(@PathParam("tid") long externalToolId, } ExternalTool externalTool = externalToolService.findById(externalToolId); + if (!externalTool.getScope().equals(ExternalTool.Scope.DATASET)) { + return error(BAD_REQUEST, "External tool does not have dataset scope."); + } ApiToken apiToken = null; User u = findUserOrDie(); if (u instanceof AuthenticatedUser) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 824cb9e2745..7b5499d4b8b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -653,9 +653,12 @@ private void exportDatasetMetadata(SettingsServiceBean settingsServiceBean, Data @GET @Path("{id}/metadata/{fmid}/toolparams/{tid}") public Response getExternalToolFMParams(@PathParam("tid") long externalToolId, - @PathParam("id") long fileId, @PathParam("fmid") long fmid, @QueryParam(value = "locale") String locale) { + @PathParam("id") String fileId, @PathParam("fmid") long fmid, @QueryParam(value = "locale") String locale) { try { ExternalTool externalTool = externalToolService.findById(externalToolId); + if (!externalTool.getScope().equals(ExternalTool.Scope.FILE)) { + return error(BAD_REQUEST, "External tool does not have file scope."); + } ApiToken apiToken = null; User u = findUserOrDie(); if (u instanceof AuthenticatedUser) { From c8f4c079d4b8eeeb39731e346d6c16afc2efd0cd Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 23 Nov 2022 15:48:08 -0500 Subject: [PATCH 197/232] final cleanup (#7940) --- .../source/admin/harvestclients.rst | 14 +++++ .../harvard/iq/dataverse/DashboardPage.java | 8 +-- .../harvest/client/HarvesterServiceBean.java | 52 +++++++++++-------- .../client/HarvestingClientServiceBean.java | 48 +++++------------ 4 files changed, 61 insertions(+), 61 deletions(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index c655d5af763..3ea7b8eda86 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -21,6 +21,20 @@ Clients are managed on the "Harvesting Clients" page accessible via the :doc:`da The process of creating a new, or editing an existing client, is largely self-explanatory. It is split into logical steps, in a way that allows the user to go back and correct the entries made earlier. The process is interactive and guidance text is provided. For example, the user is required to enter the URL of the remote OAI server. When they click *Next*, the application will try to establish a connection to the server in order to verify that it is working, and to obtain the information about the sets of metadata records and the metadata formats it supports. The choices offered to the user on the next page will be based on this extra information. If the application fails to establish a connection to the remote archive at the address specified, or if an invalid response is received, the user is given an opportunity to check and correct the URL they entered. +How to Stop a Harvesting Run in Progress +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some harvesting jobs, especially the initial full harvest of a very large set - such as the default set of public datasets at IQSS - can take many hours. In case it is necessary to terminate such a long-running job, the following mechanism is provided (note that it is only available to a sysadmin with shell access to the application server): Create an empty file in the domain logs directory with the following name: ``stopharvest_.``, where ```` is the nickname of the harvesting client and ```` is the process id of the Application Server (Payara). This flag file needs to be owned by the same user that's running Payara, so that the application can remove it after stopping the job in progress. + +For example:: + +.. code-block:: bash + + sudo touch /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 + sudo chown dataverse /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 + + + What if a Run Fails? ~~~~~~~~~~~~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java b/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java index 5b6cdd23775..99c7951c96e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java @@ -97,12 +97,8 @@ public int getNumberOfConfiguredHarvestClients() { } public long getNumberOfHarvestedDatasets() { - List configuredHarvestingClients = harvestingClientService.getAllHarvestingClients(); - if (configuredHarvestingClients == null || configuredHarvestingClients.isEmpty()) { - return 0L; - } - Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetByClients(configuredHarvestingClients); + Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetsByAllClients(); if (numOfDatasets != null && numOfDatasets > 0L) { return numOfDatasets; @@ -142,7 +138,7 @@ public String getHarvestClientsInfoLabel() { infoLabel = configuredHarvestingClients.size() + " harvesting clients configured; "; } - Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetByClients(configuredHarvestingClients); + Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetsByAllClients(); if (numOfDatasets != null && numOfDatasets > 0L) { return infoLabel + numOfDatasets + " harvested datasets"; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index 0ed95177e43..058a20451d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -48,6 +48,9 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.Path; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -85,7 +88,7 @@ public class HarvesterServiceBean { public static final String HARVEST_RESULT_FAILED="failed"; public static final String DATAVERSE_PROPRIETARY_METADATA_FORMAT="dataverse_json"; public static final String DATAVERSE_PROPRIETARY_METADATA_API="/api/datasets/export?exporter="+DATAVERSE_PROPRIETARY_METADATA_FORMAT+"&persistentId="; - public static final String DATAVERSE_HARVEST_STOP_FILE="/var/run/stopharvest_"; + public static final String DATAVERSE_HARVEST_STOP_FILE="../logs/stopharvest_"; public HarvesterServiceBean() { @@ -131,7 +134,7 @@ public List getHarvestTimers() { } /** - * Run a harvest for an individual harvesting Dataverse + * Run a harvest for an individual harvesting client * @param dataverseRequest * @param harvestingClientId * @throws IOException @@ -142,11 +145,9 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId if (harvestingClientConfig == null) { throw new IOException("No such harvesting client: id="+harvestingClientId); } - - Dataverse harvestingDataverse = harvestingClientConfig.getDataverse(); - + String logTimestamp = logFormatter.format(new Date()); - Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingDataverse.getAlias() + logTimestamp); + Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingClientConfig.getName() + logTimestamp); String logFileName = "../logs" + File.separator + "harvest_" + harvestingClientConfig.getName() + "_" + logTimestamp + ".log"; FileHandler fileHandler = new FileHandler(logFileName); hdLogger.setUseParentHandlers(false); @@ -155,15 +156,15 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId PrintWriter importCleanupLog = new PrintWriter(new FileWriter( "../logs/harvest_cleanup_" + harvestingClientConfig.getName() + "_" + logTimestamp+".txt")); - List harvestedDatasetIds = new ArrayList(); - List failedIdentifiers = new ArrayList(); - List deletedIdentifiers = new ArrayList(); + List harvestedDatasetIds = new ArrayList<>(); + List failedIdentifiers = new ArrayList<>(); + List deletedIdentifiers = new ArrayList<>(); Date harvestStartTime = new Date(); try { if (harvestingClientConfig.isHarvestingNow()) { - hdLogger.log(Level.SEVERE, "Cannot begin harvesting, Dataverse " + harvestingDataverse.getName() + " is currently being harvested."); + hdLogger.log(Level.SEVERE, "Cannot start harvest, client " + harvestingClientConfig.getName() + " is already harvesting."); } else { harvestingClientService.resetHarvestInProgress(harvestingClientId); @@ -190,12 +191,8 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId hdLogger.log(Level.SEVERE, message); logException(e, hdLogger); hdLogger.log(Level.INFO, "HARVEST NOT COMPLETED DUE TO UNEXPECTED ERROR."); - // TODO: - // even though this harvesting run failed, we may have had successfully - // processed some number of datasets, by the time the exception was thrown. - // We should record that number too. And the number of the datasets that - // had failed, that we may have counted. -- L.A. 4.4 - harvestingClientService.setHarvestFailure(harvestingClientId, new Date()); + + harvestingClientService.setHarvestFailure(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size()); } finally { harvestingClientService.resetHarvestInProgress(harvestingClientId); @@ -240,7 +237,7 @@ private void harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harv try { for (Iterator
idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) { // Before each iteration, check if this harvesting job needs to be aborted: - if (checkIfStoppingJob(harvestingClient, harvestedDatasetIds.size())) { + if (checkIfStoppingJob(harvestingClient)) { throw new StopHarvestException("Harvesting stopped by external request"); } @@ -294,7 +291,7 @@ private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, P // Make direct call to obtain the proprietary Dataverse metadata // in JSON from the remote Dataverse server: String metadataApiUrl = oaiHandler.getProprietaryDataverseMetadataURL(identifier); - logger.info("calling "+metadataApiUrl); + logger.fine("calling "+metadataApiUrl); tempFile = retrieveProprietaryDataverseMetadata(httpClient, metadataApiUrl); } else { @@ -402,11 +399,24 @@ private void deleteHarvestedDatasetIfExists(String persistentIdentifier, Dataver hdLogger.info("No dataset found for " + persistentIdentifier + ", skipping delete. "); } - private boolean checkIfStoppingJob(HarvestingClient harvestingClient, int howmany) { + private boolean checkIfStoppingJob(HarvestingClient harvestingClient) { Long pid = ProcessHandle.current().pid(); String stopFileName = DATAVERSE_HARVEST_STOP_FILE + harvestingClient.getName() + "." + pid; - - return new File(stopFileName).isFile(); + Path stopFilePath = Paths.get(stopFileName); + + if (Files.exists(stopFilePath)) { + // Now that we know that the file is there, let's (try to) delete it, + // so that the harvest can be re-run. + try { + Files.delete(stopFilePath); + } catch (IOException ioex) { + // No need to treat this is a big deal (could be a permission, etc.) + logger.warning("Failed to delete the flag file "+stopFileName + "; check permissions and delete manually."); + } + return true; + } + + return false; } private void logBeginOaiHarvest(Logger hdLogger, HarvestingClient harvestingClient) { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java index bcfc01cea99..f2a3483c84f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java @@ -167,43 +167,12 @@ public void deleteClient(Long clientId) { @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void setHarvestSuccess(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { - HarvestingClient harvestingClient = em.find(HarvestingClient.class, hcId); - if (harvestingClient == null) { - return; - } - em.refresh(harvestingClient); - - ClientHarvestRun currentRun = harvestingClient.getLastRun(); - - if (currentRun != null && currentRun.isInProgress()) { - // TODO: what if there's no current run in progress? should we just - // give up quietly, or should we make a noise of some kind? -- L.A. 4.4 - - currentRun.setSuccess(); - currentRun.setFinishTime(currentTime); - currentRun.setHarvestedDatasetCount(new Long(harvestedCount)); - currentRun.setFailedDatasetCount(new Long(failedCount)); - currentRun.setDeletedDatasetCount(new Long(deletedCount)); - } + recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.INTERRUPTED); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void setHarvestFailure(Long hcId, Date currentTime) { - HarvestingClient harvestingClient = em.find(HarvestingClient.class, hcId); - if (harvestingClient == null) { - return; - } - em.refresh(harvestingClient); - - ClientHarvestRun currentRun = harvestingClient.getLastRun(); - - if (currentRun != null && currentRun.isInProgress()) { - // TODO: what if there's no current run in progress? should we just - // give up quietly, or should we make a noise of some kind? -- L.A. 4.4 - - currentRun.setFailed(); - currentRun.setFinishTime(currentTime); - } + public void setHarvestFailure(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { + recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.FAILURE); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) @@ -230,6 +199,17 @@ public void recordHarvestJobStatus(Long hcId, Date finishTime, int harvestedCoun } } + public Long getNumberOfHarvestedDatasetsByAllClients() { + try { + return (Long) em.createNativeQuery("SELECT count(d.id) FROM dataset d " + + " WHERE d.harvestingclient_id IS NOT NULL").getSingleResult(); + + } catch (Exception ex) { + logger.info("Warning: exception looking up the total number of harvested datasets: " + ex.getMessage()); + return 0L; + } + } + public Long getNumberOfHarvestedDatasetByClients(List clients) { String clientIds = null; for (HarvestingClient client: clients) { From caf8fe5f83d8efe6a80c23702b1eb525629733bd Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 23 Nov 2022 15:54:12 -0500 Subject: [PATCH 198/232] doc change (#7940) --- doc/sphinx-guides/source/admin/harvestclients.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 3ea7b8eda86..ea44e823c75 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -26,13 +26,14 @@ How to Stop a Harvesting Run in Progress Some harvesting jobs, especially the initial full harvest of a very large set - such as the default set of public datasets at IQSS - can take many hours. In case it is necessary to terminate such a long-running job, the following mechanism is provided (note that it is only available to a sysadmin with shell access to the application server): Create an empty file in the domain logs directory with the following name: ``stopharvest_.``, where ```` is the nickname of the harvesting client and ```` is the process id of the Application Server (Payara). This flag file needs to be owned by the same user that's running Payara, so that the application can remove it after stopping the job in progress. -For example:: +For example: .. code-block:: bash sudo touch /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 sudo chown dataverse /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 +It is recommended to stop any running harvesting jobs if you need to restart the application server, otherwise the ongoing harvest will be killed, but may be left marked as if it's still in progress in the database. What if a Run Fails? From 290367118236f82784002a75a6b638d85f999d7b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 23 Nov 2022 15:55:50 -0500 Subject: [PATCH 199/232] doc change (#7940) --- doc/sphinx-guides/source/admin/harvestclients.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index ea44e823c75..6a76f721162 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -33,7 +33,7 @@ For example: sudo touch /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 sudo chown dataverse /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 -It is recommended to stop any running harvesting jobs if you need to restart the application server, otherwise the ongoing harvest will be killed, but may be left marked as if it's still in progress in the database. +We recommend that stop stop any running harvesting jobs using this mechanism if you need to restart the application server, otherwise the ongoing harvest will be killed, but may be left marked as if it's still in progress in the database. What if a Run Fails? From 398fd4819f74241a0db3a458b74515f7688aacce Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 28 Nov 2022 14:00:57 -0500 Subject: [PATCH 200/232] Improve error handling --- .../harvard/iq/dataverse/api/Datasets.java | 5 +- .../edu/harvard/iq/dataverse/api/Files.java | 5 +- .../externaltools/ExternalToolHandler.java | 54 ++++++++++--------- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index cb097e076f0..1fb1caaeac2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3600,7 +3600,10 @@ public Response getExternalToolDVParams(@PathParam("tid") long externalToolId, } ExternalTool externalTool = externalToolService.findById(externalToolId); - if (!externalTool.getScope().equals(ExternalTool.Scope.DATASET)) { + if(externalTool==null) { + return error(BAD_REQUEST, "External tool not found."); + } + if (!ExternalTool.Scope.DATASET.equals(externalTool.getScope())) { return error(BAD_REQUEST, "External tool does not have dataset scope."); } ApiToken apiToken = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 7b5499d4b8b..af0f6be6d32 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -656,7 +656,10 @@ public Response getExternalToolFMParams(@PathParam("tid") long externalToolId, @PathParam("id") String fileId, @PathParam("fmid") long fmid, @QueryParam(value = "locale") String locale) { try { ExternalTool externalTool = externalToolService.findById(externalToolId); - if (!externalTool.getScope().equals(ExternalTool.Scope.FILE)) { + if(externalTool == null) { + return error(BAD_REQUEST, "External tool not found."); + } + if (!ExternalTool.Scope.FILE.equals(externalTool.getScope())) { return error(BAD_REQUEST, "External tool does not have file scope."); } ApiToken apiToken = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java index 542c6e2cd26..88a51017b75 100644 --- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java @@ -171,32 +171,34 @@ public JsonObject getParams(JsonObject toolParameters) { public JsonObjectBuilder createPostBody(JsonObject params) { JsonObjectBuilder bodyBuilder = Json.createObjectBuilder(); bodyBuilder.add("queryParameters", params); - - JsonArray apiArray = JsonUtil.getJsonArray(externalTool.getAllowedApiCalls()); - JsonArrayBuilder apisBuilder = Json.createArrayBuilder(); - - apiArray.getValuesAs(JsonObject.class).forEach(((apiObj) -> { - logger.fine(JsonUtil.prettyPrint(apiObj)); - String name = apiObj.getJsonString(NAME).getString(); - String httpmethod = apiObj.getJsonString(HTTP_METHOD).getString(); - int timeout = apiObj.getInt(TIMEOUT); - String urlTemplate = apiObj.getJsonString(URL_TEMPLATE).getString(); - logger.fine("URL Template: " + urlTemplate); - urlTemplate = SystemConfig.getDataverseSiteUrlStatic() + urlTemplate; - String apiPath = replaceTokensWithValues(urlTemplate); - logger.fine("URL WithTokens: " + apiPath); - String url = apiPath; - // Sign if apiToken exists, otherwise send unsigned URL (i.e. for guest users) - ApiToken apiToken = getApiToken(); - if (apiToken != null) { - url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(), httpmethod, - JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + getApiToken().getTokenString()); - } - logger.fine("Signed URL: " + url); - apisBuilder.add(Json.createObjectBuilder().add(NAME, name).add(HTTP_METHOD, httpmethod) - .add(SIGNED_URL, url).add(TIMEOUT, timeout)); - })); - bodyBuilder.add("signedUrls", apisBuilder); + String apiCallStr = externalTool.getAllowedApiCalls(); + if (apiCallStr != null && !apiCallStr.isBlank()) { + JsonArray apiArray = JsonUtil.getJsonArray(externalTool.getAllowedApiCalls()); + JsonArrayBuilder apisBuilder = Json.createArrayBuilder(); + apiArray.getValuesAs(JsonObject.class).forEach(((apiObj) -> { + logger.fine(JsonUtil.prettyPrint(apiObj)); + String name = apiObj.getJsonString(NAME).getString(); + String httpmethod = apiObj.getJsonString(HTTP_METHOD).getString(); + int timeout = apiObj.getInt(TIMEOUT); + String urlTemplate = apiObj.getJsonString(URL_TEMPLATE).getString(); + logger.fine("URL Template: " + urlTemplate); + urlTemplate = SystemConfig.getDataverseSiteUrlStatic() + urlTemplate; + String apiPath = replaceTokensWithValues(urlTemplate); + logger.fine("URL WithTokens: " + apiPath); + String url = apiPath; + // Sign if apiToken exists, otherwise send unsigned URL (i.e. for guest users) + ApiToken apiToken = getApiToken(); + if (apiToken != null) { + url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(), + httpmethod, JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + + getApiToken().getTokenString()); + } + logger.fine("Signed URL: " + url); + apisBuilder.add(Json.createObjectBuilder().add(NAME, name).add(HTTP_METHOD, httpmethod) + .add(SIGNED_URL, url).add(TIMEOUT, timeout)); + })); + bodyBuilder.add("signedUrls", apisBuilder); + } return bodyBuilder; } From 77496b8ae0faab9418c57ee408b3c62ffa5e3806 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 28 Nov 2022 15:25:07 -0500 Subject: [PATCH 201/232] add content-type in requestSignedUrl --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index af9836e9ce2..16ec541f0e7 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4139,7 +4139,7 @@ A curl example using allowing access to a dataset's metadata export API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export JSON={"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeOut":5,"user":"alberteinstein"} - curl -H 'X-Dataverse-key:$API_KEY' -d $JSON $SERVER_URL/api/admin/requestSignedUrl + curl -H 'X-Dataverse-key:$API_KEY' -H 'Content-Type:application/json' -d $JSON $SERVER_URL/api/admin/requestSignedUrl Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra security. From d7ecb3ec0a02be304fbf7aa11a9d3fa656e0cea1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 28 Nov 2022 15:30:45 -0500 Subject: [PATCH 202/232] fix example json param --- doc/sphinx-guides/source/api/native-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 16ec541f0e7..35631d36a2c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4137,9 +4137,9 @@ A curl example using allowing access to a dataset's metadata export SERVER_URL=https://demo.dataverse.org export API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export JSON={"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeOut":5,"user":"alberteinstein"} + export JSON='{"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeOut":5,"user":"alberteinstein"}' - curl -H 'X-Dataverse-key:$API_KEY' -H 'Content-Type:application/json' -d $JSON $SERVER_URL/api/admin/requestSignedUrl + curl -H 'X-Dataverse-key:$API_KEY' -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/requestSignedUrl Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra security. From a667876a0180b01e693e407d0f8e29f8b45f083e Mon Sep 17 00:00:00 2001 From: landreev Date: Mon, 28 Nov 2022 15:34:33 -0500 Subject: [PATCH 203/232] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ce91db3457b..80a622b763f 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3205,7 +3205,7 @@ Managing Harvesting Clients The following API can be used to create and manage "Harvesting Clients". A Harvesting Client is a configuration entry that allows your Dataverse installation to harvest and index metadata from a specific remote location, either regularly, on a configured schedule, or on a one-off basis. For more information, see the :doc:`/admin/harvestclients` section of the Admin Guide. -List All Congigured Harvesting Clients +List All Configured Harvesting Clients ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Shows all the Harvesting Clients configured:: From 0c8216aefac3d3ec9e3b154e9fb12c566049ebfd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 28 Nov 2022 15:42:55 -0500 Subject: [PATCH 204/232] unwrap wrapped response --- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 31a874f85c9..22cf1b90188 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -2250,9 +2250,13 @@ public Response getBannerMessages(@PathParam("id") Long id) throws WrappedRespon @POST @Consumes("application/json") @Path("/requestSignedUrl") - public Response getSignedUrl(JsonObject urlInfo) throws WrappedResponse { - AuthenticatedUser superuser = findAuthenticatedUserOrDie(); - + public Response getSignedUrl(JsonObject urlInfo) { + AuthenticatedUser superuser = null; + try { + superuser = findAuthenticatedUserOrDie(); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } if (superuser == null || !superuser.isSuperuser()) { return error(Response.Status.FORBIDDEN, "Requesting signed URLs is restricted to superusers."); } From 88f78f891c0f0cc90f384e02582bd63b86b53c3f Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 28 Nov 2022 15:44:02 -0500 Subject: [PATCH 205/232] cosmetic guide improvements (#8290) --- doc/sphinx-guides/source/api/native-api.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 80a622b763f..c3b01b2aba0 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3255,16 +3255,15 @@ Create a Harvesting Client To create a new harvesting client you must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: - nickName: Alpha-numeric may also contain -, _, or %, but no spaces. Must also be unique in the installation. Must match the nickName in the Path -- dataverseAlias: The alias of an existing collection where harvested datasets will be deposited. +- dataverseAlias: The alias of an existing collection where harvested datasets will be deposited - harvestUrl: The URL of the remote OAI archive -- archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai. -- metadataFormat: A supported metadata format. For example, "oai_dc" or "ddi". +- archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai +- metadataFormat: A supported metadata format. For example, "oai_dc" or "ddi" The following optional fields are supported: - archiveDescription: What the name suggests. If not supplied, will default to "This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data." - set: The OAI set on the remote server. If not supplied, will default to none, i.e., "harvest everything". -- schedule: Harvesting schedule. Defaults to "none". - style: Defaults to "default" - a generic OAI archive. (Make sure to use "dataverse" when configuring harvesting from another Dataverse installation). An example JSON file would look like this:: From 590ee408cde0658bfdf70991066360d84857d310 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 28 Nov 2022 15:53:51 -0500 Subject: [PATCH 206/232] fix for api key param --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 35631d36a2c..6b11ff46d9e 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4139,7 +4139,7 @@ A curl example using allowing access to a dataset's metadata export API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export JSON='{"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeOut":5,"user":"alberteinstein"}' - curl -H 'X-Dataverse-key:$API_KEY' -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/requestSignedUrl + curl -H "X-Dataverse-key:$API_KEY" -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/requestSignedUrl Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra security. From 79da3efadd4878e437e8da907a06345271f1a522 Mon Sep 17 00:00:00 2001 From: landreev Date: Mon, 28 Nov 2022 15:54:37 -0500 Subject: [PATCH 207/232] Update src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java Co-authored-by: Philip Durbin --- .../java/edu/harvard/iq/dataverse/api/HarvestingClients.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index 370f6ea5d98..d04e764e5ef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -148,7 +148,7 @@ public Response harvestingClient(@PathParam("nickName") String nickName, @QueryP @Path("{nickName}") public Response createHarvestingClient(String jsonBody, @PathParam("nickName") String nickName, @QueryParam("key") String apiKey) throws IOException, JsonParseException { // Note that we don't check the user's authorization within the API - // method. Insetead, we will end up reporting a "not authorized" + // method. Instead, we will end up reporting a "not authorized" // exception thrown by the Command, if this user has no permission // to perform the action. From ba07ad9a43ca0d8dff3a47799e6907572f0fb6fe Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 28 Nov 2022 16:33:01 -0500 Subject: [PATCH 208/232] more cosmetic guide fixes (#8290) --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c3b01b2aba0..8fa35830356 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3258,7 +3258,7 @@ To create a new harvesting client you must supply a JSON file that describes the - dataverseAlias: The alias of an existing collection where harvested datasets will be deposited - harvestUrl: The URL of the remote OAI archive - archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai -- metadataFormat: A supported metadata format. For example, "oai_dc" or "ddi" +- metadataFormat: A supported metadata format. As of writing this the supported formats are "oai_dc", "oai_ddi" and "dataverse_json". The following optional fields are supported: From 469e74332ae598079dadf257498490ee3e85672b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 28 Nov 2022 18:34:38 -0500 Subject: [PATCH 209/232] added simple restassured tests (#8290) --- doc/sphinx-guides/source/api/native-api.rst | 11 +- .../iq/dataverse/api/HarvestingClientsIT.java | 122 ++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8fa35830356..003604cf531 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3252,9 +3252,14 @@ Shows a Harvesting Client with a defined nickname:: Create a Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~ -To create a new harvesting client you must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: +To create a new harvesting client:: + + POST http://$SERVER/api/harvest/clients/$nickname + +``nickName`` is the name identifying the new client. It should be alpha-numeric and may also contain -, _, or %, but no spaces. Must also be unique in the installation. + +You must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: -- nickName: Alpha-numeric may also contain -, _, or %, but no spaces. Must also be unique in the installation. Must match the nickName in the Path - dataverseAlias: The alias of an existing collection where harvested datasets will be deposited - harvestUrl: The URL of the remote OAI archive - archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai @@ -3266,6 +3271,8 @@ The following optional fields are supported: - set: The OAI set on the remote server. If not supplied, will default to none, i.e., "harvest everything". - style: Defaults to "default" - a generic OAI archive. (Make sure to use "dataverse" when configuring harvesting from another Dataverse installation). +Generally, the API will accept the output of the GET version of the API for an existing client as valid input, but some fields will be ignored. For example, as of writing this there is no way to configure a harvesting schedule via this API. + An example JSON file would look like this:: { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java new file mode 100644 index 00000000000..9eac3545e54 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -0,0 +1,122 @@ +package edu.harvard.iq.dataverse.api; + +import java.util.logging.Logger; +import com.jayway.restassured.RestAssured; +import static com.jayway.restassured.RestAssured.given; +import org.junit.Test; +import com.jayway.restassured.response.Response; +import static org.hamcrest.CoreMatchers.equalTo; +import static junit.framework.Assert.assertEquals; +import org.junit.BeforeClass; + +/** + * extremely minimal (for now) API tests for creating OAI clients. + */ +public class HarvestingClientsIT { + + private static final Logger logger = Logger.getLogger(HarvestingClientsIT.class.getCanonicalName()); + + private static final String harvestClientsApi = "/api/harvest/clients/"; + private static final String harvestCollection = "root"; + private static final String harvestUrl = "https://demo.dataverse.org/oai"; + private static final String archiveUrl = "https://demo.dataverse.org"; + private static final String harvestMetadataFormat = "oai_dc"; + private static final String archiveDescription = "RestAssured harvesting client test"; + + @BeforeClass + public static void setUpClass() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + private void setupUsers() { + Response cu0 = UtilIT.createRandomUser(); + normalUserAPIKey = UtilIT.getApiTokenFromResponse(cu0); + Response cu1 = UtilIT.createRandomUser(); + String un1 = UtilIT.getUsernameFromResponse(cu1); + Response u1a = UtilIT.makeSuperUser(un1); + adminUserAPIKey = UtilIT.getApiTokenFromResponse(cu1); + } + + private String normalUserAPIKey; + private String adminUserAPIKey; + + @Test + public void testCreateEditDeleteClient() { + setupUsers(); + String nickName = UtilIT.getRandomString(6); + + + String clientApiPath = String.format(harvestClientsApi+"%s", nickName); + String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + + "\"type\":\"oai\"," + + "\"harvestUrl\":\"%s\"," + + "\"archiveUrl\":\"%s\"," + + "\"metadataFormat\":\"%s\"}", + harvestCollection, harvestUrl, archiveUrl, harvestMetadataFormat); + + + // Try to create a client as normal user, should fail: + + Response rCreate = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) + .body(clientJson) + .post(clientApiPath); + assertEquals(401, rCreate.getStatusCode()); + + + // Try to create the same as admin user, should succeed: + + rCreate = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(clientJson) + .post(clientApiPath); + assertEquals(201, rCreate.getStatusCode()); + + // Try to update the client we have just created: + + String updateJson = String.format("{\"archiveDescription\":\"%s\"}", archiveDescription); + + Response rUpdate = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(updateJson) + .put(clientApiPath); + assertEquals(200, rUpdate.getStatusCode()); + + // Now let's retrieve the client we've just created and edited: + + Response getClientResponse = given() + .get(clientApiPath); + + logger.info("getClient.getStatusCode(): " + getClientResponse.getStatusCode()); + logger.info("getClient printresponse: " + getClientResponse.prettyPrint()); + assertEquals(200, getClientResponse.getStatusCode()); + + // ... and validate the values: + + getClientResponse.then().assertThat() + .body("status", equalTo(AbstractApiBean.STATUS_OK)) + .body("data.type", equalTo("oai")) + .body("data.nickName", equalTo(nickName)) + .body("data.archiveDescription", equalTo(archiveDescription)) + .body("data.dataverseAlias", equalTo(harvestCollection)) + .body("data.harvestUrl", equalTo(harvestUrl)) + .body("data.archiveUrl", equalTo(archiveUrl)) + .body("data.metadataFormat", equalTo(harvestMetadataFormat)); + + // Try to delete the client as normal user should fail: + + Response rDelete = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) + .delete(clientApiPath); + logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode()); + assertEquals(401, rDelete.getStatusCode()); + + // Try to delete as admin user should work: + + rDelete = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .delete(clientApiPath); + logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode()); + assertEquals(200, rDelete.getStatusCode()); + } +} From 83171f32fa82d2715ecc27f5c85ee14ba7e05199 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 28 Nov 2022 18:44:02 -0500 Subject: [PATCH 210/232] cosmetic (#8290) --- doc/sphinx-guides/source/api/native-api.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 003604cf531..9da98b2f02b 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3278,7 +3278,6 @@ An example JSON file would look like this:: { "nickName": "zenodo", "dataverseAlias": "zenodoHarvested", - "type": "oai", "harvestUrl": "https://zenodo.org/oai2d", "archiveUrl": "https://zenodo.org", "archiveDescription": "Moissonné depuis la collection LMOPS de l'entrepôt Zenodo. En cliquant sur ce jeu de données, vous serez redirigé vers Zenodo.", From 79883e35d89ee65f4f5e5cf510c7d282cc685c1c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 29 Nov 2022 14:15:03 -0500 Subject: [PATCH 211/232] typo (and force Jenkins run) #8290 --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9da98b2f02b..5e72b1f0263 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3212,7 +3212,7 @@ Shows all the Harvesting Clients configured:: GET http://$SERVER/api/harvest/clients/ -Show a specific Harvesting Client +Show a Specific Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Shows a Harvesting Client with a defined nickname:: From 9d81716b30cbfe701718fc81b1d51039562395db Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 29 Nov 2022 17:11:24 -0500 Subject: [PATCH 212/232] added the new restassured test to the list (#8290) --- tests/integration-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index 85b37c79835..85670e8324a 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT From 8794c07dafddd16c91f6ac931cbd03017d429d77 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Thu, 1 Dec 2022 13:57:44 -0500 Subject: [PATCH 213/232] making the create/edit APIs superuser-only (#8290) --- .../iq/dataverse/api/HarvestingClients.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index d04e764e5ef..42534514b68 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -147,10 +147,16 @@ public Response harvestingClient(@PathParam("nickName") String nickName, @QueryP @POST @Path("{nickName}") public Response createHarvestingClient(String jsonBody, @PathParam("nickName") String nickName, @QueryParam("key") String apiKey) throws IOException, JsonParseException { - // Note that we don't check the user's authorization within the API - // method. Instead, we will end up reporting a "not authorized" - // exception thrown by the Command, if this user has no permission - // to perform the action. + // Per the discussion during the QA of PR #9174, we decided to make + // the create/edit APIs superuser-only (the delete API was already so) + try { + User u = findUserOrDie(); + if ((!(u instanceof AuthenticatedUser) || !u.isSuperuser())) { + throw new WrappedResponse(error(Response.Status.UNAUTHORIZED, "Only superusers can create harvesting clients.")); + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } try ( StringReader rdr = new StringReader(jsonBody) ) { JsonObject json = Json.createReader(rdr).readObject(); @@ -225,6 +231,15 @@ public Response createHarvestingClient(String jsonBody, @PathParam("nickName") S @PUT @Path("{nickName}") public Response modifyHarvestingClient(String jsonBody, @PathParam("nickName") String nickName, @QueryParam("key") String apiKey) throws IOException, JsonParseException { + try { + User u = findUserOrDie(); + if ((!(u instanceof AuthenticatedUser) || !u.isSuperuser())) { + throw new WrappedResponse(error(Response.Status.UNAUTHORIZED, "Only superusers can modify harvesting clients.")); + } + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + HarvestingClient harvestingClient = null; try { harvestingClient = harvestingClientService.findByNickname(nickName); @@ -293,8 +308,7 @@ public Response deleteHarvestingClient(@PathParam("nickName") String nickName) t // Deleting a client can take a while (if there's a large amnount of // harvested content associated with it). So instead of calling the command // directly, we will be calling an async. service bean method. - // Without the command engine taking care of authorization, we'll need - // to check if the user has the right to do this explicitly: + try { User u = findUserOrDie(); From 9e9edd3f64f49c96ea58fb17ac5a4bd5b4c441fc Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 2 Dec 2022 08:51:46 -0500 Subject: [PATCH 214/232] Show values for controlled vocabulary, add tests #8944 With toArray().toString() we were getting something like this: [Ljava.lang.Object;@76d0a290 --- .../iq/dataverse/util/json/JsonPrinter.java | 8 +++--- .../iq/dataverse/api/MetadataBlocksIT.java | 26 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 5 ++++ tests/integration-tests.txt | 2 +- 4 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 1977ef9bbfb..dc547f2e52c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -555,9 +555,11 @@ public static JsonObjectBuilder json(DatasetFieldType fld) { if (fld.isControlledVocabulary()) { // If the field has a controlled vocabulary, // add all values to the resulting JSON - fieldsBld.add( - "controlledVocabularyValues", - fld.getControlledVocabularyValues().toArray().toString()); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (ControlledVocabularyValue cvv : fld.getControlledVocabularyValues()) { + jab.add(cvv.getStrValue()); + } + fieldsBld.add("controlledVocabularyValues", jab); } if (!fld.getChildDatasetFieldTypes().isEmpty()) { JsonObjectBuilder subFieldsBld = jsonObjectBuilder(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java new file mode 100644 index 00000000000..05b7a7910ff --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -0,0 +1,26 @@ +package edu.harvard.iq.dataverse.api; + +import com.jayway.restassured.RestAssured; +import com.jayway.restassured.response.Response; +import static javax.ws.rs.core.Response.Status.OK; +import org.hamcrest.CoreMatchers; +import org.junit.BeforeClass; +import org.junit.Test; + +public class MetadataBlocksIT { + + @BeforeClass + public static void setUpClass() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testGetCitationBlock() { + Response getCitationBlock = UtilIT.getMetadataBlock("citation"); + getCitationBlock.prettyPrint(); + getCitationBlock.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.fields.subject.controlledVocabularyValues[0]", CoreMatchers.is("Agricultural Sciences")); + } + +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 550d4ed1264..54a217be527 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -567,6 +567,11 @@ static Response setMetadataBlocks(String dataverseAlias, JsonArrayBuilder blocks .post("/api/dataverses/" + dataverseAlias + "/metadatablocks"); } + static Response getMetadataBlock(String block) { + return given() + .get("/api/metadatablocks/" + block); + } + static private String getDatasetXml(String title, String author, String description) { String nullLicense = null; String nullRights = null; diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index 85670e8324a..6e6668d45af 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT From 175ab7fd29a7a51a95811963113d332c8f2621c3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 2 Dec 2022 13:06:08 -0500 Subject: [PATCH 215/232] rc2 version of the gdcc-xoai lib. should've been done a long time ago :( #8843 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index e36a78b11be..bf37299f2df 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -163,7 +163,7 @@ 4.4.14 - 5.0.0-RC1 + 5.0.0-RC2 1.15.0 From 3d1e98c5a9f5f755d8d78b6151b659fe2377f3ed Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 2 Dec 2022 13:27:40 -0500 Subject: [PATCH 216/232] this method was renamed in RC2 (#8843) --- .../harvest/server/xoai/DataverseXoaiItemRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java index faf3cf9ddc4..147d42648fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java @@ -49,7 +49,7 @@ public DataverseXoaiItemRepository (OAIRecordServiceBean recordService, DatasetS } @Override - public ItemIdentifier getItem(String identifier) throws IdDoesNotExistException { + public ItemIdentifier getItemIdentifier(String identifier) throws IdDoesNotExistException { // This method is called when ListMetadataFormats request specifies // the identifier, requesting the formats available for this specific record. // In our case, under the current implementation, we need to simply look From aeffa3b6fc13a029b70630d856b5f0373a333903 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 5 Dec 2022 20:41:24 -0500 Subject: [PATCH 217/232] a few extra oai tests (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 222 +++++++++++++----- .../edu/harvard/iq/dataverse/api/UtilIT.java | 10 + 2 files changed, 176 insertions(+), 56 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index fdd034ab12e..5355b57490d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -10,7 +10,12 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import com.jayway.restassured.response.Response; import com.jayway.restassured.path.json.JsonPath; +import com.jayway.restassured.path.xml.XmlPath; +import com.jayway.restassured.path.xml.element.Node; import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import javax.json.Json; import javax.json.JsonArray; import static javax.ws.rs.core.Response.Status.FORBIDDEN; @@ -24,18 +29,32 @@ import static org.junit.Assert.assertTrue; /** - * extremely minimal API tests for creating OAI sets. + * Tests for the Harvesting Server functionality + * Note that these test BOTH the proprietary Dataverse rest APIs for creating + * and managing sets, AND the OAI-PMH functionality itself. */ public class HarvestingServerIT { private static final Logger logger = Logger.getLogger(HarvestingServerIT.class.getCanonicalName()); + private static String normalUserAPIKey; + private static String adminUserAPIKey; + private static String singleSetDatasetIdentifier; + private static String singleSetDatasetPersistentId; + @BeforeClass public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); // enable harvesting server // Gave some thought to storing the original response, and resetting afterwards - but that appears to be more complexity than it's worth Response enableHarvestingServerResponse = UtilIT.setSetting(SettingsServiceBean.Key.OAIServerEnabled,"true"); + + // Create users: + setupUsers(); + + // Create and publish some datasets: + setupDatasets(); + } @AfterClass @@ -44,7 +63,7 @@ public static void afterClass() { Response enableHarvestingServerResponse = UtilIT.setSetting(SettingsServiceBean.Key.OAIServerEnabled,"false"); } - private void setupUsers() { + private static void setupUsers() { Response cu0 = UtilIT.createRandomUser(); normalUserAPIKey = UtilIT.getApiTokenFromResponse(cu0); Response cu1 = UtilIT.createRandomUser(); @@ -52,6 +71,40 @@ private void setupUsers() { Response u1a = UtilIT.makeSuperUser(un1); adminUserAPIKey = UtilIT.getApiTokenFromResponse(cu1); } + + private static void setupDatasets() { + // create dataverse: + Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // publish dataverse: + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, adminUserAPIKey); + assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode()); + + // create dataset: + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + // retrieve the global id: + singleSetDatasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + + // publish dataset: + Response publishDataset = UtilIT.publishDatasetViaNativeApi(singleSetDatasetPersistentId, "major", adminUserAPIKey); + assertEquals(200, publishDataset.getStatusCode()); + + singleSetDatasetIdentifier = singleSetDatasetPersistentId.substring(singleSetDatasetPersistentId.lastIndexOf('/') + 1); + + logger.info("identifier: " + singleSetDatasetIdentifier); + + // Publish command is executed asynchronously, i.e. it may + // still be running after we received the OK from the publish API. + // The oaiExport step also requires the metadata exports to be done and this + // takes longer than just publish/reindex. + // So wait for all of this to finish. + UtilIT.sleepForReexport(singleSetDatasetPersistentId, adminUserAPIKey, 10); + } private String jsonForTestSpec(String name, String def) { String r = String.format("{\"name\":\"%s\",\"definition\":\"%s\"}", name, def);//description is optional @@ -63,20 +116,84 @@ private String jsonForEditSpec(String name, String def, String desc) { return r; } - private String normalUserAPIKey; - private String adminUserAPIKey; + private XmlPath validateOaiVerbResponse(Response oaiResponse, String verb) { + // confirm that the response is in fact XML: + XmlPath responseXmlPath = oaiResponse.getBody().xmlPath(); + assertNotNull(responseXmlPath); + + String dateString = responseXmlPath.getString("OAI-PMH.responseDate"); + assertNotNull(dateString); // TODO: validate that it's well-formatted! + logger.info("date string from the OAI output:"+dateString); + assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.request")); + assertEquals(verb, responseXmlPath.getString("OAI-PMH.request.@verb")); + return responseXmlPath; + } + + @Test + public void testOaiIdentify() { + // Run Identify: + Response identifyResponse = UtilIT.getOaiIdentify(); + assertEquals(OK.getStatusCode(), identifyResponse.getStatusCode()); + //logger.info("Identify response: "+identifyResponse.prettyPrint()); + + // Validate the response: + + XmlPath responseXmlPath = validateOaiVerbResponse(identifyResponse, "Identify"); + assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.Identify.baseURL")); + // Confirm that the server is reporting the correct parameters that + // our server implementation should be using: + assertEquals("2.0", responseXmlPath.getString("OAI-PMH.Identify.protocolVersion")); + assertEquals("transient", responseXmlPath.getString("OAI-PMH.Identify.deletedRecord")); + assertEquals("YYYY-MM-DDThh:mm:ssZ", responseXmlPath.getString("OAI-PMH.Identify.granularity")); + } + + @Test + public void testOaiListMetadataFormats() { + // Run ListMeatadataFormats: + Response listFormatsResponse = UtilIT.getOaiListMetadataFormats(); + assertEquals(OK.getStatusCode(), listFormatsResponse.getStatusCode()); + //logger.info("ListMetadataFormats response: "+listFormatsResponse.prettyPrint()); + + // Validate the response: + + XmlPath responseXmlPath = validateOaiVerbResponse(listFormatsResponse, "ListMetadataFormats"); + + // Check the payload of the response atgainst the list of metadata formats + // we are currently offering under OAI; will need to be explicitly + // modified if/when we add more harvestable formats. + + List listFormats = responseXmlPath.getList("OAI-PMH.ListMetadataFormats.metadataFormat"); + + assertNotNull(listFormats); + assertEquals(5, listFormats.size()); + + // The metadata formats are reported in an unpredictable ordder. We + // want to sort the prefix names for comparison purposes, and for that + // they need to be saved in a modifiable list: + List metadataPrefixes = new ArrayList<>(); + + for (int i = 0; i < listFormats.size(); i++) { + metadataPrefixes.add(responseXmlPath.getString("OAI-PMH.ListMetadataFormats.metadataFormat["+i+"].metadataPrefix")); + } + Collections.sort(metadataPrefixes); + + assertEquals("[Datacite, dataverse_json, oai_datacite, oai_dc, oai_ddi]", metadataPrefixes.toString()); + + } + + @Test - public void testSetCreation() { - setupUsers(); + public void testSetCreateAPIandOAIlistIdentifiers() { + // Create the set with Dataverse /api/harvest/server API: String setName = UtilIT.getRandomString(6); String def = "*"; // make sure the set does not exist - String u0 = String.format("/api/harvest/server/oaisets/%s", setName); + String setPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; Response r0 = given() - .get(u0); + .get(setPath); assertEquals(404, r0.getStatusCode()); // try to create set as normal user, should fail @@ -94,7 +211,7 @@ public void testSetCreation() { assertEquals(201, r2.getStatusCode()); Response getSet = given() - .get(u0); + .get(setPath); logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); logger.info("getSet printresponse: " + getSet.prettyPrint()); @@ -118,17 +235,19 @@ public void testSetCreation() { Response r4 = UtilIT.exportOaiSet(setName); assertEquals(200, r4.getStatusCode()); - // try to delete as normal user should fail + + + // try to delete as normal user, should fail Response r5 = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) - .delete(u0); + .delete(setPath); logger.info("r5.getStatusCode(): " + r5.getStatusCode()); assertEquals(400, r5.getStatusCode()); - // try to delete as admin user should work + // try to delete as admin user, should work Response r6 = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .delete(u0); + .delete(setPath); logger.info("r6.getStatusCode(): " + r6.getStatusCode()); assertEquals(200, r6.getStatusCode()); @@ -136,7 +255,7 @@ public void testSetCreation() { @Test public void testSetEdit() { - setupUsers(); + //setupUsers(); String setName = UtilIT.getRandomString(6); String def = "*"; @@ -195,46 +314,17 @@ public void testSetEdit() { // OAI set with that one dataset, and attempt to retrieve the OAI record // with GetRecord. @Test - public void testOaiFunctionality() throws InterruptedException { + public void testSingleRecordOaiSet() throws InterruptedException { - setupUsers(); - - // create dataverse: - Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey); - createDataverseResponse.prettyPrint(); - String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + //setupUsers(); - // publish dataverse: - Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, adminUserAPIKey); - assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode()); - - // create dataset: - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey); - createDatasetResponse.prettyPrint(); - Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); - - // retrieve the global id: - String datasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); - - // publish dataset: - Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPersistentId, "major", adminUserAPIKey); - assertEquals(200, publishDataset.getStatusCode()); - - String identifier = datasetPersistentId.substring(datasetPersistentId.lastIndexOf('/') + 1); - - logger.info("identifier: " + identifier); + - // Let's try and create an OAI set with the dataset we have just - // created and published: - // - however, publish command is executed asynchronously, i.e. it may - // still be running after we received the OK from the publish API. - // The oaiExport step also requires the metadata exports to be done and this - // takes longer than just publish/reindex. - // So wait for all of this to finish. - UtilIT.sleepForReexport(datasetPersistentId, adminUserAPIKey, 10); + // Let's try and create an OAI set with the "single set dataset" that + // was created as part of the initial setup: - String setName = identifier; - String setQuery = "dsPersistentId:" + identifier; + String setName = singleSetDatasetIdentifier; + String setQuery = "dsPersistentId:" + singleSetDatasetIdentifier; String apiPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; Response createSetResponse = given() @@ -277,12 +367,18 @@ public void testOaiFunctionality() throws InterruptedException { // There should be 1 and only 1 record in the response: assertEquals(1, ret.size()); // And the record should be the dataset we have just created: - assertEquals(datasetPersistentId, listIdentifiersResponse.getBody().xmlPath() + assertEquals(singleSetDatasetPersistentId, listIdentifiersResponse.getBody().xmlPath() .getString("OAI-PMH.ListIdentifiers.header.identifier")); break; } Thread.sleep(1000L); - } while (i")); // And now run GetRecord on the OAI record for the dataset: - Response getRecordResponse = UtilIT.getOaiRecord(datasetPersistentId, "oai_dc"); - - assertEquals(datasetPersistentId, getRecordResponse.getBody().xmlPath().getString("OAI-PMH.GetRecord.record.header.identifier")); + Response getRecordResponse = UtilIT.getOaiRecord(singleSetDatasetPersistentId, "oai_dc"); + + System.out.println("GetRecord response in its entirety: "+getRecordResponse.getBody().prettyPrint()); + System.out.println("one more time:"); + getRecordResponse.prettyPrint(); + + assertEquals(singleSetDatasetPersistentId, getRecordResponse.getBody().xmlPath().getString("OAI-PMH.GetRecord.record.header.identifier")); // TODO: // check the actual metadata payload of the OAI record more carefully? } + + // This test will attempt to create a set with multiple records (enough + // to trigger a paged response with a continuation token) and test its + // performance. + + + @Test + public void testMultiRecordOaiSet() throws InterruptedException { + + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 550d4ed1264..9fa47db167b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2620,6 +2620,16 @@ static Response exportOaiSet(String setName) { return given().put(apiPath); } + static Response getOaiIdentify() { + String oaiVerbPath = "/oai?verb=Identify"; + return given().get(oaiVerbPath); + } + + static Response getOaiListMetadataFormats() { + String oaiVerbPath = "/oai?verb=ListMetadataFormats"; + return given().get(oaiVerbPath); + } + static Response getOaiRecord(String datasetPersistentId, String metadataFormat) { String apiPath = String.format("/oai?verb=GetRecord&identifier=%s&metadataPrefix=%s", datasetPersistentId, metadataFormat); return given().get(apiPath); From 51fc6029c3a905d9f7c3fd5243b64fd4a8b6029e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 5 Dec 2022 20:43:49 -0500 Subject: [PATCH 218/232] small change in the guide per feedback (#7940) --- doc/sphinx-guides/source/admin/harvestclients.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 6a76f721162..e94a6aa1730 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -33,7 +33,7 @@ For example: sudo touch /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 sudo chown dataverse /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 -We recommend that stop stop any running harvesting jobs using this mechanism if you need to restart the application server, otherwise the ongoing harvest will be killed, but may be left marked as if it's still in progress in the database. +Note: If the application server is stopped and restarted, any running harvesting jobs will be killed but may remain marked as in progress in the database. We thus recommend using the mechanism here to stop ongoing harvests prior to a server restart. What if a Run Fails? From a19021089b46b6ac8051d8df313fd8e622145cb7 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 5 Dec 2022 21:20:16 -0500 Subject: [PATCH 219/232] typo (#7940) --- .../dataverse/harvest/client/HarvestingClientServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java index f2a3483c84f..13cc44ce919 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java @@ -167,7 +167,7 @@ public void deleteClient(Long clientId) { @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void setHarvestSuccess(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { - recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.INTERRUPTED); + recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.SUCCESS); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) From effd64f5043773ec36bb90ed283293ebb77d1586 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 6 Dec 2022 13:54:52 -0500 Subject: [PATCH 220/232] #3621 update placeholders for schema and namespace --- .../harvest/server/web/servlet/OAIServlet.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index 5eacb1addb6..3cfdcc1737d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -96,9 +96,15 @@ public class OAIServlet extends HttpServlet { // be calling ListIdentifiers, and then making direct calls to the export // API of the remote Dataverse, to obtain the records in native json. This // is how we should have implemented this in the first place, really. + /* + SEK + per #3621 we are adding urls to the namespace and schema + These will not resolve presently. the change is so that the + xml produced by https://demo.dataverse.org/oai?verb=ListMetadataFormats will validate + */ private static final String DATAVERSE_EXTENDED_METADATA_FORMAT = "dataverse_json"; - private static final String DATAVERSE_EXTENDED_METADATA_NAMESPACE = "Custom Dataverse metadata in JSON format (Dataverse4 to Dataverse4 harvesting only)"; - private static final String DATAVERSE_EXTENDED_METADATA_SCHEMA = "JSON schema pending"; + private static final String DATAVERSE_EXTENDED_METADATA_NAMESPACE = "https://dataverse.org/schema/core#"; + private static final String DATAVERSE_EXTENDED_METADATA_SCHEMA = "https://dataverse.org/schema/core.xsd"; private Context xoaiContext; private SetRepository setRepository; From 7a244406d36ad5f5a9ad6e01e1e29c149935324c Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Tue, 6 Dec 2022 16:02:47 -0500 Subject: [PATCH 221/232] #3621 remove # --- .../iq/dataverse/harvest/server/web/servlet/OAIServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index 3cfdcc1737d..f778fd56644 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -103,7 +103,7 @@ public class OAIServlet extends HttpServlet { xml produced by https://demo.dataverse.org/oai?verb=ListMetadataFormats will validate */ private static final String DATAVERSE_EXTENDED_METADATA_FORMAT = "dataverse_json"; - private static final String DATAVERSE_EXTENDED_METADATA_NAMESPACE = "https://dataverse.org/schema/core#"; + private static final String DATAVERSE_EXTENDED_METADATA_NAMESPACE = "https://dataverse.org/schema/core"; private static final String DATAVERSE_EXTENDED_METADATA_SCHEMA = "https://dataverse.org/schema/core.xsd"; private Context xoaiContext; From 8e70d995e8bffb4daa154e86a1e62e2c4f97788e Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 7 Dec 2022 12:32:23 -0500 Subject: [PATCH 222/232] added a release note (#7940) --- doc/release-notes/7940-stop-harvest-in-progress | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release-notes/7940-stop-harvest-in-progress diff --git a/doc/release-notes/7940-stop-harvest-in-progress b/doc/release-notes/7940-stop-harvest-in-progress new file mode 100644 index 00000000000..cb27a900f15 --- /dev/null +++ b/doc/release-notes/7940-stop-harvest-in-progress @@ -0,0 +1,4 @@ +## Mechanism added for stopping a harvest in progress + +It is now possible for an admin to stop a long-running harvesting job. See [Harvesting Clients](https://guides.dataverse.org/en/latest/admin/harvestclients.html) guide for more information. + From 2a87ae54ad8b78801889b46fe6aef7c273fea113 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 8 Dec 2022 09:45:12 -0500 Subject: [PATCH 223/232] add release note #8944 --- doc/release-notes/8944-metadatablocks.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/8944-metadatablocks.md diff --git a/doc/release-notes/8944-metadatablocks.md b/doc/release-notes/8944-metadatablocks.md new file mode 100644 index 00000000000..35bb7808e59 --- /dev/null +++ b/doc/release-notes/8944-metadatablocks.md @@ -0,0 +1,5 @@ +The API endpoint `/api/metadatablocks/{block_id}` has been extended to include the following fields: + +- `controlledVocabularyValues` - All possible values for fields with a controlled vocabulary. For example, the values "Agricultural Sciences", "Arts and Humanities", etc. for the "Subject" field. +- `isControlledVocabulary`: Whether or not this field has a controlled vocabulary. +- `multiple`: Whether or not the field supports multiple values. From ae83c2790f82956ea4164d8df8752c4f1df5e6c9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 8 Dec 2022 09:46:17 -0500 Subject: [PATCH 224/232] add docs #8944 --- .../source/admin/metadatacustomization.rst | 8 +++-- doc/sphinx-guides/source/api/native-api.rst | 33 ++++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 5f7cf85f714..9fb8626d4c4 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -386,12 +386,16 @@ Metadata Block Setup Now that you understand the TSV format used for metadata blocks, the next step is to attempt to make improvements to existing metadata blocks or create entirely new metadata blocks. For either task, you should have a Dataverse Software development environment set up for testing where you can drop the database frequently while you make edits to TSV files. Once you have tested your TSV files, you should consider making a pull request to contribute your improvement back to the community. +.. _exploring-metadata-blocks: + Exploring Metadata Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~ -In addition to studying the TSV files themselves you might find the following highly experimental and subject-to-change API endpoints useful to understand the metadata blocks that have already been loaded into your Dataverse installation: +In addition to studying the TSV files themselves you will probably find the :ref:`metadata-blocks-api` API helpful in getting a structured dump of metadata blocks in JSON format. + +There are also a few older, highly experimental, and subject-to-change API endpoints under the "admin" API documented below but the public API above is preferred. -You can get a dump of metadata fields (yes, the output is odd, please open a issue) like this: +You can get a dump of metadata fields like this: ``curl http://localhost:8080/api/admin/datasetfield`` diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 87a4d3def58..76ca38fdc70 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3007,22 +3007,47 @@ The fully expanded example above (without environment variables) looks like this curl https://demo.dataverse.org/api/info/apiTermsOfUse +.. _metadata-blocks-api: + Metadata Blocks --------------- +See also :ref:`exploring-metadata-blocks`. + Show Info About All Metadata Blocks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Lists brief info about all metadata blocks registered in the system:: +|CORS| Lists brief info about all metadata blocks registered in the system. + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + + curl $SERVER_URL/api/metadatablocks + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash - GET http://$SERVER/api/metadatablocks + curl https://demo.dataverse.org/api/metadatablocks Show Info About Single Metadata Block ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|CORS| Return data about the block whose ``identifier`` is passed. ``identifier`` can either be the block's id, or its name:: +|CORS| Return data about the block whose ``identifier`` is passed, including allowed controlled vocabulary values. ``identifier`` can either be the block's database id, or its name (i.e. "citation"). + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export IDENTIFIER=citation + + curl $SERVER_URL/api/metadatablocks/$IDENTIFIER + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash - GET http://$SERVER/api/metadatablocks/$identifier + curl https://demo.dataverse.org/api/metadatablocks/citation .. _Notifications: From c4f07f91446eedeee611a75537b3b90872817d0b Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 9 Dec 2022 17:57:29 -0500 Subject: [PATCH 225/232] more tests for the OAI server functionality (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 349 ++++++++++++------ .../edu/harvard/iq/dataverse/api/UtilIT.java | 5 + 2 files changed, 243 insertions(+), 111 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index 5355b57490d..d25ffd225d9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -9,24 +9,18 @@ import org.junit.Test; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import com.jayway.restassured.response.Response; -import com.jayway.restassured.path.json.JsonPath; import com.jayway.restassured.path.xml.XmlPath; import com.jayway.restassured.path.xml.element.Node; -import static edu.harvard.iq.dataverse.api.UtilIT.API_TOKEN_HTTP_HEADER; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import javax.json.Json; -import javax.json.JsonArray; -import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; -import org.junit.Ignore; import java.util.List; -import static junit.framework.Assert.assertEquals; +//import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; /** * Tests for the Harvesting Server functionality @@ -184,142 +178,204 @@ public void testOaiListMetadataFormats() { @Test - public void testSetCreateAPIandOAIlistIdentifiers() { - // Create the set with Dataverse /api/harvest/server API: + public void testNativeSetAPI() { String setName = UtilIT.getRandomString(6); String def = "*"; - - // make sure the set does not exist + + // This test focuses on the Create/List/Edit functionality of the + // Dataverse OAI Sets API (/api/harvest/server): + + // API Test 1. Make sure the set does not exist yet String setPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; - Response r0 = given() + Response getSetResponse = given() .get(setPath); - assertEquals(404, r0.getStatusCode()); + assertEquals(404, getSetResponse.getStatusCode()); - // try to create set as normal user, should fail - Response r1 = given() + // API Test 2. Try to create set as normal user, should fail + Response createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .body(jsonForTestSpec(setName, def)) .post(createPath); - assertEquals(400, r1.getStatusCode()); + assertEquals(400, createSetResponse.getStatusCode()); - // try to create set as admin user, should succeed - Response r2 = given() + // API Test 3. Try to create set as admin user, should succeed + createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForTestSpec(setName, def)) .post(createPath); - assertEquals(201, r2.getStatusCode()); + assertEquals(201, createSetResponse.getStatusCode()); - Response getSet = given() - .get(setPath); + // API Test 4. Retrieve the set we've just created, validate the response + getSetResponse = given().get(setPath); - logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); - logger.info("getSet printresponse: " + getSet.prettyPrint()); - assertEquals(200, getSet.getStatusCode()); + System.out.println("getSetResponse.getStatusCode(): " + getSetResponse.getStatusCode()); + System.out.println("getSetResponse, full: " + getSetResponse.prettyPrint()); + assertEquals(200, getSetResponse.getStatusCode()); + + getSetResponse.then().assertThat() + .body("status", equalTo(AbstractApiBean.STATUS_OK)) + .body("data.definition", equalTo("*")) + .body("data.description", equalTo("")) + .body("data.name", equalTo(setName)); + + // API Test 5. Retrieve all sets, check that our new set is listed Response responseAll = given() .get("/api/harvest/server/oaisets"); - logger.info("responseAll.getStatusCode(): " + responseAll.getStatusCode()); - logger.info("responseAll printresponse: " + responseAll.prettyPrint()); + System.out.println("responseAll.getStatusCode(): " + responseAll.getStatusCode()); + System.out.println("responseAll full: " + responseAll.prettyPrint()); assertEquals(200, responseAll.getStatusCode()); - - // try to create set with same name as admin user, should fail - Response r3 = given() + assertTrue(responseAll.body().jsonPath().getList("data.oaisets").size() > 0); + assertTrue(responseAll.body().jsonPath().getList("data.oaisets.name").toString().contains(setName)); // todo: simplify + + // API Test 6. Try to create a set with the same name, should fail + createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForTestSpec(setName, def)) .post(createPath); - assertEquals(400, r3.getStatusCode()); + assertEquals(400, createSetResponse.getStatusCode()); - // try to export set as admin user, should succeed (under admin API, not checking that normal user will fail) + // API Test 7. Try to export set as admin user, should succeed. Set export + // is under /api/admin, no need to try to access it as a non-admin user Response r4 = UtilIT.exportOaiSet(setName); assertEquals(200, r4.getStatusCode()); - - - - // try to delete as normal user, should fail - Response r5 = given() + + // API TEST 8. Try to delete the set as normal user, should fail + Response deleteResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .delete(setPath); - logger.info("r5.getStatusCode(): " + r5.getStatusCode()); - assertEquals(400, r5.getStatusCode()); + logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode()); + assertEquals(400, deleteResponse.getStatusCode()); - // try to delete as admin user, should work - Response r6 = given() + // API TEST 9. Delete as admin user, should work + deleteResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .delete(setPath); - logger.info("r6.getStatusCode(): " + r6.getStatusCode()); - assertEquals(200, r6.getStatusCode()); + logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode()); + assertEquals(200, deleteResponse.getStatusCode()); } @Test - public void testSetEdit() { - //setupUsers(); + public void testSetEditAPIandOAIlistSets() { + // This test focuses on testing the Edit functionality of the Dataverse + // OAI Set API and the ListSets method of the Dataverse OAI server. + + // Initial setup: crete a test set. + // Since the Create and List (POST and GET) functionality of the API + // is tested extensively in the previous test, we will not be paying + // as much attention to these methods, aside from confirming the + // expected HTTP result codes. + String setName = UtilIT.getRandomString(6); - String def = "*"; + String setDef = "*"; - // make sure the set does not exist - String u0 = String.format("/api/harvest/server/oaisets/%s", setName); + // Make sure the set does not exist + String setPath = String.format("/api/harvest/server/oaisets/%s", setName); String createPath ="/api/harvest/server/oaisets/add"; - Response r0 = given() - .get(u0); - assertEquals(404, r0.getStatusCode()); + Response getSetResponse = given() + .get(setPath); + assertEquals(404, getSetResponse.getStatusCode()); - // try to create set as admin user, should succeed - Response r1 = given() + // Create the set as admin user + Response createSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .body(jsonForTestSpec(setName, def)) + .body(jsonForTestSpec(setName, setDef)) .post(createPath); - assertEquals(201, r1.getStatusCode()); + assertEquals(201, createSetResponse.getStatusCode()); + // I. Test the Modify/Edit (POST method) functionality of the + // Dataverse OAI Sets API - // try to edit as normal user should fail - Response r2 = given() + String newDefinition = "title:New"; + String newDescription = "updated"; + + // API Test 1. Try to modify the set as normal user, should fail + Response editSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) - .body(jsonForEditSpec(setName, def,"")) - .put(u0); - logger.info("r2.getStatusCode(): " + r2.getStatusCode()); - assertEquals(400, r2.getStatusCode()); + .body(jsonForEditSpec(setName, setDef, "")) + .put(setPath); + logger.info("non-admin user editSetResponse.getStatusCode(): " + editSetResponse.getStatusCode()); + assertEquals(400, editSetResponse.getStatusCode()); - // try to edit as with blanks should fail - Response r3 = given() + // API Test 2. Try to modify as admin, but with invalid (empty) values, + // should fail + editSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(jsonForEditSpec(setName, "","")) - .put(u0); - logger.info("r3.getStatusCode(): " + r3.getStatusCode()); - assertEquals(400, r3.getStatusCode()); + .put(setPath); + logger.info("invalid values editSetResponse.getStatusCode(): " + editSetResponse.getStatusCode()); + assertEquals(400, editSetResponse.getStatusCode()); - // try to edit as with something should pass - Response r4 = given() + // API Test 3. Try to modify as admin, with sensible values + editSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .body(jsonForEditSpec(setName, "newDef","newDesc")) - .put(u0); - logger.info("r4 Status code: " + r4.getStatusCode()); - logger.info("r4.prettyPrint(): " + r4.prettyPrint()); - assertEquals(OK.getStatusCode(), r4.getStatusCode()); - - logger.info("u0: " + u0); - // now delete it... - Response r6 = given() + .body(jsonForEditSpec(setName, newDefinition, newDescription)) + .put(setPath); + logger.info("admin user editSetResponse status code: " + editSetResponse.getStatusCode()); + logger.info("admin user editSetResponse.prettyPrint(): " + editSetResponse.prettyPrint()); + assertEquals(OK.getStatusCode(), editSetResponse.getStatusCode()); + + // API Test 4. List the set, confirm that the new values are shown + getSetResponse = given().get(setPath); + + System.out.println("getSetResponse.getStatusCode(): " + getSetResponse.getStatusCode()); + System.out.println("getSetResponse, full: " + getSetResponse.prettyPrint()); + assertEquals(200, getSetResponse.getStatusCode()); + + getSetResponse.then().assertThat() + .body("status", equalTo(AbstractApiBean.STATUS_OK)) + .body("data.definition", equalTo(newDefinition)) + .body("data.description", equalTo(newDescription)) + .body("data.name", equalTo(setName)); + + // II. Test the ListSets functionality of the OAI server + + Response listSetsResponse = UtilIT.getOaiListSets(); + + // 1. Validate the service section of the OAI response: + + XmlPath responseXmlPath = validateOaiVerbResponse(listSetsResponse, "ListSets"); + + // 2. Validate the payload of the response, by confirming that the set + // we created and modified, above, is being listed by the OAI server + // and its xml record is properly formatted + + List listSets = responseXmlPath.getList("OAI-PMH.ListSets.set.list()"); // TODO - maybe try it with findAll()? + assertNotNull(listSets); + assertTrue(listSets.size() > 0); + + Node foundSetNode = null; + for (Node setNode : listSets) { + + if (setName.equals(setNode.get("setName").toString())) { + foundSetNode = setNode; + break; + } + } + + assertNotNull("Newly-created set is not listed by the OAI server", foundSetNode); + assertEquals("Incorrect description in the ListSets entry", newDescription, foundSetNode.getPath("setDescription.metadata.element.field", String.class)); + + // ok, the xml record looks good! + + // Cleanup. Delete the set with the DELETE API + Response deleteSetResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) - .delete(u0); - logger.info("r6.getStatusCode(): " + r6.getStatusCode()); - assertEquals(200, r6.getStatusCode()); + .delete(setPath); + assertEquals(200, deleteSetResponse.getStatusCode()); } - // A more elaborate test - we'll create and publish a dataset, then create an - // OAI set with that one dataset, and attempt to retrieve the OAI record - // with GetRecord. + // A more elaborate test - we will create and export an + // OAI set with a single dataset, and attempt to retrieve + // it and validate the OAI server responses of the corresponding + // ListIdentifiers, ListRecords and GetRecord methods. @Test public void testSingleRecordOaiSet() throws InterruptedException { - - //setupUsers(); - - - // Let's try and create an OAI set with the "single set dataset" that // was created as part of the initial setup: @@ -333,12 +389,18 @@ public void testSingleRecordOaiSet() throws InterruptedException { .post(createPath); assertEquals(201, createSetResponse.getStatusCode()); - // TODO: a) look up the set via native harvest/server api; - // b) look up the set via the OAI ListSets; - // export set: - // (this is asynchronous - so we should probably wait a little) + // The GET method of the oai set API, as well as the OAI ListSets + // method are tested extensively in another method in this class, so + // we'll skip checking those here. + + // Let's export the set. This is asynchronous - so we will try to + // wait a little - but in practice, everything potentially time-consuming + // must have been done when the dataset was exported, in the setup method. + Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); + Thread.sleep(1000L); + Response getSet = given() .get(apiPath); @@ -350,25 +412,38 @@ public void testSingleRecordOaiSet() throws InterruptedException { do { - // Run ListIdentifiers on this newly-created set: + // OAI Test 1. Run ListIdentifiers on this newly-created set: Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); - List ret = listIdentifiersResponse.getBody().xmlPath().getList("OAI-PMH.ListIdentifiers.header"); - assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header"); assertNotNull(ret); - logger.info("setName: " + setName); + if (logger.isLoggable(Level.FINE)) { logger.info("listIdentifiersResponse.prettyPrint:..... "); listIdentifiersResponse.prettyPrint(); } - if (ret.size() != 1) { + if (ret.isEmpty()) { + // OK, we'll sleep for another second - provided it's been less + // than 10 sec. total. i++; } else { - // There should be 1 and only 1 record in the response: + // Validate the payload of the ListRecords response: + // a) There should be 1 and only 1 record in the response: assertEquals(1, ret.size()); - // And the record should be the dataset we have just created: - assertEquals(singleSetDatasetPersistentId, listIdentifiersResponse.getBody().xmlPath() + // b) The one record in it should be the dataset we have just created: + assertEquals(singleSetDatasetPersistentId, responseXmlPath .getString("OAI-PMH.ListIdentifiers.header.identifier")); + assertEquals(setName, responseXmlPath + .getString("OAI-PMH.ListIdentifiers.header.setSpec")); + assertNotNull(responseXmlPath.getString("OAI-PMH.ListIdentifiers.header.dateStamp")); + // TODO: validate the formatting of the date string in the record + // header, above! + + // ok, ListIdentifiers response looks valid. break; } Thread.sleep(1000L); @@ -379,34 +454,86 @@ public void testSingleRecordOaiSet() throws InterruptedException { // already happened during its publishing (we made sure to wait there). // Exporting the set should not take any time - but I'll keep that code // in place since it's not going to hurt. - L.A. + System.out.println("Waited " + i + " seconds for OIA export."); //Fail if we didn't find the exported record before the timeout assertTrue(i < maxWait); + + + // OAI Test 2. Run ListRecords, request oai_dc: Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc"); assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); - List listRecords = listRecordsResponse.getBody().xmlPath().getList("OAI-PMH.ListRecords.record"); + + // Validate the service section of the OAI response: + + XmlPath responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + // Validate the payload of the response: + // (the header portion must be identical to that of ListIdentifiers above, + // plus the response must contain a metadata section with a valid oai_dc + // record) + + List listRecords = responseXmlPath.getList("OAI-PMH.ListRecords.record"); + // Same deal, there must be 1 record only in the set: assertNotNull(listRecords); assertEquals(1, listRecords.size()); - assertEquals(singleSetDatasetPersistentId, listRecordsResponse.getBody().xmlPath().getString("OAI-PMH.ListRecords.record[0].header.identifier")); - - // assert that Datacite format does not contain the XML prolog + // a) header section: + assertEquals(singleSetDatasetPersistentId, responseXmlPath.getString("OAI-PMH.ListRecords.record.header.identifier")); + assertEquals(setName, responseXmlPath + .getString("OAI-PMH.ListRecords.record.header.setSpec")); + assertNotNull(responseXmlPath.getString("OAI-PMH.ListRecords.record.header.dateStamp")); + // b) metadata section: + // in the metadata section we are showing the resolver url form of the doi: + String persistentIdUrl = singleSetDatasetPersistentId.replace("doi:", "https://doi.org/"); + assertEquals(persistentIdUrl, responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.identifier")); + assertEquals("Darwin's Finches", responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.title")); + assertEquals("Finch, Fiona", responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.creator")); + assertEquals("Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", + responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.description")); + assertEquals("Medicine, Health and Life Sciences", + responseXmlPath.getString("OAI-PMH.ListRecords.record.metadata.dc.subject")); + // ok, looks legit! + + // OAI Test 3. + // Assert that Datacite format does not contain the XML prolog + // (this is a reference to a resolved issue; generally, harvestable XML + // exports must NOT contain the "")); - // And now run GetRecord on the OAI record for the dataset: - Response getRecordResponse = UtilIT.getOaiRecord(singleSetDatasetPersistentId, "oai_dc"); + // OAI Test 4. run and validate GetRecord response + Response getRecordResponse = UtilIT.getOaiRecord(singleSetDatasetPersistentId, "oai_dc"); System.out.println("GetRecord response in its entirety: "+getRecordResponse.getBody().prettyPrint()); - System.out.println("one more time:"); - getRecordResponse.prettyPrint(); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(getRecordResponse, "GetRecord"); + + // Validate the payload of the response: + + // Note that for a set with a single record the output of ListRecrods is + // essentially identical to that of GetRecord! + // (we'll test a multi-record set in a different method) + // a) header section: + assertEquals(singleSetDatasetPersistentId, responseXmlPath.getString("OAI-PMH.GetRecord.record.header.identifier")); + assertEquals(setName, responseXmlPath + .getString("OAI-PMH.GetRecord.record.header.setSpec")); + assertNotNull(responseXmlPath.getString("OAI-PMH.GetRecord.record.header.dateStamp")); + // b) metadata section: + assertEquals(persistentIdUrl, responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.identifier")); + assertEquals("Darwin's Finches", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.title")); + assertEquals("Finch, Fiona", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.creator")); + assertEquals("Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds.", + responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.description")); + assertEquals("Medicine, Health and Life Sciences", responseXmlPath.getString("OAI-PMH.GetRecord.record.metadata.dc.subject")); - assertEquals(singleSetDatasetPersistentId, getRecordResponse.getBody().xmlPath().getString("OAI-PMH.GetRecord.record.header.identifier")); + // ok, looks legit! - // TODO: - // check the actual metadata payload of the OAI record more carefully? } // This test will attempt to create a set with multiple records (enough diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9fa47db167b..ac767279bd4 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2630,6 +2630,11 @@ static Response getOaiListMetadataFormats() { return given().get(oaiVerbPath); } + static Response getOaiListSets() { + String oaiVerbPath = "/oai?verb=ListSets"; + return given().get(oaiVerbPath); + } + static Response getOaiRecord(String datasetPersistentId, String metadataFormat) { String apiPath = String.format("/oai?verb=GetRecord&identifier=%s&metadataPrefix=%s", datasetPersistentId, metadataFormat); return given().get(apiPath); From 9cbfa31d4489ed4ce6df6e37a0fecf92f3a77d18 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 12 Dec 2022 13:51:58 -0500 Subject: [PATCH 226/232] extra (extra tedious) server tests validating paging (resumptionToken) functionality of ListIdentifiers and ListRecords (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 340 +++++++++++++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 18 +- 2 files changed, 351 insertions(+), 7 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index d25ffd225d9..3497c71e169 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -16,6 +16,8 @@ import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; import java.util.List; +import java.util.Set; +import java.util.HashSet; //import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -35,6 +37,7 @@ public class HarvestingServerIT { private static String adminUserAPIKey; private static String singleSetDatasetIdentifier; private static String singleSetDatasetPersistentId; + private static List extraDatasetsIdentifiers = new ArrayList<>(); @BeforeClass public static void setUpClass() { @@ -98,6 +101,28 @@ private static void setupDatasets() { // takes longer than just publish/reindex. // So wait for all of this to finish. UtilIT.sleepForReexport(singleSetDatasetPersistentId, adminUserAPIKey, 10); + + // ... And let's create 4 more datasets for a multi-dataset experiment: + + for (int i = 0; i < 4; i++) { + // create dataset: + createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, adminUserAPIKey); + createDatasetResponse.prettyPrint(); + datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + + // retrieve the global id: + String thisDatasetPersistentId = UtilIT.getDatasetPersistentIdFromResponse(createDatasetResponse); + + // publish dataset: + publishDataset = UtilIT.publishDatasetViaNativeApi(thisDatasetPersistentId, "major", adminUserAPIKey); + assertEquals(200, publishDataset.getStatusCode()); + + UtilIT.sleepForReexport(thisDatasetPersistentId, adminUserAPIKey, 10); + + extraDatasetsIdentifiers.add(thisDatasetPersistentId.substring(thisDatasetPersistentId.lastIndexOf('/') + 1)); + } + + } private String jsonForTestSpec(String name, String def) { @@ -423,16 +448,16 @@ public void testSingleRecordOaiSet() throws InterruptedException { assertNotNull(ret); if (logger.isLoggable(Level.FINE)) { - logger.info("listIdentifiersResponse.prettyPrint:..... "); - listIdentifiersResponse.prettyPrint(); + logger.info("listIdentifiersResponse.prettyPrint: " + + listIdentifiersResponse.prettyPrint()); } if (ret.isEmpty()) { // OK, we'll sleep for another second - provided it's been less // than 10 sec. total. i++; } else { - // Validate the payload of the ListRecords response: - // a) There should be 1 and only 1 record in the response: + // Validate the payload of the ListIdentifiers response: + // a) There should be 1 and only 1 item listed: assertEquals(1, ret.size()); // b) The one record in it should be the dataset we have just created: assertEquals(singleSetDatasetPersistentId, responseXmlPath @@ -537,12 +562,315 @@ public void testSingleRecordOaiSet() throws InterruptedException { } // This test will attempt to create a set with multiple records (enough - // to trigger a paged response with a continuation token) and test its - // performance. + // to trigger a paged respons) and test the resumption token functionality). + // Note that this test requires the OAI service to be configured with some + // non-default settings (the paging limits for ListIdentifiers and ListRecords + // must be set to something low, like 2). @Test public void testMultiRecordOaiSet() throws InterruptedException { + // Setup: Let's create a control OAI set with the 5 datasets created + // in the class init: + + String setName = UtilIT.getRandomString(6); + String setQuery = "(dsPersistentId:" + singleSetDatasetIdentifier; + for (String persistentId : extraDatasetsIdentifiers) { + setQuery = setQuery.concat(" OR dsPersistentId:" + persistentId); + } + setQuery = setQuery.concat(")"); + + String createPath = "/api/harvest/server/oaisets/add"; + + Response createSetResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(jsonForTestSpec(setName, setQuery)) + .post(createPath); + assertEquals(201, createSetResponse.getStatusCode()); + + // Dataverse OAI Sets API is tested extensively in other methods here, + // so no need to test in any more details than confirming the OK result + // above + Response exportSetResponse = UtilIT.exportOaiSet(setName); + assertEquals(200, exportSetResponse.getStatusCode()); + Thread.sleep(1000L); + + // OAI Test 1. Run ListIdentifiers on the set we've just created: + Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); + assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint()); + } + + // Validate the payload of the ListIdentifiers response: + // 1a) There should be 2 items listed: + assertEquals("Wrong number of items on the first ListIdentifiers page", + 2, ret.size()); + + // 1b) The response contains a resumptionToken for the next page of items: + String resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); + assertNotNull("No resumption token in the ListIdentifiers response", resumptionToken); + + // 1c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); + + // 1d) ... and the offset (cursor) is at the right position (0): + assertEquals(0, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor")); + + // The formatting of individual item records in the ListIdentifiers response + // is tested extensively in the previous test method, so we are not + // looking at them in such detail here; but we should record the + // identifiers listed, so that we can confirm that all the set is + // served as expected. + + Set persistentIdsInListIdentifiers = new HashSet<>(); + + for (String persistentId : ret) { + persistentIdsInListIdentifiers.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // ok, let's move on to the next ListIdentifiers page: + // (we repeat the exact same checks as the above; minus the different + // expected offset) + + // OAI Test 2. Run ListIdentifiers with the resumptionToken obtained + // in the previous step: + + listIdentifiersResponse = UtilIT.getOaiListIdentifiersWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint()); + } + + // Validate the payload of the ListIdentifiers response: + // 2a) There should still be 2 items listed: + assertEquals("Wrong number of items on the second ListIdentifiers page", + 2, ret.size()); + + // 2b) The response should contain a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); + assertNotNull("No resumption token in the ListIdentifiers response", resumptionToken); + + // 2c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); + + // 2d) ... and the offset (cursor) is at the right position (2): + assertEquals(2, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor")); + + // Record the identifiers listed on this results page: + + for (String persistentId : ret) { + persistentIdsInListIdentifiers.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // And now the next and the final ListIdentifiers page. + // This time around we should get an *empty* resumptionToken (indicating + // that there are no more results): + + // OAI Test 3. Run ListIdentifiers with the final resumptionToken + + listIdentifiersResponse = UtilIT.getOaiListIdentifiersWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listIdentifiersResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); + + ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: "+listIdentifiersResponse.prettyPrint()); + } + + // Validate the payload of the ListIdentifiers response: + // 3a) There should be only 1 item listed: + assertEquals("Wrong number of items on the final ListIdentifiers page", + 1, ret.size()); + + // 3b) The response contains a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); + assertNotNull("No resumption token in the final ListIdentifiers response", resumptionToken); + assertTrue("Non-empty resumption token in the final ListIdentifiers response", "".equals(resumptionToken)); + + // 3c) The total number of items in the set (5) is still listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); + + // 3d) ... and the offset (cursor) is at the right position (4): + assertEquals(4, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@cursor")); + // Record the last identifier listed on this final page: + persistentIdsInListIdentifiers.add(ret.get(0).substring(ret.get(0).lastIndexOf('/') + 1)); + + // Finally, let's confirm that the expected 5 datasets have been listed + // as part of this Set: + + boolean allDatasetsListed = true; + + allDatasetsListed = persistentIdsInListIdentifiers.contains(singleSetDatasetIdentifier); + for (String persistentId : extraDatasetsIdentifiers) { + allDatasetsListed = persistentIdsInListIdentifiers.contains(persistentId); + } + + assertTrue("Control datasets not properly listed in the paged ListIdentifiers response", + allDatasetsListed); + + // OK, it is safe to assume ListIdentifiers works as it should in page mode. + + // We will now repeat the exact same tests for ListRecords (again, no + // need to pay close attention to the formatting of the individual records, + // since that's tested in the previous test method, since our focus is + // testing the paging/resumptionToken functionality) + + // OAI Test 4. Run ListRecords on the set we've just created: + Response listRecordsResponse = UtilIT.getOaiListRecords(setName, "oai_dc"); + assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint()); + } + + // Validate the payload of the ListRecords response: + // 4a) There should be 2 items listed: + assertEquals("Wrong number of items on the first ListRecords page", + 2, ret.size()); + + // 4b) The response contains a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); + assertNotNull("No resumption token in the ListRecords response", resumptionToken); + + // 4c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); + + // 4d) ... and the offset (cursor) is at the right position (0): + assertEquals(0, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor")); + + Set persistentIdsInListRecords = new HashSet<>(); + + for (String persistentId : ret) { + persistentIdsInListRecords.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // ok, let's move on to the next ListRecords page: + // (we repeat the exact same checks as the above; minus the different + // expected offset) + + // OAI Test 5. Run ListRecords with the resumptionToken obtained + // in the previous step: + + listRecordsResponse = UtilIT.getOaiListRecordsWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint()); + } + + // Validate the payload of the ListRecords response: + // 4a) There should still be 2 items listed: + assertEquals("Wrong number of items on the second ListRecords page", + 2, ret.size()); + + // 4b) The response should contain a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); + assertNotNull("No resumption token in the ListRecords response", resumptionToken); + + // 4c) The total number of items in the set (5) is listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); + + // 4d) ... and the offset (cursor) is at the right position (2): + assertEquals(2, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor")); + + // Record the identifiers listed on this results page: + + for (String persistentId : ret) { + persistentIdsInListRecords.add(persistentId.substring(persistentId.lastIndexOf('/') + 1)); + } + + // And now the next and the final ListRecords page. + // This time around we should get an *empty* resumptionToken (indicating + // that there are no more results): + + // OAI Test 6. Run ListRecords with the final resumptionToken + + listRecordsResponse = UtilIT.getOaiListRecordsWithResumptionToken(resumptionToken); + assertEquals(OK.getStatusCode(), listRecordsResponse.getStatusCode()); + + // Validate the service section of the OAI response: + responseXmlPath = validateOaiVerbResponse(listRecordsResponse, "ListRecords"); + + ret = responseXmlPath.getList("OAI-PMH.ListRecords.record.header.identifier"); + assertNotNull(ret); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listRecordsResponse.prettyPrint: "+listRecordsResponse.prettyPrint()); + } + + // Validate the payload of the ListRecords response: + // 6a) There should be only 1 item listed: + assertEquals("Wrong number of items on the final ListRecords page", + 1, ret.size()); + + // 6b) The response contains a resumptionToken for the next page of items: + resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); + assertNotNull("No resumption token in the final ListRecords response", resumptionToken); + assertTrue("Non-empty resumption token in the final ListRecords response", "".equals(resumptionToken)); + + // 6c) The total number of items in the set (5) is still listed correctly: + assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); + + // 6d) ... and the offset (cursor) is at the right position (4): + assertEquals(4, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@cursor")); + + // Record the last identifier listed on this final page: + persistentIdsInListRecords.add(ret.get(0).substring(ret.get(0).lastIndexOf('/') + 1)); + + // Finally, let's confirm that the expected 5 datasets have been listed + // as part of this Set: + + allDatasetsListed = true; + + allDatasetsListed = persistentIdsInListRecords.contains(singleSetDatasetIdentifier); + for (String persistentId : extraDatasetsIdentifiers) { + allDatasetsListed = persistentIdsInListRecords.contains(persistentId); + } + + assertTrue("Control datasets not properly listed in the paged ListRecords response", + allDatasetsListed); + + // OK, it is safe to assume ListRecords works as it should in page mode + // as well. + + // And finally, let's delete the set + String setPath = String.format("/api/harvest/server/oaisets/%s", setName); + Response deleteResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .delete(setPath); + logger.info("deleteResponse.getStatusCode(): " + deleteResponse.getStatusCode()); + assertEquals("Failed to delete the control multi-record set", 200, deleteResponse.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index ac767279bd4..e669a268010 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2641,7 +2641,18 @@ static Response getOaiRecord(String datasetPersistentId, String metadataFormat) } static Response getOaiListIdentifiers(String setName, String metadataFormat) { - String apiPath = String.format("/oai?verb=ListIdentifiers&set=%s&metadataPrefix=%s", setName, metadataFormat); + + String apiPath; + if (StringUtil.nonEmpty(setName)) { + apiPath = String.format("/oai?verb=ListIdentifiers&set=%s&metadataPrefix=%s", setName, metadataFormat); + } else { + apiPath = String.format("/oai?verb=ListIdentifiers&metadataPrefix=%s", metadataFormat); + } + return given().get(apiPath); + } + + static Response getOaiListIdentifiersWithResumptionToken(String resumptionToken) { + String apiPath = String.format("/oai?verb=ListIdentifiers&resumptionToken=%s", resumptionToken); return given().get(apiPath); } @@ -2649,6 +2660,11 @@ static Response getOaiListRecords(String setName, String metadataFormat) { String apiPath = String.format("/oai?verb=ListRecords&set=%s&metadataPrefix=%s", setName, metadataFormat); return given().get(apiPath); } + + static Response getOaiListRecordsWithResumptionToken(String resumptionToken) { + String apiPath = String.format("/oai?verb=ListRecords&resumptionToken=%s", resumptionToken); + return given().get(apiPath); + } static Response changeAuthenticatedUserIdentifier(String oldIdentifier, String newIdentifier, String apiToken) { Response response; From 395d605a8e156dd2ee295a8aa2a0892cad898617 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 12 Dec 2022 17:04:44 -0500 Subject: [PATCH 227/232] An automated test of an actual harvest (#8843) --- .../iq/dataverse/api/HarvestingClients.java | 31 +--- .../iq/dataverse/api/HarvestingClientsIT.java | 169 ++++++++++++++++-- .../iq/dataverse/api/HarvestingServerIT.java | 8 + 3 files changed, 164 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index 42534514b68..b75cb687c62 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -373,13 +373,13 @@ public Response startHarvestingJob(@PathParam("nickName") String clientNickname, } if (authenticatedUser == null || !authenticatedUser.isSuperuser()) { - return error(Response.Status.FORBIDDEN, "Only the Dataverse Admin user can run harvesting jobs"); + return error(Response.Status.FORBIDDEN, "Only admin users can run harvesting jobs"); } HarvestingClient harvestingClient = harvestingClientService.findByNickname(clientNickname); if (harvestingClient == null) { - return error(Response.Status.NOT_FOUND, "No such dataverse: "+clientNickname); + return error(Response.Status.NOT_FOUND, "No such client: "+clientNickname); } DataverseRequest dataverseRequest = createDataverseRequest(authenticatedUser); @@ -391,35 +391,8 @@ public Response startHarvestingJob(@PathParam("nickName") String clientNickname, return this.accepted(); } - // This GET shows the status of the harvesting run in progress for this - // client, if present: - // @GET - // @Path("{nickName}/run") - // TODO: - - // This DELETE kills the harvesting run in progress for this client, - // if present: - // @DELETE - // @Path("{nickName}/run") - // TODO: - - - - - /* Auxiliary, helper methods: */ - /* - @Deprecated - public static JsonArrayBuilder harvestingConfigsAsJsonArray(List harvestingDataverses) { - JsonArrayBuilder hdArr = Json.createArrayBuilder(); - - for (Dataverse hd : harvestingDataverses) { - hdArr.add(harvestingConfigAsJson(hd.getHarvestingClientConfig())); - } - return hdArr; - }*/ - public static JsonObjectBuilder harvestingConfigAsJson(HarvestingClient harvestingConfig) { if (harvestingConfig == null) { return null; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 9eac3545e54..8fef360c68b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -1,34 +1,58 @@ package edu.harvard.iq.dataverse.api; import java.util.logging.Logger; +import java.util.logging.Level; import com.jayway.restassured.RestAssured; import static com.jayway.restassured.RestAssured.given; import org.junit.Test; import com.jayway.restassured.response.Response; +import static javax.ws.rs.core.Response.Status.CREATED; +import static javax.ws.rs.core.Response.Status.UNAUTHORIZED; +import static javax.ws.rs.core.Response.Status.ACCEPTED; +import static javax.ws.rs.core.Response.Status.OK; import static org.hamcrest.CoreMatchers.equalTo; -import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import org.junit.BeforeClass; /** - * extremely minimal (for now) API tests for creating OAI clients. + * This class tests Harvesting Client functionality. + * Note that these methods test BOTH the proprietary Dataverse rest API for + * creating and managing harvesting clients, AND the underlining OAI-PMH + * harvesting functionality itself. I.e., we will use the Dataverse + * /api/harvest/clients/ api to run an actual harvest of a control set and + * then validate the resulting harvested content. */ public class HarvestingClientsIT { private static final Logger logger = Logger.getLogger(HarvestingClientsIT.class.getCanonicalName()); private static final String harvestClientsApi = "/api/harvest/clients/"; - private static final String harvestCollection = "root"; + private static final String rootCollection = "root"; private static final String harvestUrl = "https://demo.dataverse.org/oai"; private static final String archiveUrl = "https://demo.dataverse.org"; private static final String harvestMetadataFormat = "oai_dc"; private static final String archiveDescription = "RestAssured harvesting client test"; + private static final String controlOaiSet = "controlTestSet"; + private static final int datasetsInControlSet = 7; + private static String normalUserAPIKey; + private static String adminUserAPIKey; + private static String harvestCollectionAlias; @BeforeClass public static void setUpClass() { RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + + // Create the users, an admin and a non-admin: + setupUsers(); + + // Create a collection that we will use to harvest remote content into: + setupCollection(); + } - private void setupUsers() { + private static void setupUsers() { Response cu0 = UtilIT.createRandomUser(); normalUserAPIKey = UtilIT.getApiTokenFromResponse(cu0); Response cu1 = UtilIT.createRandomUser(); @@ -36,13 +60,22 @@ private void setupUsers() { Response u1a = UtilIT.makeSuperUser(un1); adminUserAPIKey = UtilIT.getApiTokenFromResponse(cu1); } + + private static void setupCollection() { + Response createDataverseResponse = UtilIT.createRandomDataverse(adminUserAPIKey); + createDataverseResponse.prettyPrint(); + assertEquals(CREATED.getStatusCode(), createDataverseResponse.getStatusCode()); + + harvestCollectionAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - private String normalUserAPIKey; - private String adminUserAPIKey; + // publish dataverse: + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(harvestCollectionAlias, adminUserAPIKey); + assertEquals(OK.getStatusCode(), publishDataverse.getStatusCode()); + } @Test public void testCreateEditDeleteClient() { - setupUsers(); + //setupUsers(); String nickName = UtilIT.getRandomString(6); @@ -52,7 +85,7 @@ public void testCreateEditDeleteClient() { + "\"harvestUrl\":\"%s\"," + "\"archiveUrl\":\"%s\"," + "\"metadataFormat\":\"%s\"}", - harvestCollection, harvestUrl, archiveUrl, harvestMetadataFormat); + rootCollection, harvestUrl, archiveUrl, harvestMetadataFormat); // Try to create a client as normal user, should fail: @@ -61,7 +94,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .body(clientJson) .post(clientApiPath); - assertEquals(401, rCreate.getStatusCode()); + assertEquals(UNAUTHORIZED.getStatusCode(), rCreate.getStatusCode()); // Try to create the same as admin user, should succeed: @@ -70,7 +103,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(clientJson) .post(clientApiPath); - assertEquals(201, rCreate.getStatusCode()); + assertEquals(CREATED.getStatusCode(), rCreate.getStatusCode()); // Try to update the client we have just created: @@ -80,7 +113,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .body(updateJson) .put(clientApiPath); - assertEquals(200, rUpdate.getStatusCode()); + assertEquals(OK.getStatusCode(), rUpdate.getStatusCode()); // Now let's retrieve the client we've just created and edited: @@ -89,7 +122,7 @@ public void testCreateEditDeleteClient() { logger.info("getClient.getStatusCode(): " + getClientResponse.getStatusCode()); logger.info("getClient printresponse: " + getClientResponse.prettyPrint()); - assertEquals(200, getClientResponse.getStatusCode()); + assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode()); // ... and validate the values: @@ -98,7 +131,7 @@ public void testCreateEditDeleteClient() { .body("data.type", equalTo("oai")) .body("data.nickName", equalTo(nickName)) .body("data.archiveDescription", equalTo(archiveDescription)) - .body("data.dataverseAlias", equalTo(harvestCollection)) + .body("data.dataverseAlias", equalTo(rootCollection)) .body("data.harvestUrl", equalTo(harvestUrl)) .body("data.archiveUrl", equalTo(archiveUrl)) .body("data.metadataFormat", equalTo(harvestMetadataFormat)); @@ -109,7 +142,7 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, normalUserAPIKey) .delete(clientApiPath); logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode()); - assertEquals(401, rDelete.getStatusCode()); + assertEquals(UNAUTHORIZED.getStatusCode(), rDelete.getStatusCode()); // Try to delete as admin user should work: @@ -117,6 +150,112 @@ public void testCreateEditDeleteClient() { .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) .delete(clientApiPath); logger.info("rDelete.getStatusCode(): " + rDelete.getStatusCode()); - assertEquals(200, rDelete.getStatusCode()); + assertEquals(OK.getStatusCode(), rDelete.getStatusCode()); + } + + @Test + public void testHarvestingClientRun() throws InterruptedException { + // This test will create a client and attempt to perform an actual + // harvest and validate the resulting harvested content. + + // Setup: create the client via the API + // since this API is tested somewhat extensively in the previous + // method, we don't need to pay too much attention to this method, aside + // from confirming the expected HTTP status code. + + String nickName = UtilIT.getRandomString(6); + + String clientApiPath = String.format(harvestClientsApi+"%s", nickName); + String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + + "\"type\":\"oai\"," + + "\"harvestUrl\":\"%s\"," + + "\"archiveUrl\":\"%s\"," + + "\"set\":\"%s\"," + + "\"metadataFormat\":\"%s\"}", + harvestCollectionAlias, harvestUrl, archiveUrl, controlOaiSet, harvestMetadataFormat); + + Response createResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .body(clientJson) + .post(clientApiPath); + assertEquals(CREATED.getStatusCode(), createResponse.getStatusCode()); + + // API TEST 1. Run the harvest using the configuration (client) we have + // just created + + String runHarvestApiPath = String.format(harvestClientsApi+"%s/run", nickName); + + // TODO? - verify that a non-admin user cannot perform this operation (401) + + Response runResponse = given() + .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) + .post(runHarvestApiPath); + assertEquals(ACCEPTED.getStatusCode(), runResponse.getStatusCode()); + + // API TEST 2. As indicated by the ACCEPTED status code above, harvesting + // is an asynchronous operation that will be performed in the background. + // Verify that this "in progress" status is properly reported while it's + // running, and that it completes in some reasonable amount of time. + + int i = 0; + int maxWait=20; // a very conservative interval; this harvest has no business taking this long + do { + // keep checking the status of the client with the GET api: + Response getClientResponse = given() + .get(clientApiPath); + + assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode()); + assertEquals(AbstractApiBean.STATUS_OK, getClientResponse.body().jsonPath().getString("status")); + + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: " + + getClientResponse.prettyPrint()); + } + + String clientStatus = getClientResponse.body().jsonPath().getString("data.status"); + assertNotNull(clientStatus); + + if ("inProgress".equals(clientStatus)) { + // we'll sleep for another second + i++; + } else { + // Check the values in the response: + // a) Confirm that the harvest has completed: + assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); + + // b) Confirm that it has actually succeeded: + assertEquals("Last harvest not reported a success", "SUCCESS", getClientResponse.body().jsonPath().getString("data.lastResult")); + String harvestTimeStamp = getClientResponse.body().jsonPath().getString("data.lastHarvest"); + assertNotNull(harvestTimeStamp); + + // c) Confirm that the other timestamps match: + assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastSuccessful")); + assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastNonEmpty")); + + // d) Confirm that the correct number of datasets have been harvested: + assertEquals(datasetsInControlSet, getClientResponse.body().jsonPath().getInt("data.lastDatasetsHarvested")); + + // ok, it looks like the harvest has completed successfully. + break; + } + Thread.sleep(1000L); + } while (i Date: Mon, 12 Dec 2022 17:10:35 -0500 Subject: [PATCH 228/232] comments (#8843) --- .../iq/dataverse/api/HarvestingClientsIT.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 8fef360c68b..448faa20b0b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -75,7 +75,9 @@ private static void setupCollection() { @Test public void testCreateEditDeleteClient() { - //setupUsers(); + // This method focuses on testing the native Dataverse harvesting client + // API. + String nickName = UtilIT.getRandomString(6); @@ -158,7 +160,7 @@ public void testHarvestingClientRun() throws InterruptedException { // This test will create a client and attempt to perform an actual // harvest and validate the resulting harvested content. - // Setup: create the client via the API + // Setup: create the client via native API // since this API is tested somewhat extensively in the previous // method, we don't need to pay too much attention to this method, aside // from confirming the expected HTTP status code. @@ -246,8 +248,11 @@ public void testHarvestingClientRun() throws InterruptedException { // Fail if it hasn't completed in maxWait seconds assertTrue(i < maxWait); - // TODO: use the native Dataverses/Datasets apis to verify that the expected - // datasets have been harvested. + // TODO(?) use the native Dataverses/Datasets apis to verify that the expected + // datasets have been harvested. This may or may not be necessary, seeing + // how we have already confirmed the number of successfully harvested + // datasets from the control set; somewhat hard to imagine a practical + // situation where that would not be enough (?). // Cleanup: delete the client From 8e310c35801d5ce6c1f033236d2b588d6ef1f9a9 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 13 Dec 2022 13:54:24 -0500 Subject: [PATCH 229/232] logic! (#8843) --- .../iq/dataverse/api/HarvestingServerIT.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index b5563c926e5..dad32bcaa60 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -565,7 +565,11 @@ public void testSingleRecordOaiSet() throws InterruptedException { // to trigger a paged respons) and test the resumption token functionality). // Note that this test requires the OAI service to be configured with some // non-default settings (the paging limits for ListIdentifiers and ListRecords - // must be set to something low, like 2). + // must be set to 2, in order to be able to trigger this paging behavior without + // having to create and export too many datasets). + // So you will need to do this: + // asadmin create-jvm-options "-Ddataverse.oai.server.maxidentifiers=2" + // asadmin create-jvm-options "-Ddataverse.oai.server.maxrecords=2" @Test @@ -616,7 +620,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { // 1b) The response contains a resumptionToken for the next page of items: String resumptionToken = responseXmlPath.getString("OAI-PMH.ListIdentifiers.resumptionToken"); - assertNotNull("No resumption token in the ListIdentifiers response", resumptionToken); + assertNotNull("No resumption token in the ListIdentifiers response (has the jvm option dataverse.oai.server.maxidentifiers been configured?)", resumptionToken); // 1c) The total number of items in the set (5) is listed correctly: assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListIdentifiers.resumptionToken.@completeListSize")); @@ -722,7 +726,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { allDatasetsListed = persistentIdsInListIdentifiers.contains(singleSetDatasetIdentifier); for (String persistentId : extraDatasetsIdentifiers) { - allDatasetsListed = persistentIdsInListIdentifiers.contains(persistentId); + allDatasetsListed = allDatasetsListed && persistentIdsInListIdentifiers.contains(persistentId); } assertTrue("Control datasets not properly listed in the paged ListIdentifiers response", @@ -756,7 +760,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { // 4b) The response contains a resumptionToken for the next page of items: resumptionToken = responseXmlPath.getString("OAI-PMH.ListRecords.resumptionToken"); - assertNotNull("No resumption token in the ListRecords response", resumptionToken); + assertNotNull("No resumption token in the ListRecords response (has the jvm option dataverse.oai.server.maxrecords been configured?)", resumptionToken); // 4c) The total number of items in the set (5) is listed correctly: assertEquals(5, responseXmlPath.getInt("OAI-PMH.ListRecords.resumptionToken.@completeListSize")); @@ -856,7 +860,7 @@ public void testMultiRecordOaiSet() throws InterruptedException { allDatasetsListed = persistentIdsInListRecords.contains(singleSetDatasetIdentifier); for (String persistentId : extraDatasetsIdentifiers) { - allDatasetsListed = persistentIdsInListRecords.contains(persistentId); + allDatasetsListed = allDatasetsListed && persistentIdsInListRecords.contains(persistentId); } assertTrue("Control datasets not properly listed in the paged ListRecords response", @@ -879,6 +883,6 @@ public void testMultiRecordOaiSet() throws InterruptedException { // Some ideas: // - Test handling of deleted dataset records // - Test "from" and "until" time parameters - // - Test validating full verb response records against XML schema + // - Validate full verb response records against XML schema // (for each supported metadata format, possibly?) } From b5986fa94c954de72f4280e1e9bde81ed9389910 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 13 Dec 2022 16:44:38 -0500 Subject: [PATCH 230/232] cleanup (#8843) --- .../harvest/client/FastGetRecord.java | 2 +- .../harvest/client/HarvesterServiceBean.java | 2 +- .../iq/dataverse/api/HarvestingClientsIT.java | 66 +++++++------- .../iq/dataverse/api/HarvestingServerIT.java | 88 ++++++++----------- 4 files changed, 73 insertions(+), 85 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java index 5b3e4df331d..c5e3a93e2df 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java @@ -130,7 +130,7 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref int responseCode = 0; con = (HttpURLConnection) url.openConnection(); - con.setRequestProperty("User-Agent", "DataverseHarvester/3.0"); + con.setRequestProperty("User-Agent", "Dataverse Harvesting Client v5"); con.setRequestProperty("Accept-Encoding", "compress, gzip, identify"); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index e7156dfe9aa..a0c52e4b80c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -372,7 +372,7 @@ File retrieveProprietaryDataverseMetadata (HttpClient client, String remoteApiUr HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(remoteApiUrl)) .GET() - .header("User-Agent", "DataverseHarvester/6.0") + .header("User-Agent", "Dataverse Harvesting Client v5") .build(); HttpResponse response; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 448faa20b0b..d9b4d502f59 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -4,6 +4,7 @@ import java.util.logging.Level; import com.jayway.restassured.RestAssured; import static com.jayway.restassured.RestAssured.given; +import com.jayway.restassured.path.json.JsonPath; import org.junit.Test; import com.jayway.restassured.response.Response; import static javax.ws.rs.core.Response.Status.CREATED; @@ -28,14 +29,14 @@ public class HarvestingClientsIT { private static final Logger logger = Logger.getLogger(HarvestingClientsIT.class.getCanonicalName()); - private static final String harvestClientsApi = "/api/harvest/clients/"; - private static final String rootCollection = "root"; - private static final String harvestUrl = "https://demo.dataverse.org/oai"; - private static final String archiveUrl = "https://demo.dataverse.org"; - private static final String harvestMetadataFormat = "oai_dc"; - private static final String archiveDescription = "RestAssured harvesting client test"; - private static final String controlOaiSet = "controlTestSet"; - private static final int datasetsInControlSet = 7; + private static final String HARVEST_CLIENTS_API = "/api/harvest/clients/"; + private static final String ROOT_COLLECTION = "root"; + private static final String HARVEST_URL = "https://demo.dataverse.org/oai"; + private static final String ARCHIVE_URL = "https://demo.dataverse.org"; + private static final String HARVEST_METADATA_FORMAT = "oai_dc"; + private static final String ARCHIVE_DESCRIPTION = "RestAssured harvesting client test"; + private static final String CONTROL_OAI_SET = "controlTestSet"; + private static final int DATASETS_IN_CONTROL_SET = 7; private static String normalUserAPIKey; private static String adminUserAPIKey; private static String harvestCollectionAlias; @@ -81,13 +82,13 @@ public void testCreateEditDeleteClient() { String nickName = UtilIT.getRandomString(6); - String clientApiPath = String.format(harvestClientsApi+"%s", nickName); + String clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName); String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + "\"type\":\"oai\"," + "\"harvestUrl\":\"%s\"," + "\"archiveUrl\":\"%s\"," + "\"metadataFormat\":\"%s\"}", - rootCollection, harvestUrl, archiveUrl, harvestMetadataFormat); + ROOT_COLLECTION, HARVEST_URL, ARCHIVE_URL, HARVEST_METADATA_FORMAT); // Try to create a client as normal user, should fail: @@ -109,7 +110,7 @@ public void testCreateEditDeleteClient() { // Try to update the client we have just created: - String updateJson = String.format("{\"archiveDescription\":\"%s\"}", archiveDescription); + String updateJson = String.format("{\"archiveDescription\":\"%s\"}", ARCHIVE_DESCRIPTION); Response rUpdate = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) @@ -132,11 +133,11 @@ public void testCreateEditDeleteClient() { .body("status", equalTo(AbstractApiBean.STATUS_OK)) .body("data.type", equalTo("oai")) .body("data.nickName", equalTo(nickName)) - .body("data.archiveDescription", equalTo(archiveDescription)) - .body("data.dataverseAlias", equalTo(rootCollection)) - .body("data.harvestUrl", equalTo(harvestUrl)) - .body("data.archiveUrl", equalTo(archiveUrl)) - .body("data.metadataFormat", equalTo(harvestMetadataFormat)); + .body("data.archiveDescription", equalTo(ARCHIVE_DESCRIPTION)) + .body("data.dataverseAlias", equalTo(ROOT_COLLECTION)) + .body("data.harvestUrl", equalTo(HARVEST_URL)) + .body("data.archiveUrl", equalTo(ARCHIVE_URL)) + .body("data.metadataFormat", equalTo(HARVEST_METADATA_FORMAT)); // Try to delete the client as normal user should fail: @@ -167,14 +168,14 @@ public void testHarvestingClientRun() throws InterruptedException { String nickName = UtilIT.getRandomString(6); - String clientApiPath = String.format(harvestClientsApi+"%s", nickName); + String clientApiPath = String.format(HARVEST_CLIENTS_API+"%s", nickName); String clientJson = String.format("{\"dataverseAlias\":\"%s\"," + "\"type\":\"oai\"," + "\"harvestUrl\":\"%s\"," + "\"archiveUrl\":\"%s\"," + "\"set\":\"%s\"," + "\"metadataFormat\":\"%s\"}", - harvestCollectionAlias, harvestUrl, archiveUrl, controlOaiSet, harvestMetadataFormat); + harvestCollectionAlias, HARVEST_URL, ARCHIVE_URL, CONTROL_OAI_SET, HARVEST_METADATA_FORMAT); Response createResponse = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, adminUserAPIKey) @@ -185,7 +186,7 @@ public void testHarvestingClientRun() throws InterruptedException { // API TEST 1. Run the harvest using the configuration (client) we have // just created - String runHarvestApiPath = String.format(harvestClientsApi+"%s/run", nickName); + String runHarvestApiPath = String.format(HARVEST_CLIENTS_API+"%s/run", nickName); // TODO? - verify that a non-admin user cannot perform this operation (401) @@ -207,35 +208,36 @@ public void testHarvestingClientRun() throws InterruptedException { .get(clientApiPath); assertEquals(OK.getStatusCode(), getClientResponse.getStatusCode()); - assertEquals(AbstractApiBean.STATUS_OK, getClientResponse.body().jsonPath().getString("status")); + JsonPath responseJsonPath = getClientResponse.body().jsonPath(); + assertNotNull("Invalid JSON in GET client response", responseJsonPath); + assertEquals(AbstractApiBean.STATUS_OK, responseJsonPath.getString("status")); - if (logger.isLoggable(Level.FINE)) { - logger.info("listIdentifiersResponse.prettyPrint: " - + getClientResponse.prettyPrint()); - } - - String clientStatus = getClientResponse.body().jsonPath().getString("data.status"); + String clientStatus = responseJsonPath.getString("data.status"); assertNotNull(clientStatus); - if ("inProgress".equals(clientStatus)) { + if ("inProgress".equals(clientStatus) || "IN PROGRESS".equals(responseJsonPath.getString("data.lastResult"))) { // we'll sleep for another second i++; } else { + if (logger.isLoggable(Level.FINE)) { + logger.info("getClientResponse.prettyPrint: " + + getClientResponse.prettyPrint()); + } // Check the values in the response: // a) Confirm that the harvest has completed: assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); // b) Confirm that it has actually succeeded: - assertEquals("Last harvest not reported a success", "SUCCESS", getClientResponse.body().jsonPath().getString("data.lastResult")); - String harvestTimeStamp = getClientResponse.body().jsonPath().getString("data.lastHarvest"); + assertEquals("Last harvest not reported a success", "SUCCESS", responseJsonPath.getString("data.lastResult")); + String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest"); assertNotNull(harvestTimeStamp); // c) Confirm that the other timestamps match: - assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastSuccessful")); - assertEquals(harvestTimeStamp, getClientResponse.body().jsonPath().getString("data.lastNonEmpty")); + assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastSuccessful")); + assertEquals(harvestTimeStamp, responseJsonPath.getString("data.lastNonEmpty")); // d) Confirm that the correct number of datasets have been harvested: - assertEquals(datasetsInControlSet, getClientResponse.body().jsonPath().getInt("data.lastDatasetsHarvested")); + assertEquals(DATASETS_IN_CONTROL_SET, responseJsonPath.getInt("data.lastDatasetsHarvested")); // ok, it looks like the harvest has completed successfully. break; diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index dad32bcaa60..d10e0c4c6d7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -18,7 +18,6 @@ import java.util.List; import java.util.Set; import java.util.HashSet; -//import static junit.framework.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -141,9 +140,12 @@ private XmlPath validateOaiVerbResponse(Response oaiResponse, String verb) { assertNotNull(responseXmlPath); String dateString = responseXmlPath.getString("OAI-PMH.responseDate"); - assertNotNull(dateString); // TODO: validate that it's well-formatted! - logger.info("date string from the OAI output:"+dateString); - assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.request")); + assertNotNull(dateString); + // TODO: validate the formatting of the date string in the record + // header, above. (could be slightly tricky - since this formatting + // is likely locale-specific) + logger.fine("date string from the OAI output:"+dateString); + //assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.request")); assertEquals(verb, responseXmlPath.getString("OAI-PMH.request.@verb")); return responseXmlPath; } @@ -153,12 +155,11 @@ public void testOaiIdentify() { // Run Identify: Response identifyResponse = UtilIT.getOaiIdentify(); assertEquals(OK.getStatusCode(), identifyResponse.getStatusCode()); - //logger.info("Identify response: "+identifyResponse.prettyPrint()); // Validate the response: XmlPath responseXmlPath = validateOaiVerbResponse(identifyResponse, "Identify"); - assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.Identify.baseURL")); + //assertEquals("http://localhost:8080/oai", responseXmlPath.getString("OAI-PMH.Identify.baseURL")); // Confirm that the server is reporting the correct parameters that // our server implementation should be using: assertEquals("2.0", responseXmlPath.getString("OAI-PMH.Identify.protocolVersion")); @@ -171,7 +172,6 @@ public void testOaiListMetadataFormats() { // Run ListMeatadataFormats: Response listFormatsResponse = UtilIT.getOaiListMetadataFormats(); assertEquals(OK.getStatusCode(), listFormatsResponse.getStatusCode()); - //logger.info("ListMetadataFormats response: "+listFormatsResponse.prettyPrint()); // Validate the response: @@ -253,7 +253,7 @@ public void testNativeSetAPI() { System.out.println("responseAll full: " + responseAll.prettyPrint()); assertEquals(200, responseAll.getStatusCode()); assertTrue(responseAll.body().jsonPath().getList("data.oaisets").size() > 0); - assertTrue(responseAll.body().jsonPath().getList("data.oaisets.name").toString().contains(setName)); // todo: simplify + assertTrue(responseAll.body().jsonPath().getList("data.oaisets.name", String.class).contains(setName)); // API Test 6. Try to create a set with the same name, should fail createSetResponse = given() @@ -369,22 +369,14 @@ public void testSetEditAPIandOAIlistSets() { // we created and modified, above, is being listed by the OAI server // and its xml record is properly formatted - List listSets = responseXmlPath.getList("OAI-PMH.ListSets.set.list()"); // TODO - maybe try it with findAll()? - assertNotNull(listSets); - assertTrue(listSets.size() > 0); - - Node foundSetNode = null; - for (Node setNode : listSets) { - - if (setName.equals(setNode.get("setName").toString())) { - foundSetNode = setNode; - break; - } - } + List listSets = responseXmlPath.getList("OAI-PMH.ListSets.set.list().findAll{it.setName=='"+setName+"'}", Node.class); + + // 2a. Confirm that our set is listed: + assertNotNull("Unexpected response from ListSets", listSets); + assertTrue("Newly-created set isn't properly listed by the OAI server", listSets.size() == 1); + // 2b. Confirm that the set entry contains the updated description: + assertEquals("Incorrect description in the ListSets entry", newDescription, listSets.get(0).getPath("setDescription.metadata.element.field", String.class)); - assertNotNull("Newly-created set is not listed by the OAI server", foundSetNode); - assertEquals("Incorrect description in the ListSets entry", newDescription, foundSetNode.getPath("setDescription.metadata.element.field", String.class)); - // ok, the xml record looks good! // Cleanup. Delete the set with the DELETE API @@ -416,26 +408,30 @@ public void testSingleRecordOaiSet() throws InterruptedException { // The GET method of the oai set API, as well as the OAI ListSets // method are tested extensively in another method in this class, so - // we'll skip checking those here. + // we'll skip looking too closely into those here. - // Let's export the set. This is asynchronous - so we will try to - // wait a little - but in practice, everything potentially time-consuming - // must have been done when the dataset was exported, in the setup method. + // A quick test that the new set is listed under native API + Response getSet = given() + .get(apiPath); + assertEquals(200, getSet.getStatusCode()); + + // Export the set. Response exportSetResponse = UtilIT.exportOaiSet(setName); assertEquals(200, exportSetResponse.getStatusCode()); - Thread.sleep(1000L); - - Response getSet = given() - .get(apiPath); + + // Strictly speaking, exporting an OAI set is an asynchronous operation. + // So the code below was written to expect to have to wait for up to 10 + // additional seconds for it to complete. In retrospect, this is + // most likely unnecessary (because the only potentially expensive part + // of the process is the metadata export, and in this case that must have + // already happened - when the dataset was published (that operation + // now has its own wait mechanism). But I'll keep this extra code in + // place since it's not going to hurt. - L.A. - logger.info("getSet.getStatusCode(): " + getSet.getStatusCode()); - logger.fine("getSet printresponse: " + getSet.prettyPrint()); - assertEquals(200, getSet.getStatusCode()); int i = 0; int maxWait=10; do { - // OAI Test 1. Run ListIdentifiers on this newly-created set: Response listIdentifiersResponse = UtilIT.getOaiListIdentifiers(setName, "oai_dc"); @@ -445,17 +441,14 @@ public void testSingleRecordOaiSet() throws InterruptedException { XmlPath responseXmlPath = validateOaiVerbResponse(listIdentifiersResponse, "ListIdentifiers"); List ret = responseXmlPath.getList("OAI-PMH.ListIdentifiers.header"); - assertNotNull(ret); - if (logger.isLoggable(Level.FINE)) { - logger.info("listIdentifiersResponse.prettyPrint: " - + listIdentifiersResponse.prettyPrint()); - } - if (ret.isEmpty()) { - // OK, we'll sleep for another second - provided it's been less - // than 10 sec. total. + if (ret == null || ret.isEmpty()) { + // OK, we'll sleep for another second i++; } else { + if (logger.isLoggable(Level.FINE)) { + logger.info("listIdentifiersResponse.prettyPrint: " + listIdentifiersResponse.prettyPrint()); + } // Validate the payload of the ListIdentifiers response: // a) There should be 1 and only 1 item listed: assertEquals(1, ret.size()); @@ -465,20 +458,13 @@ public void testSingleRecordOaiSet() throws InterruptedException { assertEquals(setName, responseXmlPath .getString("OAI-PMH.ListIdentifiers.header.setSpec")); assertNotNull(responseXmlPath.getString("OAI-PMH.ListIdentifiers.header.dateStamp")); - // TODO: validate the formatting of the date string in the record - // header, above! + // TODO: validate the formatting of the date string here as well. // ok, ListIdentifiers response looks valid. break; } Thread.sleep(1000L); } while (i Date: Tue, 13 Dec 2022 19:43:44 -0500 Subject: [PATCH 231/232] one more jenkins run, with a bit more logging (#8843) --- .../edu/harvard/iq/dataverse/api/HarvestingClientsIT.java | 6 ++---- .../edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index d9b4d502f59..3fc72125145 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -219,10 +219,8 @@ public void testHarvestingClientRun() throws InterruptedException { // we'll sleep for another second i++; } else { - if (logger.isLoggable(Level.FINE)) { - logger.info("getClientResponse.prettyPrint: " - + getClientResponse.prettyPrint()); - } + logger.info("getClientResponse.prettyPrint: " + + getClientResponse.prettyPrint()); // Check the values in the response: // a) Confirm that the harvest has completed: assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java index d10e0c4c6d7..ccc0629bb69 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingServerIT.java @@ -429,6 +429,7 @@ public void testSingleRecordOaiSet() throws InterruptedException { // now has its own wait mechanism). But I'll keep this extra code in // place since it's not going to hurt. - L.A. + Thread.sleep(1000L); // initial sleep interval int i = 0; int maxWait=10; do { From f6d08bb2fcef119b39a3e3a193dc5826026ea7a9 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Wed, 14 Dec 2022 10:55:12 -0500 Subject: [PATCH 232/232] trigger another Jenkins run, with the time delay slightly rearranged in the wait for an async. operation + some extra logging (#8843) --- .../edu/harvard/iq/dataverse/api/HarvestingClientsIT.java | 8 ++++++-- .../edu/harvard/iq/dataverse/api/HarvestingServerIT.java | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 3fc72125145..094eb0df77c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -203,6 +203,11 @@ public void testHarvestingClientRun() throws InterruptedException { int i = 0; int maxWait=20; // a very conservative interval; this harvest has no business taking this long do { + // Give it an initial 1 sec. delay, to make sure the client state + // has been updated in the database, which can take some appreciable + // amount of time on a heavily-loaded server running a full suite of + // tests: + Thread.sleep(1000L); // keep checking the status of the client with the GET api: Response getClientResponse = given() .get(clientApiPath); @@ -226,7 +231,7 @@ public void testHarvestingClientRun() throws InterruptedException { assertEquals("Unexpected client status: "+clientStatus, "inActive", clientStatus); // b) Confirm that it has actually succeeded: - assertEquals("Last harvest not reported a success", "SUCCESS", responseJsonPath.getString("data.lastResult")); + assertEquals("Last harvest not reported a success (took "+i+" seconds)", "SUCCESS", responseJsonPath.getString("data.lastResult")); String harvestTimeStamp = responseJsonPath.getString("data.lastHarvest"); assertNotNull(harvestTimeStamp); @@ -240,7 +245,6 @@ public void testHarvestingClientRun() throws InterruptedException { // ok, it looks like the harvest has completed successfully. break; } - Thread.sleep(1000L); } while (i