Merge remote-tracking branch 'IQSS/develop' into

IQSS/10137-2-add_flag_to_make_reason_optional

doc/release-notes/10494-payara-upgrade.md

@@ -0,0 +1,119 @@
+# Upgrade Payara to v6.2024.6
+
+With this version of Dataverse, we encourage you to upgrade to version 6.2024.6.
+This will address security issues accumulated since the release of 6.2023.8, which was required since Dataverse release 6.0.
+
+## Instructions for Upgrading
+
+If you are using GDCC containers, this upgrade is included when pulling new release images.
+No manual intervention is necessary.
+
+We recommend you ensure you followed all update instructions from the past releases regarding Payara.
+(Latest Payara update was for [v6.0](https://github.com/IQSS/dataverse/releases/tag/v6.0))
+
+Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc.
+
+The steps below are a simple matter of reusing your existing domain directory with the new distribution.
+But we also recommend that you review the Payara upgrade instructions as it could be helpful during any troubleshooting:
+[Payara Release Notes](https://docs.payara.fish/community/docs/Release%20Notes/Release%20Notes%206.2024.6.html)
+We assume you are already on a Dataverse 6.x installation, using a Payara 6.x release.
+
+```shell
+export PAYARA=/usr/local/payara6
+```
+
+(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell)
+
+1\. Undeploy the previous version
+
+```shell
+    $PAYARA/bin/asadmin list-applications
+    $PAYARA/bin/asadmin undeploy dataverse<-version>
+```
+
+2\. Stop Payara
+
+```shell
+    service payara stop
+    rm -rf $PAYARA/glassfish/domains/domain1/generated
+    rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache
+    rm -rf $PAYARA/glassfish/domains/domain1/lib/databases
+```
+
+3\. Move the current Payara directory out of the way
+
+```shell
+    mv $PAYARA $PAYARA.MOVED
+```
+
+4\. Download the new Payara version (6.2024.6), and unzip it in its place
+
+5\. Replace the brand new payara/glassfish/domains/domain1 with your old, preserved domain1
+
+6\. Make sure that you have the following `--add-opens` options in your domain.xml. If not present, add them:
+
+```diff
+--- payara-6.2023.8/glassfish/domains/domain1/config/domain.xml
++++ payara-6.2024.6/glassfish/domains/domain1/config/domain.xml
+@@ -212,12 +212,16 @@
+                 <jvm-options>--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>
+                 <jvm-options>--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>
+                 <jvm-options>--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>
++                <jvm-options>--add-opens=java.management/javax.management=ALL-UNNAMED</jvm-options>
++                <jvm-options>--add-opens=java.management/javax.management.openmbean=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED</jvm-options>
++                <jvm-options>[17|]--add-opens=java.base/java.io=ALL-UNNAMED</jvm-options>
++                <jvm-options>[21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED</jvm-options>
+                 <jvm-options>-Xmx512m</jvm-options>
+                 <jvm-options>-XX:NewRatio=2</jvm-options>
+                 <jvm-options>-XX:+UnlockDiagnosticVMOptions</jvm-options>
+@@ -447,12 +451,16 @@
+                 <jvm-options>--add-opens=java.naming/javax.naming.spi=ALL-UNNAMED</jvm-options>
+                 <jvm-options>--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED</jvm-options>
+                 <jvm-options>--add-opens=java.logging/java.util.logging=ALL-UNNAMED</jvm-options>
++                <jvm-options>--add-opens=java.management/javax.management=ALL-UNNAMED</jvm-options>
++                <jvm-options>--add-opens=java.management/javax.management.openmbean=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=java.base/sun.net.www=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=java.base/sun.security.util=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-opens=java.base/java.lang.invoke=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-opens=java.desktop/java.beans=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED</jvm-options>
+                 <jvm-options>[17|]--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED</jvm-options>
++                <jvm-options>[17|]--add-opens=java.base/java.io=ALL-UNNAMED</jvm-options>
++                <jvm-options>[21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED</jvm-options>
+                 <jvm-options>-Xmx512m</jvm-options>
+                 <jvm-options>-XX:NewRatio=2</jvm-options>
+                 <jvm-options>-XX:+UnlockDiagnosticVMOptions</jvm-options>
+```
+(You can also save this as a patch file and try to apply it.)
+
+TODO: For the combined 6.3 release note, I would consider replacing the patch format above with just the 4 specific options, for clarity etc. (L.A.) As in: 
+```
+         <jvm-options>--add-opens=java.management/javax.management=ALL-UNNAMED</jvm-options>
+         <jvm-options>--add-opens=java.management/javax.management.openmbean=ALL-UNNAMED</jvm-options>
+         <jvm-options>[17|]--add-opens=java.base/java.io=ALL-UNNAMED</jvm-options>
+         <jvm-options>[21|]--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED</jvm-options>
+```
+
+7\. Start Payara
+
+```shell
+    service payara start
+```
+
+8\. Deploy this version.
+
+```shell
+    $PAYARA/bin/asadmin deploy dataverse-6.3.war
+```
+
+9\. Restart payara
+
+```shell
+    service payara stop
+    service payara start

doc/release-notes/10611-harvested-origin-facet.md

@@ -0,0 +1,10 @@
+NOTE that this release note supercedes the 10464-add-name-harvesting-client-facet.md note from the PR 10464.
+
+An option has been added to index the name of the Harvesting Client as the "Metadata Source" of harvested datasets and files; if enabled, the Metadata Source facet will be showing separate entries for the content harvested from different sources, instead of the current, default behavior where there is one "Harvested" facet for all such content.
+
+
+TODO: for the v6.3 release note:
+If you choose to enable the extended "Metadata Souce" facet for harvested content, set the optional feature flage (jvm option) `dataverse.feature.index-harvested-metadata-source=true` before reindexing.
+
+[Please note that the upgrade instruction in 6.3 will contain a suggestion to run full reindex, as part of the Solr upgrade, so the sentence above will need to be added to that section]
+

doc/sphinx-guides/source/admin/monitoring.rst

@@ -154,7 +154,7 @@ Tips:
 MicroProfile Metrics endpoint
 -----------------------------
 
-Payara provides the metrics endpoint: <https://docs.payara.fish/community/docs/6.2023.9/Technical%20Documentation/MicroProfile/Metrics/Metrics%20Rest%20Endpoint.html>_
+Payara provides the metrics endpoint: <https://docs.payara.fish/community/docs/Technical%20Documentation/MicroProfile/Metrics/Metrics%20Rest%20Endpoint.html>_
 The metrics you can retrieve that way:
 - `index_permit_wait_time_seconds_mean` displays how long does it take to receive a permit to index a dataset.
 - `index_time_seconds` displays how long does it take to index a dataset.

doc/sphinx-guides/source/developers/classic-dev-env.rst

@@ -93,15 +93,15 @@ On Linux, install ``jq`` from your package manager or download a binary from htt
 Install Payara
 ~~~~~~~~~~~~~~
 
-Payara 6.2023.8 or higher is required.
+Payara 6.2024.6 or higher is required.
 
 To install Payara, run the following commands:
 
 ``cd /usr/local``
 
-``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip``
+``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip``
 
-``sudo unzip payara-6.2023.8.zip``
+``sudo unzip payara-6.2024.6.zip``
 
 ``sudo chown -R $USER /usr/local/payara6``
 

doc/sphinx-guides/source/installation/config.rst

@@ -3277,10 +3277,14 @@ please find all known feature flags below. Any of these flags can be activated u
     * - reduce-solr-deletes
       - Avoids deleting and recreating solr documents for dataset files when reindexing. 
       - ``Off``
+    * - reduce-solr-deletes
+      - Avoids deleting and recreating solr documents for dataset files when reindexing. 
+      - ``Off``
     * - disable-return-to-author-reason
       - Removes the reason field in the `Publish/Return To Author` dialog that was added as a required field in v6.2 and makes the reason an optional parameter in the :ref:`return-a-dataset` API call. 
       - ``Off``
 
+
 **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable
 ``DATAVERSE_FEATURE_XXX`` (e.g. ``DATAVERSE_FEATURE_API_SESSION_AUTH=1``). These environment variables can be set in your shell before starting Payara. If you are using :doc:`Docker for development </container/dev-usage>`, you can set them in the `docker compose <https://docs.docker.com/compose/environment-variables/set-environment-variables/>`_ file.
 

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 17 the default with the ``alternatives`` co
 Payara
 ------
 
-Payara 6.2023.8 is recommended. Newer versions might work fine. Regular updates are recommended.
+Payara 6.2024.6 is recommended. Newer versions might work fine. Regular updates are recommended.
 
 Installing Payara
 =================
@@ -55,8 +55,8 @@ Installing Payara
 
 - Download and install Payara (installed in ``/usr/local/payara6`` in the example commands below)::
 
-	# wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip
-	# unzip payara-6.2023.8.zip
+	# wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip
+	# unzip payara-6.2024.6.zip
 	# mv payara6 /usr/local
 
 If nexus.payara.fish is ever down for maintenance, Payara distributions are also available from https://repo1.maven.org/maven2/fish/payara/distributions/payara/

doc/sphinx-guides/source/qa/test-automation.md

@@ -52,7 +52,7 @@ Go to the end of the log and then scroll up, looking for the failure. A failed A
 
 ```
 TASK [dataverse : download payara zip] *****************************************
-fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: <urlopen error timed out>", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip"}
+fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: <urlopen error timed out>", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip"}
 ```
 
 In the example above, if Payara can't be downloaded, we're obviously going to have problems deploying Dataverse to it!

modules/container-base/src/main/docker/Dockerfile

@@ -166,8 +166,6 @@ RUN <<EOF
     ${ASADMIN} create-jvm-options '-XX\:MetaspaceSize=${ENV=MEM_METASPACE_SIZE}'
     ${ASADMIN} create-jvm-options '-XX\:MaxMetaspaceSize=${ENV=MEM_MAX_METASPACE_SIZE}'
     ${ASADMIN} create-jvm-options '-XX\:+IgnoreUnrecognizedVMOptions'
-    # Workaround for FISH-7722: Failed to deploy war with @Stateless https://github.com/payara/Payara/issues/6337
-    ${ASADMIN} create-jvm-options --add-opens=java.base/java.io=ALL-UNNAMED
     # Disable autodeploy and hot reload
     ${ASADMIN} set configs.config.server-config.admin-service.das-config.dynamic-reload-enabled="false"
     ${ASADMIN} set configs.config.server-config.admin-service.das-config.autodeploy-enabled="false"

modules/dataverse-parent/pom.xml

@@ -148,14 +148,13 @@
         <argLine>-Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} -Duser.language=${project.language} -Duser.region=${project.region}</argLine>
 
         <!-- Major system components and dependencies -->
-        <payara.version>6.2023.8</payara.version>
+        <payara.version>6.2024.6</payara.version>
         <postgresql.version>42.7.2</postgresql.version>
         <solr.version>9.4.1</solr.version>
         <aws.version>1.12.290</aws.version>
         <google.library.version>26.30.0</google.library.version>
 
         <!-- Basic libs, logging -->
-        <jakartaee-api.version>8.0.0</jakartaee-api.version>
         <slf4j.version>1.7.35</slf4j.version>
         <commons.io.version>2.15.1</commons.io.version>
         <commons.logging.version>1.2</commons.logging.version>
@@ -386,18 +385,6 @@
                 <enabled>false</enabled>
             </snapshots>
         </repository>
-        <!-- This repository is about to being phased out. See https://github.com/payara/Payara_PatchedProjects/issues/366 -->
-        <repository>
-            <id>payara-patched-externals</id>
-            <name>Payara Patched Externals</name>
-            <url>https://raw.github.com/payara/Payara_PatchedProjects/master</url>
-            <releases>
-                <enabled>true</enabled>
-            </releases>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </repository>
         <repository>
             <id>central-repo</id>
             <name>Central Repository</name>

pom.xml

@@ -224,9 +224,6 @@
             <groupId>fish.payara.api</groupId>
             <artifactId>payara-api</artifactId>
             <scope>provided</scope>
-            <!-- Note: The version was provided by Payara BOM till 6.2023.7, when they removed the Core BOM from it as
-                       meant for internal use only. Simply referencing the version property here solves the problem. -->
-            <version>${payara.version}</version>
         </dependency>
 
         <!-- JSON-P -->

scripts/installer/as-setup.sh

@@ -111,15 +111,13 @@ function preliminary_setup()
   ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.label=Fake DOI Provider"
   ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.authority=10.5072"
   ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.fake.shoulder=FK2/"
+  ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.default-provider=fake"
   # jvm-options use colons as separators, escape as literal
   #DOI_DATACITERESTAPIURL_ESC=`echo $DOI_DATACITERESTAPIURL | sed -e 's/:/\\\:/'`
   #./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.pid.testDC.datacite.rest-api-url=$DOI_DATACITERESTAPIURL_ESC"
 
   ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true"
 
-  # Workaround for FISH-7722: Failed to deploy war with @Stateless https://github.com/payara/Payara/issues/6337
-  ./asadmin $ASADMIN_OPTS create-jvm-options --add-opens=java.base/java.io=ALL-UNNAMED
-
   # enable comet support
   ./asadmin $ASADMIN_OPTS set server-config.network-config.protocols.protocol.http-listener-1.http.comet-support-enabled="true"
 

src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java

@@ -573,7 +573,7 @@ public boolean requestAccess(DataFile dataFile, GuestbookResponse gbr){
 
     public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, AuthenticatedUser requestor) {
         Timestamp ts = new Timestamp(new Date().getTime());
-        permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset).stream().forEach((au) -> {
+        permissionService.getUsersWithPermissionOn(Permission.ManageFilePermissions, dataset).stream().forEach((au) -> {
             userNotificationService.sendNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, true);
         });
         //send the user that requested access a notification that they requested the access

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -221,7 +221,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id"
     // WORKS on published datasets, which are open to the world. -- L.A. 4.5
     @GET
     @Path("/export")
-    @Produces({"application/xml", "application/json", "application/html", "application/ld+json" })
+    @Produces({"application/xml", "application/json", "application/html", "application/ld+json", "*/*" })
     public Response exportDataset(@QueryParam("persistentId") String persistentId, @QueryParam("exporter") String exporter, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) {
 
         try {

src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java

@@ -987,8 +987,14 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
 
         if (dataset.isHarvested()) {
             solrInputDocument.addField(SearchFields.IS_HARVESTED, true);
-            solrInputDocument.addField(SearchFields.METADATA_SOURCE,
+            if (FeatureFlags.INDEX_HARVESTED_METADATA_SOURCE.enabled()) {
+                // New - as of 6.3 - option of indexing the actual origin of 
+                // harvested objects as the metadata source:
+                solrInputDocument.addField(SearchFields.METADATA_SOURCE,
                                         dataset.getHarvestedFrom() != null ? dataset.getHarvestedFrom().getName() : HARVESTED);
+            } else {
+                solrInputDocument.addField(SearchFields.METADATA_SOURCE, HARVESTED);
+            }
         } else {
             solrInputDocument.addField(SearchFields.IS_HARVESTED, false);
             solrInputDocument.addField(SearchFields.METADATA_SOURCE, rdvName); //rootDataverseName);
@@ -1495,7 +1501,14 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                         }
                         if (datafile.isHarvested()) {
                             datafileSolrInputDocument.addField(SearchFields.IS_HARVESTED, true);
-                            datafileSolrInputDocument.addField(SearchFields.METADATA_SOURCE, HARVESTED);
+                            if (FeatureFlags.INDEX_HARVESTED_METADATA_SOURCE.enabled()) {
+                                // New - as of 6.3 - option of indexing the actual origin of 
+                                // harvested objects as the metadata source:
+                                datafileSolrInputDocument.addField(SearchFields.METADATA_SOURCE,
+                                        dataset.getHarvestedFrom() != null ? dataset.getHarvestedFrom().getName() : HARVESTED);
+                            } else {
+                                datafileSolrInputDocument.addField(SearchFields.METADATA_SOURCE, HARVESTED);
+                            }
                         } else {
                             datafileSolrInputDocument.addField(SearchFields.IS_HARVESTED, false);
                             datafileSolrInputDocument.addField(SearchFields.METADATA_SOURCE, rdvName);

src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java

@@ -577,8 +577,7 @@ public SolrQueryResponse search(
             solrSearchResult.setDvTree(dvTree);
             solrSearchResult.setDatasetValid(datasetValid);
 
-            String originSource = (String) solrDocument.getFieldValue(SearchFields.METADATA_SOURCE);
-            if (IndexServiceBean.HARVESTED.equals(originSource)) {
+            if (Boolean.TRUE.equals((Boolean) solrDocument.getFieldValue(SearchFields.IS_HARVESTED))) {
                 solrSearchResult.setHarvested(true);
             }
 
@@ -1110,7 +1109,7 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
             }
 
             String ret = sb.toString();
-            logger.info("Returning experimental query: " + ret);
+            logger.fine("Returning experimental query: " + ret);
             return ret;
         }