Merge branch 'develop' into 7000-mpconfig-version

conf/solr/8.11.1/schema.xml

@@ -228,6 +228,11 @@
 
     <field name="dsPersistentId" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="filePersistentId" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <!-- Dataverse geospatial search -->
+    <!-- https://solr.apache.org/guide/8_11/spatial-search.html#rpt -->
+    <field name="geolocation" type="location_rpt" multiValued="true" stored="true" indexed="true"/>
+    <!-- https://solr.apache.org/guide/8_11/spatial-search.html#bboxfield -->
+    <field name="boundingBox" type="bbox" multiValued="true" stored="true" indexed="true"/>
 
     <!--
         METADATA SCHEMA FIELDS
@@ -1104,6 +1109,9 @@
     -->
     <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
                geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
+    <!-- Dataverse - per GeoBlacklight, adding field type for bboxField that enables, among other things, overlap ratio calculations -->
+    <fieldType name="bbox" class="solr.BBoxField"
+           geo="true" distanceUnits="kilometers" numberType="pdouble" />
 
     <!-- Payloaded field types -->
     <fieldType name="delimited_payloads_float" stored="false" indexed="true" class="solr.TextField">

doc/JAVADOC_GUIDE.md

@@ -88,7 +88,7 @@ Here's a better approach:
     /** The dataverse we move the dataset <em>from</em> */
     private Dataverse sourceDataverse;
 
-    /** The dataverse we movet the dataset <em>to</em> */
+    /** The dataverse we move the dataset <em>to</em> */
     private Dataverse destinationDataverse;
 
 

doc/mergeParty/readme.md

@@ -73,10 +73,10 @@ Note that before we were asking `isGuest` and now we ask `isAuthenticated`, so t
 ## Other Added Things
 
 ### Settings bean
-Settings (in `edu.harvard.iq.dataverse.settings`) are where the application stores its more complex, admin-editable configuration. Technically, its a persistent `Map<String,String>`, that can be accessed via API (`edu.harvard.iq.dataverse.api.Admin`, on path `{server}/api/s/settings`). Currenly used for the signup mechanism.
+Settings (in `edu.harvard.iq.dataverse.settings`) are where the application stores its more complex, admin-editable configuration. Technically, its a persistent `Map<String,String>`, that can be accessed via API (`edu.harvard.iq.dataverse.api.Admin`, on path `{server}/api/s/settings`). Currently used for the signup mechanism.
 
 ### Admin API
-Accessible under url `{server}/api/s/`, API calls to this bean should be editing confugurations, allowing full indexing and more. The idea behing putting all of them under the `/s/` path is that we can later block these calls using a filter. This way, we could, say, allow access from localhost only. Or, we could block this completely based on some environemnt variable.
+Accessible under url `{server}/api/s/`, API calls to this bean should be editing configurations, allowing full indexing and more. The idea behind putting all of them under the `/s/` path is that we can later block these calls using a filter. This way, we could, say, allow access from localhost only. Or, we could block this completely based on some environment variable.
 
 ### `setup-all.sh` script
 A new script that sets up the users and the dataverses, sets the system up for built-in signup, and then indexes the dataverses using solr. Requires the [jq utility](http://stedolan.github.io/jq/). On Macs with [homebrew](http://brew.sh) installed, getting this utility is a `brew install jq` command away.

doc/release-notes/5.12.1-release-notes.md

@@ -0,0 +1,115 @@
+# Dataverse Software 5.12.1
+
+This release brings new features, enhancements, and bug fixes to the Dataverse Software. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.
+
+## Release Highlights
+
+### Bug Fix for "Internal Server Error" When Creating a New Remote Account
+
+Unfortunately, as of 5.11 new remote users have seen "Internal Server Error" when creating an account (or checking notifications just after creating an account). Remote users are those who log in with institutional (Shibboleth), OAuth (ORCID, GitHub, or Google) or OIDC providers.
+
+This is a transient error that can be worked around by reloading the browser (or logging out and back in again) but it's obviously a very poor user experience and a bad first impression. This bug is the primary reason we are putting out this patch release. Other features and bug fixes are coming along for the ride.
+
+### Ability to Disable OAuth Sign Up While Allowing Existing Accounts to Log In
+
+A new option called `:AllowRemoteAuthSignUp` has been added providing a mechanism for disabling new account signups for specific OAuth2 authentication providers (Orcid, GitHub, Google etc.) while still allowing logins for already-existing accounts using this authentication method.
+
+See the [Installation Guide](https://guides.dataverse.org/en/5.12.1/installation/config.html#allowremoteauthsignup) for more information on the setting. 
+
+### Production Date Now Used for Harvested Datasets in Addition to Distribution Date (`oai_dc` format)
+
+Fix the year displayed in citation for harvested dataset, especially for `oai_dc` format.
+
+For normal datasets, the date used is the "citation date" which is by default the publication date (the first release date) unless you [change it](https://guides.dataverse.org/en/5.12.1/api/native-api.html#set-citation-date-field-type-for-a-dataset).
+
+However, for a harvested dataset, the distribution date was used instead and this date is not always present in the harvested metadata.
+
+Now, the production date is used for harvested dataset in addition to distribution date when harvesting with the `oai_dc` format.
+
+### Publication Date Now Used for Harvested Dataset if Production Date is Not Set (`oai_dc` format)
+
+For exports and harvesting in `oai_dc` format, if "Production Date" is not set, "Publication Date" is now used instead. This change is reflected in the [Dataverse 4+ Metadata Crosswalk][] linked from the [Appendix][] of the User Guide.
+
+[Dataverse 4+ Metadata Crosswalk]: https://docs.google.com/spreadsheets/d/10Luzti7svVTVKTA-px27oq3RxCUM-QbiTkm8iMd5C54/edit#gid=1901625433&range=K7
+[Appendix]: https://guides.dataverse.org/en/5.12.1/user/appendix.html
+
+## Major Use Cases and Infrastructure Enhancements
+
+Changes and fixes in this release include:
+
+- Users creating an account by logging in with Shibboleth, OAuth, or OIDC should not see errors. (Issue 9029, PR #9030)
+- When harvesting datasets, I want the Production Date if I can't get the Distribution Date (PR #8732)
+- When harvesting datasets, I want the Publication Date if I can't get the Production Date (PR #8733)
+- As a sysadmin I'd like to disable (temporarily or permanently) sign ups from OAuth providers while allowing existing users to continue to log in from that provider (PR #9112)
+- As a C/C++ developer I want to use Dataverse APIs (PR #9070)
+
+## New DB Settings
+
+The following DB settings have been added:
+
+- `:AllowRemoteAuthSignUp`
+
+See the [Database Settings](https://guides.dataverse.org/en/5.12.1/installation/config.html#database-settings) section of the Guides for more information.
+
+## Complete List of Changes
+
+For the complete list of code changes in this release, see the [5.12.1 Milestone](https://github.com/IQSS/dataverse/milestone/106?closed=1) in GitHub.
+
+For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.
+
+## Installation
+
+If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.12.1/installation/). Please also contact us to get added to the [Dataverse Project Map](https://guides.dataverse.org/en/5.10/installation/config.html#putting-your-dataverse-installation-on-the-map-at-dataverse-org) if you have not done so already.
+
+## Upgrade Instructions
+
+Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc.
+
+0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.12.1.
+
+If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.
+
+```shell
+export PAYARA=/usr/local/payara5
+```
+
+(or `setenv PAYARA /usr/local/payara5` if you are using a `csh`-like shell)
+
+1\. Undeploy the previous version
+
+```shell
+    $PAYARA/bin/asadmin list-applications
+    $PAYARA/bin/asadmin undeploy dataverse<-version>
+```
+
+2\. Stop Payara
+
+```shell
+    service payara stop
+    rm -rf $PAYARA/glassfish/domains/domain1/generated
+```
+
+6\. Start Payara
+
+```shell
+    service payara start
+```
+
+7\. Deploy this version.
+
+```shell
+    $PAYARA/bin/asadmin deploy dataverse-5.12.1.war
+```
+
+8\. Restart payara
+
+```shell
+    service payara stop
+    service payara start
+```
+
+## Upcoming Versions of Payara
+
+With the recent release of Payara 6 ([Payara 6.2022.1](https://github.com/payara/Payara/releases/tag/payara-server-6.2022.1) being the first version), the days of free-to-use Payara 5.x Platform Community versions [are numbered](https://blog.payara.fish/whats-new-in-the-november-2022-payara-platform-release). Specifically, Payara's blog post says, "Payara Platform Community 5.2022.4 has been released today as the penultimate Payara 5 Community release."
+
+Given the end of free-to-use Payara 5 versions, we plan to get the Dataverse software working on Payara 6 (#8305), which will require substantial efforts from the IQSS team and community members, as this also means shifting our app to be a [Jakarta EE 10](https://jakarta.ee/release/10/) application (upgrading from EE 8). We are currently working out the details and will share news as soon as we can. Rest assured we will do our best to provide you with a smooth transition. You can follow along in Issue #8305 and related pull requests and you are, of course, very welcome to participate by testing and otherwise contributing, as always.

doc/release-notes/6656-file-uploads.md

@@ -0,0 +1 @@
+new JVM option: dataverse.files.uploads

doc/release-notes/7715-signed-urls-for-external-tools.md

@@ -0,0 +1,3 @@
+# Improved Security for External Tools
+
+This release adds support for configuring external tools to use signed URLs to access the Dataverse API. This eliminates the need for tools to have access to the user's apiToken in order to access draft or restricted datasets and datafiles. Signed URLS can be transferred via POST or via a callback when triggering a tool via GET.

doc/release-notes/7940-stop-harvest-in-progress

@@ -0,0 +1,4 @@
+## Mechanism added for stopping a harvest in progress
+
+It is now possible for an admin to stop a long-running harvesting job. See [Harvesting Clients](https://guides.dataverse.org/en/latest/admin/harvestclients.html) guide for more information. 
+

doc/release-notes/8239-geospatial-indexing.md

@@ -0,0 +1,5 @@
+Support for indexing the "Geographic Bounding Box" fields ("West Longitude", "East Longitude", "North Latitude", and "South Latitude") from the Geospatial metadata block has been added.
+
+Geospatial search is supported but only via API using two new parameters: `geo_point` and `geo_radius`.
+
+A Solr schema update is required.

doc/release-notes/8671-sorting-licenses.md

@@ -0,0 +1,7 @@
+## License sorting
+
+Licenses as shown in the dropdown in UI can be now sorted by the superusers. See [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide for reference.
+
+## Backward Incompatibilities
+
+License files are now required to contain the new "sortOrder" column. When attempting to create a new license without this field, an error would be returned. See [Configuring Licenses](https://guides.dataverse.org/en/5.10/installation/config.html#configuring-licenses) section of the Installation Guide for reference.

doc/release-notes/8838-cstr.md

@@ -0,0 +1,13 @@
+### CSRT PID Types Added to Related Publication ID Type field
+
+A persistent identifier, [CSRT](https://www.cstr.cn/search/specification/), is added to the Related Publication field's ID Type child field. For datasets published with CSRT IDs, Dataverse will also include them in the datasets' Schema.org metadata exports.
+
+The CSRT
+
+### Required Upgrade Steps
+
+Update the Citation metadata block:
+
+- `wget https://github.com/IQSS/dataverse/releases/download/v#.##/citation.tsv`
+- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"`
+- Add the updated citation.properties file to the appropriate directory

doc/release-notes/8944-metadatablocks.md

@@ -0,0 +1,5 @@
+The API endpoint `/api/metadatablocks/{block_id}` has been extended to include the following fields:
+
+- `controlledVocabularyValues` - All possible values for fields with a controlled vocabulary. For example, the values "Agricultural Sciences", "Arts and Humanities", etc. for the "Subject" field.
+- `isControlledVocabulary`:  Whether or not this field has a controlled vocabulary.
+- `multiple`: Whether or not the field supports multiple values.

doc/release-notes/9117-file-type-detection.md

@@ -0,0 +1,5 @@
+NetCDF and HDF5 files are now detected based on their content rather than just their file extension.
+
+Both "classic" NetCDF 3 files and more modern NetCDF 4 files are detected based on content.
+
+Detection for HDF4 files is only done through the file extension ".hdf", as before.

doc/sphinx-guides/SphinxRSTCheatSheet.md

@@ -10,7 +10,7 @@ RST Cheat Sheet for Sphinx v 1.2.2
 | Bold text                     | **text**                                                   |                                                            |
 | Italics/emphasis              | *text*                                                     |                                                            |
 | literal                       | ``literal``                                                |                                                            |
-| Internal cross-reference link | See section 5.3.1 of Sphinx documentationand example below | See section 5.3.1 of Sphinx documentationand example below |
+| Internal cross-reference link | See section 5.3.1 of Sphinx documentation and example below | See section 5.3.1 of Sphinx documentation and example below |
 | code block                    | .. code-block:: guess<?xml version="1.0""?>                | Allows for code blocks to be displayed properly            |
 
 For more cheats please visit the [RST cheat sheet google doc] (https://docs.google.com/document/d/105H3iwPwgnPqwuMJI7q-h6FLtXV_EUCiwq2P13lADgA/edit?usp=sharing)

doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv

@@ -1,5 +1,5 @@
 Tool	Type	Scope	Description
 Data Explorer	explore	file	A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse. 
 Whole Tale	explore	dataset	A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide <https://wholetale.readthedocs.io/en/stable/users_guide/integration.html>`_.
-File Previewers	explore	file	A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, text, video, tabular data, spreadsheets, and GeoJSON - allowing them to be viewed without downloading. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers
+File Previewers	explore	file	A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, text, video, tabular data, spreadsheets, GeoJSON, and ZipFiles - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers
 Data Curation Tool	configure	file	A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions.

doc/sphinx-guides/source/_static/api/add-license.json

@@ -3,5 +3,6 @@
   "uri": "http://creativecommons.org/licenses/by/4.0",
   "shortDescription": "Creative Commons Attribution 4.0 International License.",
   "iconUrl": "https://i.creativecommons.org/l/by/4.0/88x31.png",
-  "active": true
+  "active": true,
+  "sortOrder": 2
 }

doc/sphinx-guides/source/_static/api/dataset-add-subject-metadata.json

@@ -2,7 +2,7 @@
         "typeName": "subject",
                 "value": 
                     ["Astronomy and Astrophysics", "Agricultural Sciences",
-"Arts and Humanities", "Physics"]
+"Arts and Humanities", "Physics", "Mathematical Sciences"]
 
         }
 

doc/sphinx-guides/source/_static/api/ddi_dataset.xml

@@ -88,12 +88,12 @@
         <geoBndBox>
           <westBL>10</westBL>
           <eastBL>20</eastBL>
-          <northBL>30</northBL>
-          <southBL>40</southBL>
+          <northBL>40</northBL>
+          <southBL>30</southBL>
         </geoBndBox>
         <geoBndBox>
-          <southBL>80</southBL>
-          <northBL>70</northBL>
+          <southBL>70</southBL>
+          <northBL>80</northBL>
           <eastBL>60</eastBL>
           <westBL>50</westBL>
         </geoBndBox>

doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json

@@ -12,8 +12,16 @@
         "PID": "{datasetPid}"
       },
       {
-        "apiToken": "{apiToken}"
+        "locale":"{localeCode}"
       }
-    ]
+    ],
+    "allowedApiCalls": [
+        {
+          "name":"retrieveDatasetJson",
+          "httpMethod":"GET",
+          "urlTemplate":"/api/v1/datasets/{datasetId}",
+          "timeOut":10
+        }
+      ]
   }
 }