Merge branch 'develop' into 7418-datasourcedefinition IQSS#7418

conf/docker-aio/0prep_deps.sh

@@ -4,10 +4,10 @@ if [ ! -d dv/deps ]; then
 fi
 wdir=`pwd`
 
-if [ ! -e dv/deps/payara-5.2020.2.zip ]; then
+if [ ! -e dv/deps/payara-5.2020.6.zip ]; then
 	echo "payara dependency prep"
 	# no more fiddly patching :)
-	wget https://github.com/payara/Payara/releases/download/payara-server-5.2020.2/payara-5.2020.2.zip  -O dv/deps/payara-5.2020.2.zip
+	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip  -O dv/deps/payara-5.2020.6.zip
 fi
 
 if [ ! -e dv/deps/solr-7.7.2dv.tgz ]; then

conf/docker-aio/c8.dockerfile

@@ -18,7 +18,7 @@ COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
 RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz 
-RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.2.zip ; ln -s /opt/payara5 /opt/glassfish4
+RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
 COPY domain-restmonitor.xml /opt/payara5/glassfish/domains/domain1/config/domain.xml

conf/docker-aio/run-test-suite.sh

@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT -Ddataverse.test.baseurl=$dvurl
+mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl

conf/docker-aio/testscripts/install

@@ -15,7 +15,6 @@ export SMTP_SERVER=localhost
 export MEM_HEAP_SIZE=2048
 export GLASSFISH_DOMAIN=domain1
 cd scripts/installer
-cp pgdriver/postgresql-42.2.12.jar $GLASSFISH_ROOT/glassfish/lib
 #cp ../../conf/jhove/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
 cp /opt/dv/testdata/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
 cp /opt/dv/testdata/jhoveConfig.xsd $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhoveConfig.xsd

doc/release-notes/5345-ejb-timers.md

@@ -0,0 +1,4 @@
+Reset the EJB timer database back to default:
+```
+<payara install path>/asadmin set configs.config.server-config.ejb-container.ejb-timer-service.timer-datasource=jdbc/__TimerPool
+```

doc/release-notes/7417-payara-5.2020.6.md

@@ -0,0 +1,12 @@
+## Update to Payara Platform 5.2020.6
+
+In contrast to good old Glassfish 4, the Payara application server
+is a maintained product, receiving both feature and - more important -
+security and dependent technology updates. You should update the
+application server platform on a regular basis, as it is a key component
+of a running Dataverse deployment.
+
+<!-- REQUIREMENT MARKER FOR LATER USE WITH #6819 AS HTML COMMENT. MAY BE OMMITED. -->
+
+Instructions how to update can be found in the
+[Payara documentation](https://docs.payara.fish/community/docs/5.2020.6/documentation/user-guides/upgrade-payara.html)

doc/sphinx-guides/source/_static/util/clear_timer.sh

@@ -1,9 +1,8 @@
 #!/bin/sh
 
-# EBJ timers sometimes cause problems; utility to clear generated directories and database rows
+# EBJ timers sometimes cause problems; utility to clear generated directories
 
-# assumes this script is being run as root, and that the postgres user had passwordless 
-# access to the database (local sockets, or appropriate environmental variables).
+# assumes this script is being run as root
 
 # will restart Payara if it's stopped; comment out the `start-domain` command at the end
 # if you'd like to avoid that.
@@ -14,19 +13,11 @@ PAYARA_DIR=/usr/local/payara5
 # directory within Payara (defaults)
 DV_DIR=${PAYARA_DIR}/glassfish/domains/domain1
 
-# name of dataverse database
-DV_DB=dvndb
-
-# OS user for the database
-DB_USER=postgres
-
 # stop the domain (generates a warning if app server is stopped)
 ${PAYARA_DIR}/bin/asadmin stop-domain
 
 rm -rf ${PAYARA_DIR}/${DV_DIR}/generated/
 rm -rf ${PAYARA_DIR}/${DV_DIR}/osgi-cache/felix
 
-sudo -u ${DB_USER} psql ${DV_DB} -c 'delete from "EJB__TIMER__TBL"';
-
 # restart the domain (also generates a warning if app server is stopped)
 ${PAYARA_DIR}/bin/asadmin start-domain

doc/sphinx-guides/source/admin/timers.rst

@@ -22,9 +22,7 @@ The following JVM option instructs the application to act as the dedicated timer
 
 ``-Ddataverse.timerServer=true``
 
-**IMPORTANT:** Note that this option is automatically set by the Dataverse installer script. That means that when **configuring a multi-server cluster**, it will be the responsibility of the installer to remove the option from the :fixedwidthplain:`domain.xml` of every node except the one intended to be the timer server. We also recommend that the following entry in the :fixedwidthplain:`domain.xml`: ``<ejb-timer-service timer-datasource="jdbc/VDCNetDS">`` is changed back to ``<ejb-timer-service>`` on all the non-timer server nodes. Similarly, this option is automatically set by the installer script. Changing it back to the default setting on a server that doesn't need to run the timer will prevent a potential race condition, where multiple servers try to get a lock on the timer database. 
-
-**Note** that for the timer to work, the version of the PostgreSQL JDBC driver your instance is using must match the version of your PostgreSQL database. See the :ref:`timer-not-working` section of Troubleshooting in the Admin Guide.
+**IMPORTANT:** Note that this option is automatically set by the Dataverse installer script. That means that when **configuring a multi-server cluster**, it will be the responsibility of the installer to remove the option from the :fixedwidthplain:`domain.xml` of every node except the one intended to be the timer server.
 
 Harvesting Timers 
 -----------------
@@ -44,7 +42,7 @@ This timer runs a daily job that tries to export all the local, published datase
 
 This daily job will also update all the harvestable OAI sets configured on your server, adding new and/or newly published datasets or marking deaccessioned datasets as "deleted" in the corresponding sets as needed. 
 
-This job is automatically scheduled to run at 2AM local time every night. If really necessary, it is possible (for an advanced user) to change that time by directly editing the EJB timer application table in the database.  
+This job is automatically scheduled to run at 2AM local time every night.
 
 .. _saved-search-timer:
 

doc/sphinx-guides/source/admin/troubleshooting.rst

@@ -98,14 +98,12 @@ We don't know what's causing this issue, but here's a known workaround:
 
 - Stop Payara; 
 
-- Remove the ``generated`` and ``osgi-cache`` directories; 
-
-- Delete all the rows from the ``EJB__TIMER__TBL`` table in the database;
+- Remove the ``generated`` and ``osgi-cache`` directories;
 
 - Start Payara
 
 The shell script below performs the steps above. 
-Note that it may or may not work on your system, so it is provided as an example only, downloadable :download:`here </_static/util/clear_timer.sh>`. Aside from the configuration values that need to be changed to reflect your environment (the Payara directory, name of the database, etc.) the script relies on the database being configured in a certain way for access. (See the comments in the script for more information)
+Note that it may or may not work on your system, so it is provided as an example only, downloadable :download:`here </_static/util/clear_timer.sh>`. The configuration values might need to be changed to reflect your environment (the Payara directory). See the comments in the script for more information.
 
 .. literalinclude:: ../_static/util/clear_timer.sh
 
@@ -114,7 +112,7 @@ Note that it may or may not work on your system, so it is provided as an example
 Timer Not Working
 -----------------
 
-Dataverse relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, this may be the result of the incompatibility between the version of PostgreSQL database you are using, and PostgreSQL JDBC driver in use by your instance of Payara. The symptoms:
+Dataverse relies on EJB timers to perform scheduled tasks: harvesting from remote servers, updating the local OAI sets and running metadata exports. (See :doc:`timers` for details.) If these scheduled jobs are not running on your server, you might experience the following symptoms:
 
 If you are seeing the following in your server.log...
 
@@ -127,9 +125,9 @@ followed by an Exception stack trace with these lines in it:
 :fixedwidthplain:`Exception Description: Could not deserialize object from byte array` ...
 
 
-... it most likely means that it is the JDBC driver incompatibility that's preventing the timer from working correctly. 
-Make sure you install the correct version of the driver. For example, if you are running the version 9.3 of PostgreSQL, make sure you have the driver postgresql-9.3-1104.jdbc4.jar in your :fixedwidthplain:`<PAYARA FOLDER>/glassfish/lib` directory. Go `here <https://jdbc.postgresql.org/download.html>`_
-to download the correct version of the driver. If you have an older driver in glassfish/lib, make sure to remove it, replace it with the new version and restart Payara. (You may need to remove the entire contents of :fixedwidthplain:`<PAYARA FOLDER>/glassfish/domains/domain1/generated` before you start Payara). 
+... you should reach out by opening an issue. In the good old days of running Dataverse 4 running on Glassfish 4, this
+was a hint for an unsupported JDBC driver. In Dataverse 5 this would be a new regression and its cause would need to be
+investigated.
 
 
 Constraint Violations Issues

doc/sphinx-guides/source/api/native-api.rst

@@ -1176,7 +1176,7 @@ The fully expanded example above (without environment variables) looks like this
 
 .. code-block:: bash
 
-  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"} https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB"
 
 You should expect a 201 ("CREATED") response and JSON indicating the database id that has been assigned to your newly uploaded file.
 

doc/sphinx-guides/source/developers/dev-environment.rst

@@ -85,9 +85,9 @@ To install Payara, run the following commands:
 
 ``cd /usr/local``
 
-``sudo curl -O -L https://github.com/payara/Payara/releases/download/payara-server-5.2020.2/payara-5.2020.2.zip``
+``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip``
 
-``sudo unzip payara-5.2020.2.zip``
+``sudo unzip payara-5.2020.6.zip``
 
 ``sudo chown -R $USER /usr/local/payara5``
 

doc/sphinx-guides/source/developers/tips.rst

@@ -176,7 +176,6 @@ Switching from Glassfish to Payara
 If you already have a working dev environment with Glassfish and want to switch to Payara, you must do the following:
 
 - Copy the "domain1" directory from Glassfish to Payara.
-- Copy the PostgreSQL driver into place like this: ``cp scripts/installer/pgdriver/postgresql-42.*.jar /usr/local/payara5/glassfish/lib``
 
 ----
 

doc/sphinx-guides/source/installation/installation-main.rst

@@ -208,8 +208,7 @@ Fresh Reinstall
 Early on when you're installing Dataverse, you may think, "I just want to blow away what I've installed and start over." That's fine. You don't have to uninstall the various components like Payara, PostgreSQL and Solr, but you should be conscious of how to clear out their data. For Payara, a common helpful process is to:
 
 - Stop Payara; 
-- Remove the ``generated`` and ``osgi-cache`` directories; 
-- Delete all the rows from the ``EJB__TIMER__TBL`` table in the database;
+- Remove the ``generated`` and ``osgi-cache`` directories;
 - Start Payara
 
 Drop database

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -47,7 +47,7 @@ If you don't want to be prompted, here is an example of the non-interactive invo
 Payara
 ------
 
-Payara 5.2020.2 is recommended. Newer versions might work fine.
+Payara 5.2020.6 is recommended. Newer versions might work fine, regular updates are recommended.
 
 Installing Payara
 =================
@@ -58,8 +58,8 @@ Installing Payara
 
 - Download and install Payara (installed in ``/usr/local/payara5`` in the example commands below)::
 
-	# wget https://github.com/payara/Payara/releases/download/payara-server-5.2020.2/payara-5.2020.2.zip
-	# unzip payara-5.2020.2.zip
+	# wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip
+	# unzip payara-5.2020.6.zip
 	# mv payara5 /usr/local
 
 If you intend to install and run Payara under a service account (and we hope you do), chown -R the Payara hierarchy to root to protect it but give the service account access to the below directories:

downloads/download.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-curl -L -O https://github.com/payara/Payara/releases/download/payara-server-5.2020.2/payara-5.2020.2.zip
+curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip
 curl -L -O https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz
 curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar
 curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar

pom.xml

@@ -26,7 +26,8 @@
         <skipUnitTests>false</skipUnitTests>
 
         <jakartaee-api.version>8.0.0</jakartaee-api.version>
-        <payara.version>5.2020.2</payara.version>
+        <payara.version>5.2020.6</payara.version>
+        <postgresql.version>42.2.18</postgresql.version>
         <aws.version>1.11.762</aws.version>
         <commons.logging.version>1.2</commons.logging.version>
         <httpcomponents.client.version>4.5.5</httpcomponents.client.version>
@@ -263,6 +264,11 @@
             <artifactId>jbcrypt</artifactId>
             <version>0.3m</version>
         </dependency>
+        <dependency>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>${postgresql.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.flywaydb</groupId>
             <artifactId>flyway-core</artifactId>

scripts/database/homebrew/devinstall

@@ -15,6 +15,5 @@ export FILES_DIR=$HOME/dataverse/files
 export MEM_HEAP_SIZE=2048
 export GLASSFISH_DOMAIN=domain1
 export GLASSFISH_ROOT=/Applications/NetBeans/glassfish4
-cp pgdriver/postgresql-9.1-902.jdbc4.jar $GLASSFISH_ROOT/glassfish/lib
 cp ../../conf/jhove/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
 ./glassfish-setup.sh

scripts/deploy/phoenix.dataverse.org/install

@@ -15,6 +15,5 @@ export SMTP_SERVER=localhost
 export MEM_HEAP_SIZE=2048
 export GLASSFISH_DOMAIN=domain1
 cd scripts/installer
-cp pgdriver/postgresql-8.4-703.jdbc4.jar $GLASSFISH_ROOT/glassfish/lib
 cp ../../conf/jhove/jhove.conf $GLASSFISH_ROOT/glassfish/domains/$GLASSFISH_DOMAIN/config/jhove.conf
 ./glassfish-setup.sh

scripts/installer/Makefile

@@ -1,7 +1,6 @@
 INSTALLER_ZIP_DIR=dvinstall
 DISTRIBUTION_WAR_FILE=${INSTALLER_ZIP_DIR}/dataverse.war
 GLASSFISH_SETUP_SCRIPT=${INSTALLER_ZIP_DIR}/as-setup.sh
-POSTGRES_DRIVERS=${INSTALLER_ZIP_DIR}/pgdriver
 API_SCRIPTS=${INSTALLER_ZIP_DIR}/setup-datasetfields.sh ${INSTALLER_ZIP_DIR}/setup-users.sh ${INSTALLER_ZIP_DIR}/setup-builtin-roles.sh ${INSTALLER_ZIP_DIR}/setup-dvs.sh ${INSTALLER_ZIP_DIR}/data ${INSTALLER_ZIP_DIR}/setup-identity-providers.sh  ${INSTALLER_ZIP_DIR}/setup-all.sh ${INSTALLER_ZIP_DIR}/post-install-api-block.sh
 JHOVE_CONFIG=${INSTALLER_ZIP_DIR}/jhove.conf
 JHOVE_SCHEMA=${INSTALLER_ZIP_DIR}/jhoveConfig.xsd
@@ -15,9 +14,9 @@ installer:	dvinstall.zip
 clean: 
 	/bin/rm -rf ${INSTALLER_ZIP_DIR} dvinstall.zip
 
-dvinstall.zip:	${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
+dvinstall.zip:	${GLASSFISH_SETUP_SCRIPT} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
 	@echo making installer...
-	zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${POSTGRES_DRIVERS} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
+	zip -r dvinstall.zip ${GLASSFISH_SETUP_SCRIPT} ${JSF_PATCH_DIR} ${DISTRIBUTION_WAR_FILE} ${API_SCRIPTS} ${JHOVE_CONFIG} ${JHOVE_SCHEMA} ${SOLR_SCHEMA} ${SOLR_CONFIG} ${PYTHON_FILES} ${INSTALL_SCRIPT}
 	@echo 
 	@echo "Done!"
 
@@ -45,11 +44,6 @@ ${GLASSFISH_SETUP_SCRIPT}:	as-setup.sh ${INSTALLER_ZIP_DIR}
 	@echo copying glassfish setup
 	/bin/cp as-setup.sh ${INSTALLER_ZIP_DIR}
 
-${POSTGRES_DRIVERS}:	 pgdriver/postgresql-42.2.12.jar
-	mkdir -p ${POSTGRES_DRIVERS}
-	@echo copying postgres driver
-	/bin/cp pgdriver/postgresql-42.2.12.jar ${POSTGRES_DRIVERS}
-
 ${API_SCRIPTS}:	../api/setup-datasetfields.sh ../api/setup-users.sh ../api/setup-dvs.sh ../api/setup-identity-providers.sh ../api/setup-all.sh ../api/post-install-api-block.sh ../api/setup-builtin-roles.sh ../api/data ${INSTALLER_ZIP_DIR}
 	@echo copying api scripts
 	/bin/cp -R ../api/setup-datasetfields.sh ../api/setup-users.sh ../api/setup-dvs.sh ../api/setup-identity-providers.sh ../api/setup-all.sh ../api/post-install-api-block.sh ../api/setup-builtin-roles.sh ../api/data ${INSTALLER_ZIP_DIR}

scripts/installer/README.txt

@@ -20,7 +20,6 @@ from scripts/installer (this directory):
 
 install
 glassfish-setup.sh
-pgdriver (the entire directory with all its contents)
 
 from scripts/api:
 

scripts/installer/as-setup.sh

@@ -156,11 +156,6 @@ function final_setup(){
         ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.db.port=${DB_PORT}"
         ./asadmin $ASADMIN_OPTS create-system-properties "dataverse.db.name=${DB_NAME}"
 
-        ###
-        # Set up the data source for the timers
-
-        ./asadmin $ASADMIN_OPTS set configs.config.server-config.ejb-container.ejb-timer-service.timer-datasource=jdbc/VDCNetDS
-
         ./asadmin $ASADMIN_OPTS create-jvm-options "\-Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"
 
 	### 

scripts/installer/install

@@ -840,19 +840,6 @@ sub setup_appserver {
 # with the MacOS glassfish build...). Verify, and if still the case,
 # add a check.
 
-    print "\nInstalling the PostgresQL driver for Payara5... ";
-#    system( "/bin/cp", "pgdriver/" . $postgres_jdbc, $glassfish_dir . "/glassfish/lib" );
-    my $pgdriver_success = copy("pgdriver/" . $postgres_jdbc, $glassfish_dir . "/glassfish/lib" );
-
-    unless ($pgdriver_success) 
-    {
-	print "\n*********************\n";
-	print "ERROR! Failed to copy the postgres driver into " . $glassfish_dir . "/glassfish/lib - check the directory permissions!\n";
-	exit 1;
-    }
-
-    print "done!\n";
-
     print "\n*********************\n";
     print "PLEASE NOTE, SOME OF THE ASADMIN COMMANDS ARE GOING TO FAIL,\n";
     print "FOR EXAMPLE, IF A CONFIGURATION SETTING THAT WE ARE TRYING\n";