Integrate Bulk Export (#1487)

* Start working on subscription processor

* Work on new scheduler

* Test fixes

* Scheduler refactoring

* Fix test failure

* One more test fix

* Updates to scheduler

* More scheduler work

* Tests now all passing

* Ongoing work on export

* Ongoing scheduler work

* Ongoing testing

* Work on export task

* Sync master

* Ongoing work

* Bump xml patch version

* Work on provider

* Work on bulk

* Work on export scheduler

* More test fies

* More test fixes

* Compile fix

* Reduce logging

* Improve logging

* Reuse bulk export jobs

* Export provider

* Improve logging in bulk export

* Work on bulk export service

* One more bugfix

* Ongoing work on Bulk Data

* Add changelog

hapi-fhir-android/src/test/java/ca/uhn/fhir/android/client/GenericClientDstu3IT.java

@@ -8,6 +8,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Date;
 
+import ca.uhn.fhir.rest.api.PreferHeader;
 import org.hl7.fhir.dstu3.model.*;
 import org.junit.*;
 import org.mockito.ArgumentCaptor;
@@ -20,7 +21,6 @@
 import ca.uhn.fhir.parser.IParser;
 import ca.uhn.fhir.rest.api.Constants;
 import ca.uhn.fhir.rest.api.MethodOutcome;
-import ca.uhn.fhir.rest.api.PreferReturnEnum;
 import ca.uhn.fhir.rest.client.api.IGenericClient;
 import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum;
 import ca.uhn.fhir.rest.client.exceptions.FhirClientConnectionException;
@@ -246,7 +246,7 @@ public void testCreateWithPreferRepresentationServerReturnsResource() {
 		Patient pt = new Patient();
 		pt.getText().setDivAsString("A PATIENT");
 
-		MethodOutcome outcome = client.create().resource(pt).prefer(PreferReturnEnum.REPRESENTATION).execute();
+		MethodOutcome outcome = client.create().resource(pt).prefer(PreferHeader.PreferReturnEnum.REPRESENTATION).execute();
 
 		assertNull(outcome.getOperationOutcome());
 		assertNotNull(outcome.getResource());

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/Constants.java

@@ -220,17 +220,31 @@ public class Constants {
 	public static final String CASCADE_DELETE = "delete";
 	public static final int MAX_RESOURCE_NAME_LENGTH = 100;
 	public static final String CACHE_CONTROL_PRIVATE = "private";
+	public static final String CT_FHIR_NDJSON = "application/fhir+ndjson";
+	public static final String CT_APP_NDJSON = "application/ndjson";
+	public static final String CT_NDJSON = "ndjson";
+	public static final Set<String> CTS_NDJSON;
+	public static final String HEADER_PREFER_RESPOND_ASYNC = "respond-async";
 	public static final int STATUS_HTTP_412_PAYLOAD_TOO_LARGE = 413;
 	public static final String OPERATION_NAME_GRAPHQL = "$graphql";
 	/**
 	 * Note that this constant is used in a number of places including DB column lengths! Be careful if you decide to change it.
 	 */
 	public static final int REQUEST_ID_LENGTH = 16;
+	public static final int STATUS_HTTP_202_ACCEPTED = 202;
+	public static final String HEADER_X_PROGRESS = "X-Progress";
+	public static final String HEADER_RETRY_AFTER = "Retry-After";
 
 	static {
 		CHARSET_UTF8 = StandardCharsets.UTF_8;
 		CHARSET_US_ASCII = StandardCharsets.ISO_8859_1;
 
+		HashSet<String> ctsNdjson = new HashSet<>();
+		ctsNdjson.add(CT_FHIR_NDJSON);
+		ctsNdjson.add(CT_APP_NDJSON);
+		ctsNdjson.add(CT_NDJSON);
+		CTS_NDJSON = Collections.unmodifiableSet(ctsNdjson);
+
 		HashMap<Integer, String> statusNames = new HashMap<>();
 		statusNames.put(200, "OK");
 		statusNames.put(201, "Created");

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/api/PreferHeader.java

@@ -0,0 +1,61 @@
+package ca.uhn.fhir.rest.api;
+
+import javax.annotation.Nullable;
+import java.util.HashMap;
+
+public class PreferHeader {
+
+	private PreferReturnEnum myReturn;
+	private boolean myRespondAsync;
+
+	public @Nullable
+	PreferReturnEnum getReturn() {
+		return myReturn;
+	}
+
+	public PreferHeader setReturn(PreferReturnEnum theReturn) {
+		myReturn = theReturn;
+		return this;
+	}
+
+	public boolean getRespondAsync() {
+		return myRespondAsync;
+	}
+
+	public PreferHeader setRespondAsync(boolean theRespondAsync) {
+		myRespondAsync = theRespondAsync;
+		return this;
+	}
+
+	/**
+	 * Represents values for "return" value as provided in the the <a href="https://tools.ietf.org/html/rfc7240#section-4.2">HTTP Prefer header</a>.
+	 */
+	public enum PreferReturnEnum {
+
+		REPRESENTATION("representation"), MINIMAL("minimal"), OPERATION_OUTCOME("OperationOutcome");
+
+		private static HashMap<String, PreferReturnEnum> ourValues;
+		private String myHeaderValue;
+
+		PreferReturnEnum(String theHeaderValue) {
+			myHeaderValue = theHeaderValue;
+		}
+
+		public String getHeaderValue() {
+			return myHeaderValue;
+		}
+
+		public static PreferReturnEnum fromHeaderValue(String theHeaderValue) {
+			if (ourValues == null) {
+				HashMap<String, PreferReturnEnum> values = new HashMap<>();
+				for (PreferReturnEnum next : PreferReturnEnum.values()) {
+					values.put(next.getHeaderValue(), next);
+				}
+				ourValues = values;
+			}
+			return ourValues.get(theHeaderValue);
+		}
+
+	}
+
+}

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/gclient/ICreateTyped.java

@@ -21,7 +21,7 @@
  */
 
 import ca.uhn.fhir.rest.api.MethodOutcome;
-import ca.uhn.fhir.rest.api.PreferReturnEnum;
+import ca.uhn.fhir.rest.api.PreferHeader;
 
 public interface ICreateTyped extends IClientExecutable<ICreateTyped, MethodOutcome> {
 
@@ -47,6 +47,6 @@ public interface ICreateTyped extends IClientExecutable<ICreateTyped, MethodOutc
 	 * 
 	 * @since HAPI 1.1
 	 */
-	ICreateTyped prefer(PreferReturnEnum theReturn);
+	ICreateTyped prefer(PreferHeader.PreferReturnEnum theReturn);
 
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/gclient/IPatchExecutable.java

@@ -21,7 +21,7 @@
  */
 
 import ca.uhn.fhir.rest.api.MethodOutcome;
-import ca.uhn.fhir.rest.api.PreferReturnEnum;
+import ca.uhn.fhir.rest.api.PreferHeader;
 
 public interface IPatchExecutable extends IClientExecutable<IPatchExecutable, MethodOutcome>{
 
@@ -32,6 +32,6 @@ public interface IPatchExecutable extends IClientExecutable<IPatchExecutable, Me
 	 * 
 	 * @since HAPI 1.1
 	 */
-	IPatchExecutable prefer(PreferReturnEnum theReturn);
+	IPatchExecutable prefer(PreferHeader.PreferReturnEnum theReturn);
 
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/gclient/IUpdateExecutable.java

@@ -21,7 +21,7 @@
  */
 
 import ca.uhn.fhir.rest.api.MethodOutcome;
-import ca.uhn.fhir.rest.api.PreferReturnEnum;
+import ca.uhn.fhir.rest.api.PreferHeader;
 
 public interface IUpdateExecutable extends IClientExecutable<IUpdateExecutable, MethodOutcome>{
 
@@ -32,6 +32,6 @@ public interface IUpdateExecutable extends IClientExecutable<IUpdateExecutable,
 	 * 
 	 * @since HAPI 1.1
 	 */
-	IUpdateExecutable prefer(PreferReturnEnum theReturn);
+	IUpdateExecutable prefer(PreferHeader.PreferReturnEnum theReturn);
 
 }

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ArrayUtil.java

@@ -0,0 +1,25 @@
+package ca.uhn.fhir.util;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.commons.lang3.StringUtils.*;
+
+public class ArrayUtil {
+
+	/** Non instantiable */
+	private ArrayUtil() {}
+
+	/**
+	 * Takes in a list like "foo, bar,, baz" and returns a set containing only ["foo", "bar", "baz"]
+	 */
+	public static Set<String> commaSeparatedListToCleanSet(String theValueAsString) {
+		Set<String> resourceTypes;
+		resourceTypes = Arrays.stream(split(theValueAsString, ","))
+			.map(t->trim(t))
+			.filter(t->isNotBlank(t))
+			.collect(Collectors.toSet());
+		return resourceTypes;
+	}
+}

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/StopWatch.java

@@ -63,6 +63,9 @@ public StopWatch(Date theStart) {
 		myStarted = theStart.getTime();
 	}
 
+	public StopWatch(long theL) {
+	}
+
 	private void addNewlineIfContentExists(StringBuilder theB) {
 		if (theB.length() > 0) {
 			theB.append("\n");
@@ -231,7 +234,12 @@ public double getThroughput(long theNumOperations, TimeUnit theUnit) {
 
 		double denominator = ((double) millisElapsed) / ((double) periodMillis);
 
-		return (double) theNumOperations / denominator;
+		double throughput = (double) theNumOperations / denominator;
+		if (throughput > theNumOperations) {
+			throughput = theNumOperations;
+		}
+
+		return throughput;
 	}
 
 	public void restart() {

hapi-fhir-cli/hapi-fhir-cli-jpaserver/pom.xml

@@ -5,9 +5,9 @@
 		if you are using this file as a basis for your own project. -->
 	<parent>
 		<groupId>ca.uhn.hapi.fhir</groupId>
-		<artifactId>hapi-fhir-cli</artifactId>
+		<artifactId>hapi-deployable-pom</artifactId>
 		<version>4.1.0-SNAPSHOT</version>
-		<relativePath>../pom.xml</relativePath>
+		<relativePath>../../hapi-deployable-pom</relativePath>
 	</parent>
 
 	<artifactId>hapi-fhir-cli-jpaserver</artifactId>
@@ -131,6 +131,10 @@
 					<artifactId>Saxon-HE</artifactId>
 					<groupId>net.sf.saxon</groupId>
 				</exclusion>
+				<exclusion>
+					<groupId>org.glassfish.jaxb</groupId>
+					<artifactId>jaxb-core</artifactId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 	</dependencies>
@@ -183,15 +187,6 @@
 				</configuration>
 			</plugin>
 
-			<!-- This plugin is just a part of the HAPI internal build process, you do not need to incude it in your own projects -->
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-deploy-plugin</artifactId>
-				<configuration>
-					<skip>true</skip>
-				</configuration>
-			</plugin>
-
 			<!-- This is to run the integration tests -->
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>

hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/CommonConfig.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.demo;
 
+/*-
+ * #%L
+ * HAPI FHIR - Command Line Client - Server WAR
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.model.entity.ModelConfig;
 import ca.uhn.fhir.jpa.search.LuceneSearchMappingFactory;

hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/ContextHolder.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.demo;
 
+/*-
+ * #%L
+ * HAPI FHIR - Command Line Client - Server WAR
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import org.apache.commons.cli.ParseException;

hapi-fhir-cli/hapi-fhir-cli-jpaserver/src/main/java/ca/uhn/fhir/jpa/demo/FhirServerConfig.java

@@ -1,5 +1,25 @@
 package ca.uhn.fhir.jpa.demo;
 
+/*-
+ * #%L
+ * HAPI FHIR - Command Line Client - Server WAR
+ * %%
+ * Copyright (C) 2014 - 2019 University Health Network
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import ca.uhn.fhir.jpa.config.BaseJavaConfigDstu2;
 import ca.uhn.fhir.jpa.dao.DaoConfig;
 import ca.uhn.fhir.jpa.util.SubscriptionsRequireManualActivationInterceptorDstu2;