Skip to content

feature/batches execution #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jan 22, 2021
Merged

feature/batches execution #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6f39a8a
- adds an execute method for JSON batches to avoid customers juggling…
baywet Jan 20, 2021
53c58ba
- reverts string serialization bypass to avoid unconsistent behavior
baywet Jan 21, 2021
73219ae
- adds defensive programming
baywet Jan 21, 2021
f227c49
- replaces try catches in batch unit tests by assert throws for readb…
baywet Jan 21, 2021
91e2945
- replaces calls to obsolete APIs
baywet Jan 21, 2021
21944eb
- enables and fixes last disabled tests for CoreHttpProvider
baywet Jan 21, 2021
927ab2e
- typo
baywet Jan 22, 2021
b26bcdd
Merge pull request #115 from microsoftgraph/feature/enable-disabled-t…
baywet Jan 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 58 additions & 23 deletions src/main/java/com/microsoft/graph/content/MSBatchRequestContent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package com.microsoft.graph.content;

import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand All @@ -17,6 +18,8 @@
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.microsoft.graph.core.ClientException;
import com.microsoft.graph.core.IBaseClient;
import com.google.gson.JsonParseException;

import okhttp3.Headers;
Expand All @@ -40,20 +43,14 @@ public class MSBatchRequestContent {
*
* @param batchRequestStepsArray List of batch steps for batching
*/
public MSBatchRequestContent(@Nonnull final List<MSBatchRequestStep> batchRequestStepsArray) {
if (batchRequestStepsArray.size() > MAX_NUMBER_OF_REQUESTS)
public MSBatchRequestContent(@Nonnull final MSBatchRequestStep... batchRequestStepsArray) {
if (batchRequestStepsArray.length > MAX_NUMBER_OF_REQUESTS)
throw new IllegalArgumentException("Number of batch request steps cannot exceed " + MAX_NUMBER_OF_REQUESTS);

this.batchRequestStepsHashMap = new LinkedHashMap<>();
for (final MSBatchRequestStep requestStep : batchRequestStepsArray)
addBatchRequestStep(requestStep);
}

/**
* Creates empty batch request content
*/
public MSBatchRequestContent() {
this.batchRequestStepsHashMap = new LinkedHashMap<>();
if(requestStep != null)
addBatchRequestStep(requestStep);
}

/**
Expand All @@ -63,6 +60,8 @@ public MSBatchRequestContent() {
* given
*/
public boolean addBatchRequestStep(@Nonnull final MSBatchRequestStep batchRequestStep) {
if(batchRequestStep == null)
throw new IllegalArgumentException("batchRequestStep parameter cannot be null");
if (batchRequestStepsHashMap.containsKey(batchRequestStep.getRequestId()) ||
batchRequestStepsHashMap.size() >= MAX_NUMBER_OF_REQUESTS)
return false;
Expand All @@ -78,11 +77,13 @@ public boolean addBatchRequestStep(@Nonnull final MSBatchRequestStep batchReques
*/
@Nonnull
public String addBatchRequestStep(@Nonnull final Request request, @Nullable final String... arrayOfDependsOnIds) {
if(request == null)
throw new IllegalArgumentException("request parameter cannot be null");
String requestId;
do {
requestId = Integer.toString(ThreadLocalRandom.current().nextInt(1, Integer.MAX_VALUE));
} while(batchRequestStepsHashMap.keySet().contains(requestId));
if(addBatchRequestStep(new MSBatchRequestStep(requestId, request, Arrays.asList(arrayOfDependsOnIds))))
if(addBatchRequestStep(new MSBatchRequestStep(requestId, request, arrayOfDependsOnIds)))
return requestId;
else
throw new IllegalArgumentException("unable to add step to batch. Number of batch request steps cannot exceed " + MAX_NUMBER_OF_REQUESTS);
Expand All @@ -100,29 +101,63 @@ public boolean removeBatchRequestStepWithId(@Nonnull final String requestId) {
batchRequestStepsHashMap.remove(requestId);
removed = true;
for (final Map.Entry<String, MSBatchRequestStep> steps : batchRequestStepsHashMap.entrySet()) {
if (steps.getValue() != null && steps.getValue().getArrayOfDependsOnIds() != null) {
while (steps.getValue().getArrayOfDependsOnIds().remove(requestId))
if (steps.getValue() != null && steps.getValue().getDependsOnIds() != null) {
while (steps.getValue().getDependsOnIds().remove(requestId))
;
}
}
}
return removed;
}

/**
* @return Batch request content's json as String
*/
@Nonnull
public String getBatchRequestContent() {
private JsonObject getBatchRequestContentAsJson() {
final JsonObject batchRequestContentMap = new JsonObject();
final JsonArray batchContentArray = new JsonArray();
for (final Map.Entry<String, MSBatchRequestStep> requestStep : batchRequestStepsHashMap.entrySet()) {
batchContentArray.add(getBatchRequestObjectFromRequestStep(requestStep.getValue()));
}
batchRequestContentMap.add("requests", batchContentArray);
return batchRequestContentMap;
}
/**
* @return Batch request content's json as String
*/
@Nonnull
public String getBatchRequestContent() {
return getBatchRequestContentAsJson().toString();
}

final String content = batchRequestContentMap.toString();
return content;
/**
* Executes the batch requests and returns the response
* @param client client to use for the request
* @return the batch response
* @throws ClientException when the batch couldn't be executed because of client issues.
*/
@Nonnull
public MSBatchResponseContent execute(@Nonnull final IBaseClient client) {
final JsonObject content = getBatchRequestContentAsJson();
return new MSBatchResponseContent(client.getServiceRoot() + "/",
content,
client.customRequest("/$batch")
.buildRequest()
.post(content)
.getAsJsonObject());
}
/**
* Executes the batch requests asynchronously and returns the response
* @param client client to use for the request
* @return a future with the batch response
*/
@Nonnull
public CompletableFuture<MSBatchResponseContent> executeAsync(@Nonnull final IBaseClient client) {
if(client == null) {
throw new IllegalArgumentException("client parameter cannot be null");
}
final JsonObject content = getBatchRequestContentAsJson();
return client.customRequest("/$batch")
.buildRequest()
.postAsync(content)
.thenApply(resp -> new MSBatchResponseContent(client.getServiceRoot() + "/", content, resp.getAsJsonObject()));
}

private static final Pattern protocolAndHostReplacementPattern = Pattern.compile("(?i)^http[s]?:\\/\\/graph\\.microsoft\\.com\\/(?>v1\\.0|beta)\\/?"); // (?i) case insensitive
Expand All @@ -146,9 +181,9 @@ private JsonObject getBatchRequestObjectFromRequestStep(final MSBatchRequestStep
contentmap.add("headers", headerMap);
}

final List<String> arrayOfDependsOnIds = batchRequestStep.getArrayOfDependsOnIds();
final HashSet<String> arrayOfDependsOnIds = batchRequestStep.getDependsOnIds();
if (arrayOfDependsOnIds != null) {
final JsonArray array = new JsonArray();
final JsonArray array = new JsonArray(arrayOfDependsOnIds.size());
for (final String dependsOnId : arrayOfDependsOnIds)
array.add(dependsOnId);
contentmap.add("dependsOn", array);
Expand Down
15 changes: 8 additions & 7 deletions src/main/java/com/microsoft/graph/content/MSBatchRequestStep.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.microsoft.graph.content;

import java.util.List;
import java.util.Arrays;
import java.util.HashSet;

import javax.annotation.Nullable;
import javax.annotation.Nonnull;
Expand All @@ -13,25 +14,25 @@
public class MSBatchRequestStep {
private String requestId;
private Request request;
private List<String> arrayOfDependsOnIds;
private HashSet<String> dependsOnIds;

/**
* Initializes a batch step from a raw HTTP request
* @param requestId the id to assign to this step
* @param request the request to send in the batch
* @param arrayOfDependsOnIds the ids of steps this step depends on
*/
public MSBatchRequestStep(@Nonnull final String requestId, @Nonnull final Request request, @Nullable final List<String> arrayOfDependsOnIds) {
public MSBatchRequestStep(@Nonnull final String requestId, @Nonnull final Request request, @Nullable final String... arrayOfDependsOnIds) {
if(requestId == null)
throw new IllegalArgumentException("Request Id cannot be null.");
if(requestId.length() == 0)
throw new IllegalArgumentException("Request Id cannot be empty.");
if(request == null)
new IllegalArgumentException("Request cannot be null.");
throw new IllegalArgumentException("Request cannot be null.");

this.requestId = requestId;
this.request = request;
this.arrayOfDependsOnIds = arrayOfDependsOnIds;
this.dependsOnIds = new HashSet<>(Arrays.asList(arrayOfDependsOnIds));
}

/**
Expand All @@ -57,7 +58,7 @@ public Request getRequest() {
* @return the list of steps this step depends on
*/
@Nullable
public List<String> getArrayOfDependsOnIds(){
return arrayOfDependsOnIds;
public HashSet<String> getDependsOnIds(){
return dependsOnIds;
}
}
89 changes: 58 additions & 31 deletions src/main/java/com/microsoft/graph/content/MSBatchResponseContent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.google.gson.JsonParseException;

import okhttp3.MediaType;
import okhttp3.Protocol;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
Expand All @@ -26,17 +27,34 @@
*/
public class MSBatchResponseContent {

private final Response batchResponse;
private LinkedHashMap<String, Request> batchRequestsHashMap;
private final Protocol protocol;
private final String message;
private LinkedHashMap<String, Request> batchRequestsHashMap = new LinkedHashMap<>();
private JsonArray batchResponseArray;
private String nextLink;

/**
* @param batchResponse OkHttp batch response on execution of batch requests
*/
public MSBatchResponseContent(@Nullable final Response batchResponse) {
this.batchResponse = batchResponse;
update(batchResponse);
this.message = batchResponse.message();
this.protocol = batchResponse.protocol();
}
/**
* instantiates a new response
* internal only, used when the content executes the requests
* @param baseUrl the base service URL without a trailing slash
* @param batchRequestData the batch request payload data as a JSON string
* @param batchResponseData the batch response body as a JSON string
*/
protected MSBatchResponseContent(@Nonnull final String baseUrl, @Nonnull final JsonObject batchRequestData, @Nonnull final JsonObject batchResponseData) {
this.protocol = Protocol.HTTP_1_1;
this.message = "OK";
final Map<String, Request> requestMap = createBatchRequestsHashMap(baseUrl, batchRequestData);
if (requestMap != null)
batchRequestsHashMap.putAll(requestMap);
updateFromResponseBody(batchResponseData);
}

/**
Expand Down Expand Up @@ -66,14 +84,13 @@ public Response getResponseById(@Nonnull final String requestId) {
// Put corresponding request into the constructed response
builder.request(batchRequestsHashMap.get(requestId));
// copy protocol and message same as of batch response
builder.protocol(batchResponse.protocol());
builder.message(batchResponse.message());
builder.protocol(protocol);
builder.message(message);

// Put status code of the corresponding request in JsonArray
final JsonElement statusElement = jsonresponse.get("status");
if (statusElement != null && statusElement.isJsonPrimitive()) {
final Long status = statusElement.getAsLong();
builder.code(status.intValue());
builder.code(statusElement.getAsInt());
}

// Put body from response array for corresponding id into constructing response
Expand All @@ -82,7 +99,7 @@ public Response getResponseById(@Nonnull final String requestId) {
final JsonObject JsonObject = jsonBodyElement.getAsJsonObject();
final String bodyAsString = JsonObject.toString();
final ResponseBody responseBody = ResponseBody
.create(MediaType.parse("application/json; charset=utf-8"), bodyAsString);
.create(bodyAsString, MediaType.parse("application/json; charset=utf-8"));
builder.body(responseBody);
}

Expand Down Expand Up @@ -144,37 +161,36 @@ public void update(@Nonnull final Response batchResponse) {
throw new IllegalArgumentException("Batch Response cannot be null");

final Map<String, Request> requestMap = createBatchRequestsHashMap(batchResponse);
if (batchRequestsHashMap == null)
batchRequestsHashMap = new LinkedHashMap<>();
if (requestMap != null)
batchRequestsHashMap.putAll(requestMap);

if (batchResponse.body() != null) {
try {
final String batchResponseData = batchResponse.body().string();
if (batchResponseData != null) {
final JsonObject batchResponseObj = stringToJSONObject(batchResponseData);
if (batchResponseObj != null) {

final JsonElement nextLinkElement = batchResponseObj.get("@odata.nextLink");
if (nextLinkElement != null && nextLinkElement.isJsonPrimitive())
nextLink = nextLinkElement.getAsString();

if (batchResponseArray == null)
batchResponseArray = new JsonArray();

final JsonElement responseArrayElement = batchResponseObj.get("responses");
if (responseArrayElement != null && responseArrayElement.isJsonArray()) {
final JsonArray responseArray = responseArrayElement.getAsJsonArray();
batchResponseArray.addAll(responseArray);
}
}
updateFromResponseBody(stringToJSONObject(batchResponseData));
}
} catch (final IOException e) {
e.printStackTrace();
}
}
}
private void updateFromResponseBody(@Nonnull final JsonObject batchResponseObj) {
if (batchResponseObj != null) {
final JsonElement nextLinkElement = batchResponseObj.get("@odata.nextLink");
if (nextLinkElement != null && nextLinkElement.isJsonPrimitive())
nextLink = nextLinkElement.getAsString();

if (batchResponseArray == null)
batchResponseArray = new JsonArray();

final JsonElement responseArrayElement = batchResponseObj.get("responses");
if (responseArrayElement != null && responseArrayElement.isJsonArray()) {
final JsonArray responseArray = responseArrayElement.getAsJsonArray();
batchResponseArray.addAll(responseArray);
}
}
}

/**
* @return nextLink of batch response
Expand All @@ -188,8 +204,20 @@ private Map<String, Request> createBatchRequestsHashMap(final Response batchResp
if (batchResponse == null)
return null;
try {
final Map<String, Request> batchRequestsHashMap = new LinkedHashMap<>();
final JsonObject requestJSONObject = requestBodyToJSONObject(batchResponse.request());
final String baseUrl = batchResponse.request().url().toString().replace("$batch", "");
return createBatchRequestsHashMap(baseUrl, requestJSONObject);
} catch (IOException ex) {
ex.printStackTrace();
return null;
}
}
private Map<String, Request> createBatchRequestsHashMap(@Nonnull final String baseUrl, @Nonnull final JsonObject requestJSONObject) {
if(baseUrl == null || baseUrl == "" || requestJSONObject == null) {
return null;
}
try {
final Map<String, Request> batchRequestsHashMap = new LinkedHashMap<>();
final JsonElement requestArrayElement = requestJSONObject.get("requests");
if (requestArrayElement != null && requestArrayElement.isJsonArray()) {
final JsonArray requestArray = requestArrayElement.getAsJsonArray();
Expand All @@ -202,8 +230,7 @@ private Map<String, Request> createBatchRequestsHashMap(final Response batchResp

final JsonElement urlElement = requestObject.get("url");
if (urlElement != null && urlElement.isJsonPrimitive()) {
final StringBuilder fullUrl = new StringBuilder(
batchResponse.request().url().toString().replace("$batch", ""));
final StringBuilder fullUrl = new StringBuilder(baseUrl);
fullUrl.append(urlElement.getAsString());
builder.url(fullUrl.toString());
}
Expand All @@ -227,7 +254,7 @@ private Map<String, Request> createBatchRequestsHashMap(final Response batchResp
final JsonObject JsonObject = jsonBodyElement.getAsJsonObject();
final String bodyAsString = JsonObject.toString();
final RequestBody requestBody = RequestBody
.create(MediaType.parse("application/json; charset=utf-8"), bodyAsString);
.create(bodyAsString, MediaType.parse("application/json; charset=utf-8"));
builder.method(jsonMethodElement.getAsString(), requestBody);
} else if (jsonMethodElement != null) {
builder.method(jsonMethodElement.getAsString(), null);
Expand All @@ -240,7 +267,7 @@ private Map<String, Request> createBatchRequestsHashMap(final Response batchResp
}
return batchRequestsHashMap;

} catch (IOException | JsonParseException e) {
} catch (JsonParseException e) {
e.printStackTrace();
}
return null;
Expand Down
Loading