Skip to content

Commit

Permalink
Remove unused md5 field + optimise file upload (#216)
Browse files Browse the repository at this point in the history
  • Loading branch information
@timja
timja authored Feb 10, 2022
1 parent 89b407b commit 467495d
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 87 deletions.
18 changes: 0 additions & 18 deletions src/main/java/com/microsoftopentechnologies/windowsazurestorage/AzureBlob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,18 @@ public class AzureBlob implements Serializable {

private final String blobName;
private final String blobURL;
private final String md5;
private final long byteSize;
private final String storageType;
private final String credentialsId;

@Deprecated
public AzureBlob(
String blobName,
String blobURL,
String md5,
long byteSize,
String storageType) {
this(blobName, blobURL, md5, byteSize, storageType, null);
}

public AzureBlob(
String blobName,
String blobURL,
String md5,
long byteSize,
String storageType,
String credentialsId) {
this.blobName = blobName;
this.blobURL = blobURL;
this.md5 = md5;
this.byteSize = byteSize;
this.storageType = storageType;
this.credentialsId = credentialsId;
Expand All @@ -56,11 +43,6 @@ public String getBlobURL() {
return blobURL;
}

@Exported
public String getMd5() {
return md5;
}

@Exported
public long getSizeInBytes() {
return byteSize;
Expand Down
35 changes: 5 additions & 30 deletions src/main/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import com.azure.storage.blob.options.BlobUploadFromFileOptions;
import com.azure.storage.blob.sas.BlobSasPermission;
import com.azure.storage.file.share.ShareFileClient;
import com.azure.storage.file.share.models.ShareFileUploadInfo;
import com.azure.storage.file.share.sas.ShareFileSasPermission;
import com.microsoftopentechnologies.windowsazurestorage.AzureBlob;
import com.microsoftopentechnologies.windowsazurestorage.AzureBlobMetadataPair;
Expand All @@ -50,16 +49,12 @@
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpStatus;

import javax.xml.bind.DatatypeConverter;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.Serializable;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -118,11 +113,10 @@ class FileUploadThread implements Runnable {
public void run() {
try {
AzureBlob azureBlob;
String uploadedFileHash = uploadCloudFile(uploadItem, filePath);
uploadCloudFile(uploadItem, filePath);
azureBlob = new AzureBlob(
uploadItem.getShareName(),
uploadItem.getFileUrl(),
uploadedFileHash,
filePath.length(),
Constants.FILE_STORAGE,
getServiceData().getCredentialsId()
Expand Down Expand Up @@ -229,7 +223,6 @@ protected static class UploadResult implements Serializable {
private static final long serialVersionUID = -3112548564900823521L;
private int statusCode;
private String responseBody;
private String fileHash;
private String name;
private String url;
private long byteSize;
Expand All @@ -242,20 +235,17 @@ protected static class UploadResult implements Serializable {
*
* @param statusCode Status code for uploading task, the same as the http response code.
* @param responseBody Response from the server side. Provide detailed information when the task fails.
* @param fileHash The hash code for the uploaded object, calculate on the agent
* to save resources on master.
* @param name The name of the uploaded object.
* @param url The target url of the uploaded object.
* @param byteSize The byte size of the uploaded object.
* @param storageType Storage type of the uploaded object.
* @param startTime Start time of the uploading task.
* @param endTime End time of the uploading task.
*/
public UploadResult(int statusCode, String responseBody, String fileHash, String name,
public UploadResult(int statusCode, String responseBody, String name,
String url, long byteSize, String storageType, long startTime, long endTime) {
this.statusCode = statusCode;
this.responseBody = responseBody;
this.fileHash = fileHash;
this.name = name;
this.url = url;
this.byteSize = byteSize;
Expand All @@ -272,10 +262,6 @@ public String getResponseBody() {
return responseBody;
}

public String getFileHash() {
return fileHash;
}

public String getName() {
return name;
}
Expand Down Expand Up @@ -359,7 +345,6 @@ protected void updateAzureBlobs(List<UploadResult> results,
AzureBlob azureBlob = new AzureBlob(
result.getName(),
result.getUrl(),
result.getFileHash(),
result.getByteSize(),
result.getStorageType(),
serviceData.getCredentialsId());
Expand Down Expand Up @@ -437,18 +422,9 @@ public UploadResult call() {
if (!uploadObject.getMetadata().isEmpty()) {
blockBlobClient.setMetadata(uploadObject.getMetadata());
}
byte[] md5 = block.getValue().getContentMd5();
long endTime = System.currentTimeMillis();

String fileHash = null;
// seen to occur when uploading large files, (768mb in this case)
// 201 response code with no hash in response
if (md5 != null) {
fileHash = new String(md5, StandardCharsets.UTF_8);
}

return new UploadResult(block.getStatusCode(), null,
fileHash,
uploadObject.getName(),
uploadObject.getUrl(), length, uploadObject.getStorageType(),
startTime, endTime);
Expand Down Expand Up @@ -576,21 +552,20 @@ protected void waitForUploadEnd() throws InterruptedException, WAStorageExceptio
}
}

protected String uploadCloudFile(ShareFileClient fileClient, FilePath localPath)
protected void uploadCloudFile(ShareFileClient fileClient, FilePath localPath)
throws WAStorageException {
long startTime = System.currentTimeMillis();
File file = new File(localPath.getRemote());
try (FileInputStream fis = new FileInputStream(file); BufferedInputStream bis = new BufferedInputStream(fis)) {
try {
long bytes = Files.size(file.toPath());
fileClient.create(bytes);

ShareFileUploadInfo response = fileClient.upload(bis, bis.available(), null);
fileClient.uploadFromFile(file.getAbsolutePath());

long endTime = System.currentTimeMillis();
if (getServiceData().isVerbose()) {
println("Uploaded file with uri " + fileClient.getFileUrl() + " in " + getTime(endTime - startTime));
}
return DatatypeConverter.printHexBinary(response.getContentMd5());
} catch (Exception e) {
throw new WAStorageException("Failed uploading file", e);
}
Expand Down
49 changes: 10 additions & 39 deletions ...n/java/com/microsoftopentechnologies/windowsazurestorage/service/UploadToFileService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,12 @@

import com.azure.core.credential.AzureSasCredential;
import com.azure.core.http.rest.PagedIterable;
import com.azure.core.http.rest.Response;
import com.azure.storage.file.share.ShareClient;
import com.azure.storage.file.share.ShareDirectoryClient;
import com.azure.storage.file.share.ShareFileClient;
import com.azure.storage.file.share.ShareServiceClient;
import com.azure.storage.file.share.ShareServiceClientBuilder;
import com.azure.storage.file.share.models.ShareFileItem;
import com.azure.storage.file.share.models.ShareFileUploadInfo;
import com.azure.storage.file.share.models.ShareFileUploadOptions;
import com.microsoftopentechnologies.windowsazurestorage.beans.StorageAccountInfo;
import com.microsoftopentechnologies.windowsazurestorage.exceptions.WAStorageException;
import com.microsoftopentechnologies.windowsazurestorage.helper.AzureUtils;
Expand All @@ -42,13 +39,10 @@
import jenkins.model.Jenkins;
import org.apache.commons.lang.StringUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -64,8 +58,6 @@

public class UploadToFileService extends UploadService {

private static final String EMPTY_STRING_MD5 = "68b329da9893e34099c7d8ad5cb9c940";

public UploadToFileService(UploadServiceData serviceData) {
super(serviceData);
}
Expand Down Expand Up @@ -158,50 +150,29 @@ public List<UploadResult> invoke(File f, VirtualChannel channel) {
private UploadResult uploadCloudFile(ShareFileClient fileClient, UploadObject uploadObject) {
long startTime = System.currentTimeMillis();
File file = new File(uploadObject.getSrc().getRemote());
try (FileInputStream fis = new FileInputStream(file);
BufferedInputStream bis = new BufferedInputStream(fis)) {
try {
long bytes = Files.size(file.toPath());
fileClient.create(bytes);

// https://github.com/Azure/azure-sdk-for-java/issues/26867
if (bytes == 0L) {
long endTime = System.currentTimeMillis();
return new UploadResult(HttpStatus.SC_CREATED, null,
EMPTY_STRING_MD5,
file.getName(),
uploadObject.getUrl(),
bytes,
uploadObject.getStorageType(),
startTime,
endTime
);
}


ShareFileUploadOptions fileUploadOptions = new ShareFileUploadOptions(bis);
Response<ShareFileUploadInfo> response = fileClient
.uploadWithResponse(fileUploadOptions, null, null);
fileClient.uploadFromFile(file.getAbsolutePath());

long endTime = System.currentTimeMillis();

String fileHash = null;
byte[] md5 = response.getValue().getContentMd5();
if (md5 != null) {
fileHash = new String(md5, StandardCharsets.UTF_8);
}

return new UploadResult(response.getStatusCode(), null,
fileHash,
return new UploadResult(HttpStatus.SC_CREATED, null,
file.getName(),
uploadObject.getUrl(), file.length(), uploadObject.getStorageType(),
startTime, endTime);
} catch (Exception e) {
LOGGER.log(Level.SEVERE, "Failed uploading file", e);
return new UploadResult(ERROR_ON_UPLOAD, null,
return new UploadResult(ERROR_ON_UPLOAD,
null,
file.getName(),
uploadObject.getUrl(), file.length(), uploadObject.getStorageType(),
startTime, 0);
uploadObject.getUrl(),
file.length(),
uploadObject.getStorageType(),
startTime,
0
);
}
}
}
Expand Down

0 comments on commit 467495d

Please sign in to comment.