Skip to content

Commit cebb0bb

Browse files
mziccardmziccard
committed
Add BlobWriteOption to support MD5 and CRC32C checks on create/write
- Add MD5 and CRC32C computation to create and create from byte array - Change BlobTargetOption... to BlobWriteOption... in create from stream and writer - Change BlobTargetOption... to BlobWriteOption in Blob.writer - Update unit tests - Update and add integration tests
1 parent 30992ae commit cebb0bb

File tree

7 files changed

+181
-27
lines changed

7 files changed

+181
-27
lines changed

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
2424
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
2525
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
26+
import static com.google.gcloud.spi.StorageRpc.Option.IF_MD5_MATCH;
27+
import static com.google.gcloud.spi.StorageRpc.Option.IF_CRC32C_MATCH;
2628
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
2729
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
2830
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
@@ -106,6 +108,15 @@ private static StorageException translate(GoogleJsonError exception) {
106108
return new StorageException(exception.getCode(), exception.getMessage(), retryable);
107109
}
108110

111+
private static void applyOptions(StorageObject storageObject, Map<Option, ?> options) {
112+
if (IF_MD5_MATCH.getBoolean(options) == null) {
113+
storageObject.setMd5Hash(null);
114+
}
115+
if (IF_CRC32C_MATCH.getBoolean(options) == null) {
116+
storageObject.setCrc32c(null);
117+
}
118+
}
119+
109120
@Override
110121
public Bucket create(Bucket bucket, Map<Option, ?> options) throws StorageException {
111122
try {
@@ -123,6 +134,7 @@ public Bucket create(Bucket bucket, Map<Option, ?> options) throws StorageExcept
123134
@Override
124135
public StorageObject create(StorageObject storageObject, final InputStream content,
125136
Map<Option, ?> options) throws StorageException {
137+
applyOptions(storageObject, options);
126138
try {
127139
Storage.Objects.Insert insert = storage.objects()
128140
.insert(storageObject.getBucket(), storageObject,
@@ -491,6 +503,7 @@ public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObj
491503
@Override
492504
public String open(StorageObject object, Map<Option, ?> options)
493505
throws StorageException {
506+
applyOptions(object, options);
494507
try {
495508
Insert req = storage.objects().insert(object.getBucket(), object);
496509
GenericUrl url = req.buildHttpRequest().getUrl();

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ enum Option {
4242
IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
4343
IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
4444
IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
45+
IF_MD5_MATCH("md5Hash"),
46+
IF_CRC32C_MATCH("crc32c"),
4547
PREFIX("prefix"),
4648
MAX_RESULTS("maxResults"),
4749
PAGE_TOKEN("pageToken"),

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.google.common.collect.Lists;
2525
import com.google.gcloud.spi.StorageRpc;
2626
import com.google.gcloud.storage.Storage.BlobTargetOption;
27+
import com.google.gcloud.storage.Storage.BlobWriteOption;
2728
import com.google.gcloud.storage.Storage.CopyRequest;
2829
import com.google.gcloud.storage.Storage.SignUrlOption;
2930

@@ -274,7 +275,7 @@ public BlobReadChannel reader(BlobSourceOption... options) {
274275
* @param options target blob options
275276
* @throws StorageException upon failure
276277
*/
277-
public BlobWriteChannel writer(BlobTargetOption... options) {
278+
public BlobWriteChannel writer(BlobWriteOption... options) {
278279
return storage.writer(info, options);
279280
}
280281

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java‎

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,67 @@ public static BlobTargetOption metagenerationMatch() {
145145
public static BlobTargetOption metagenerationNotMatch() {
146146
return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH);
147147
}
148+
149+
static BlobWriteOption[] convert(BlobTargetOption[] options, BlobWriteOption... optionsToAdd) {
150+
BlobWriteOption[] writeOptions = new BlobWriteOption[options.length + optionsToAdd.length];
151+
int index = 0;
152+
for (BlobTargetOption option : options) {
153+
writeOptions[index++] = new BlobWriteOption(option);
154+
}
155+
for (BlobWriteOption option : optionsToAdd) {
156+
writeOptions[index++] = option;
157+
}
158+
return writeOptions;
159+
}
160+
}
161+
162+
class BlobWriteOption extends Option {
163+
164+
private static final long serialVersionUID = -3880421670966224580L;
165+
166+
BlobWriteOption(BlobTargetOption option) {
167+
super(option.rpcOption(), option.value());
168+
}
169+
170+
private BlobWriteOption(StorageRpc.Option rpcOption, Object value) {
171+
super(rpcOption, value);
172+
}
173+
174+
private BlobWriteOption(StorageRpc.Option rpcOption) {
175+
this(rpcOption, null);
176+
}
177+
178+
public static BlobWriteOption predefinedAcl(PredefinedAcl acl) {
179+
return new BlobWriteOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
180+
}
181+
182+
public static BlobWriteOption doesNotExist() {
183+
return new BlobWriteOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L);
184+
}
185+
186+
public static BlobWriteOption generationMatch() {
187+
return new BlobWriteOption(StorageRpc.Option.IF_GENERATION_MATCH);
188+
}
189+
190+
public static BlobWriteOption generationNotMatch() {
191+
return new BlobWriteOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH);
192+
}
193+
194+
public static BlobWriteOption metagenerationMatch() {
195+
return new BlobWriteOption(StorageRpc.Option.IF_METAGENERATION_MATCH);
196+
}
197+
198+
public static BlobWriteOption metagenerationNotMatch() {
199+
return new BlobWriteOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH);
200+
}
201+
202+
public static BlobWriteOption md5Match() {
203+
return new BlobWriteOption(StorageRpc.Option.IF_MD5_MATCH, true);
204+
}
205+
206+
public static BlobWriteOption crc32cMatch() {
207+
return new BlobWriteOption(StorageRpc.Option.IF_CRC32C_MATCH, true);
208+
}
148209
}
149210

150211
class BlobSourceOption extends Option {
@@ -510,10 +571,12 @@ public static Builder builder() {
510571

511572
/**
512573
* Create a new blob. Direct upload is used to upload {@code content}. For large content,
513-
* {@link #writer} is recommended as it uses resumable upload.
574+
* {@link #writer} is recommended as it uses resumable upload. MD5 and CRC32C hashes of
575+
* {@code content} are computed and used for validating transferred data.
514576
*
515577
* @return a complete blob information.
516578
* @throws StorageException upon failure
579+
* @see <a href="https://cloud.google.com/storage/docs/hashes-etags">Hashes and ETags</a>
517580
*/
518581
BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options);
519582

@@ -524,7 +587,7 @@ public static Builder builder() {
524587
* @return a complete blob information.
525588
* @throws StorageException upon failure
526589
*/
527-
BlobInfo create(BlobInfo blobInfo, InputStream content, BlobTargetOption... options);
590+
BlobInfo create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options);
528591

529592
/**
530593
* Return the requested bucket or {@code null} if not found.
@@ -683,7 +746,7 @@ public static Builder builder() {
683746
*
684747
* @throws StorageException upon failure
685748
*/
686-
BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options);
749+
BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options);
687750

688751
/**
689752
* Generates a signed URL for a blob.

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import com.google.common.collect.Lists;
4141
import com.google.common.collect.Maps;
4242
import com.google.common.collect.Sets;
43+
import com.google.common.hash.Hashing;
4344
import com.google.common.io.BaseEncoding;
4445
import com.google.common.primitives.Ints;
4546
import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
@@ -93,13 +94,14 @@ public RetryResult beforeEval(Exception exception) {
9394
static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
9495
.abortOn(RuntimeException.class).interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
9596
private static final byte[] EMPTY_BYTE_ARRAY = {};
97+
private static final String EMPTY_BYTE_ARRAY_MD5 = "1B2M2Y8AsgTpgAmY7PhCfg==";
98+
private static final String EMPTY_BYTE_ARRAY_CRC32C = "AAAAAA==";
9699

97100
private final StorageRpc storageRpc;
98101

99102
StorageImpl(StorageOptions options) {
100103
super(options);
101104
storageRpc = options.storageRpc();
102-
// todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors
103105
// todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite
104106
// todo: check if we need to expose https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert vs using bucket update/patch
105107
}
@@ -123,18 +125,29 @@ public com.google.api.services.storage.model.Bucket call() {
123125

124126
@Override
125127
public BlobInfo create(BlobInfo blobInfo, BlobTargetOption... options) {
126-
return create(blobInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY), options);
128+
BlobInfo updatedInfo = blobInfo.toBuilder()
129+
.md5(EMPTY_BYTE_ARRAY_MD5)
130+
.crc32c(EMPTY_BYTE_ARRAY_CRC32C)
131+
.build();
132+
return create(updatedInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY),
133+
BlobTargetOption.convert(
134+
options, BlobWriteOption.md5Match(), BlobWriteOption.crc32cMatch()));
127135
}
128136

129137
@Override
130-
public BlobInfo create(BlobInfo blobInfo, final byte[] content, BlobTargetOption... options) {
131-
return create(blobInfo,
132-
new ByteArrayInputStream(firstNonNull(content, EMPTY_BYTE_ARRAY)), options);
138+
public BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) {
139+
content = firstNonNull(content, EMPTY_BYTE_ARRAY);
140+
BlobInfo updatedInfo = blobInfo.toBuilder()
141+
.md5(BaseEncoding.base64().encode(Hashing.md5().hashBytes(content).asBytes()))
142+
.crc32c(BaseEncoding.base64().encode(
143+
Ints.toByteArray(Hashing.crc32c().hashBytes(content).asInt())))
144+
.build();
145+
return create(updatedInfo, new ByteArrayInputStream(content), BlobTargetOption.convert(
146+
options, BlobWriteOption.md5Match(), BlobWriteOption.crc32cMatch()));
133147
}
134148

135149
@Override
136-
public BlobInfo create(BlobInfo blobInfo, final InputStream content,
137-
BlobTargetOption... options) {
150+
public BlobInfo create(BlobInfo blobInfo, final InputStream content, BlobWriteOption... options) {
138151
final StorageObject blobPb = blobInfo.toPb();
139152
final Map<StorageRpc.Option, ?> optionsMap = optionMap(blobInfo, options);
140153
try {
@@ -544,7 +557,7 @@ public BlobReadChannel reader(BlobId blob, BlobSourceOption... options) {
544557
}
545558

546559
@Override
547-
public BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) {
560+
public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {
548561
final Map<StorageRpc.Option, ?> optionsMap = optionMap(blobInfo, options);
549562
return new BlobWriteChannelImpl(options(), blobInfo, optionsMap);
550563
}

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,22 @@ public void testCreateBlobFail() {
145145
assertTrue(storage.delete(bucket, blobName));
146146
}
147147

148+
@Test
149+
public void testCreateBlobMd5Fail() throws UnsupportedEncodingException {
150+
String blobName = "test-create-blob-md5-fail";
151+
BlobInfo blob = BlobInfo.builder(bucket, blobName)
152+
.contentType(CONTENT_TYPE)
153+
.md5("O1R4G1HJSDUISJjoIYmVhQ==")
154+
.build();
155+
ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
156+
try {
157+
storage.create(blob, stream, Storage.BlobWriteOption.md5Match());
158+
fail("StorageException was expected");
159+
} catch (StorageException ex) {
160+
// expected
161+
}
162+
}
163+
148164
@Test
149165
public void testUpdateBlob() {
150166
String blobName = "test-update-blob";
@@ -449,7 +465,7 @@ public void testWriteChannelFail() throws UnsupportedEncodingException, IOExcept
449465
BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(-1L).build();
450466
try {
451467
try (BlobWriteChannel writer =
452-
storage.writer(blob, Storage.BlobTargetOption.generationMatch())) {
468+
storage.writer(blob, Storage.BlobWriteOption.generationMatch())) {
453469
writer.write(ByteBuffer.allocate(42));
454470
}
455471
fail("StorageException was expected");
@@ -458,6 +474,20 @@ public void testWriteChannelFail() throws UnsupportedEncodingException, IOExcept
458474
}
459475
}
460476

477+
@Test
478+
public void testWriteChannelExistingBlob() throws UnsupportedEncodingException, IOException {
479+
String blobName = "test-write-channel-existing-blob";
480+
BlobInfo blob = BlobInfo.builder(bucket, blobName).build();
481+
BlobInfo remoteBlob = storage.create(blob);
482+
byte[] stringBytes;
483+
try (BlobWriteChannel writer = storage.writer(remoteBlob)) {
484+
stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8);
485+
writer.write(ByteBuffer.wrap(stringBytes));
486+
}
487+
assertArrayEquals(stringBytes, storage.readAllBytes(blob.blobId()));
488+
assertTrue(storage.delete(bucket, blobName));
489+
}
490+
461491
@Test
462492
public void testGetSignedUrl() throws IOException {
463493
String blobName = "test-get-signed-url-blob";

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java‎

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ public class StorageImplTest {
7676
private static final String BLOB_NAME2 = "n2";
7777
private static final String BLOB_NAME3 = "n3";
7878
private static final byte[] BLOB_CONTENT = {0xD, 0xE, 0xA, 0xD};
79+
private static final String CONTENT_MD5 = "O1R4G1HJSDUISJjoIYmVhQ==";
80+
private static final String CONTENT_CRC32C = "9N3EPQ==";
7981
private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
8082

8183
// BucketInfo objects
@@ -121,6 +123,27 @@ public class StorageImplTest {
121123
StorageRpc.Option.IF_GENERATION_MATCH, BLOB_INFO1.generation(),
122124
StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.metageneration());
123125

126+
// Blob write options (create, writer)
127+
private static final Storage.BlobWriteOption BLOB_WRITE_METAGENERATION =
128+
Storage.BlobWriteOption.metagenerationMatch();
129+
private static final Storage.BlobWriteOption BLOB_WRITE_NOT_EXIST =
130+
Storage.BlobWriteOption.doesNotExist();
131+
private static final Storage.BlobWriteOption BLOB_WRITE_PREDEFINED_ACL =
132+
Storage.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE);
133+
private static final Storage.BlobWriteOption BLOB_WRITE_MD5_HASH =
134+
Storage.BlobWriteOption.md5Match();
135+
private static final Storage.BlobWriteOption BLOB_WRITE_CRC2C =
136+
Storage.BlobWriteOption.crc32cMatch();
137+
private static final Map<StorageRpc.Option, ?> BLOB_WRITE_OPTIONS_SIMPLE = ImmutableMap.of(
138+
StorageRpc.Option.IF_MD5_MATCH, true,
139+
StorageRpc.Option.IF_CRC32C_MATCH, true);
140+
private static final Map<StorageRpc.Option, ?> BLOB_WRITE_OPTIONS_COMPLEX = ImmutableMap.of(
141+
StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.metageneration(),
142+
StorageRpc.Option.IF_GENERATION_MATCH, 0L,
143+
StorageRpc.Option.PREDEFINED_ACL, BUCKET_TARGET_PREDEFINED_ACL.value(),
144+
StorageRpc.Option.IF_MD5_MATCH, true,
145+
StorageRpc.Option.IF_CRC32C_MATCH, true);
146+
124147
// Bucket source options
125148
private static final Storage.BucketSourceOption BUCKET_SOURCE_METAGENERATION =
126149
Storage.BucketSourceOption.metagenerationMatch(BUCKET_INFO1.metageneration());
@@ -250,10 +273,10 @@ public void testCreateBlob() throws IOException {
250273
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock);
251274
EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries());
252275
Capture<ByteArrayInputStream> capturedStream = Capture.newInstance();
253-
EasyMock
254-
.expect(
255-
storageRpcMock.create(EasyMock.eq(BLOB_INFO1.toPb()), EasyMock.capture(capturedStream),
256-
EasyMock.eq(EMPTY_RPC_OPTIONS)))
276+
EasyMock.expect(storageRpcMock.create(
277+
EasyMock.eq(BLOB_INFO1.toBuilder().md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build().toPb()),
278+
EasyMock.capture(capturedStream),
279+
EasyMock.eq(BLOB_WRITE_OPTIONS_SIMPLE)))
257280
.andReturn(BLOB_INFO1.toPb());
258281
EasyMock.replay(optionsMock, storageRpcMock);
259282
storage = StorageFactory.instance().get(optionsMock);
@@ -271,10 +294,14 @@ public void testCreateEmptyBlob() throws IOException {
271294
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock);
272295
EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries());
273296
Capture<ByteArrayInputStream> capturedStream = Capture.newInstance();
274-
EasyMock
275-
.expect(
276-
storageRpcMock.create(EasyMock.eq(BLOB_INFO1.toPb()), EasyMock.capture(capturedStream),
277-
EasyMock.eq(EMPTY_RPC_OPTIONS)))
297+
EasyMock.expect(storageRpcMock.create(
298+
EasyMock.eq(BLOB_INFO1.toBuilder()
299+
.md5("1B2M2Y8AsgTpgAmY7PhCfg==")
300+
.crc32c("AAAAAA==")
301+
.build()
302+
.toPb()),
303+
EasyMock.capture(capturedStream),
304+
EasyMock.eq(BLOB_WRITE_OPTIONS_SIMPLE)))
278305
.andReturn(BLOB_INFO1.toPb());
279306
EasyMock.replay(optionsMock, storageRpcMock);
280307
storage = StorageFactory.instance().get(optionsMock);
@@ -290,9 +317,14 @@ public void testCreateBlobWithOptions() throws IOException {
290317
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock);
291318
EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries());
292319
Capture<ByteArrayInputStream> capturedStream = Capture.newInstance();
293-
EasyMock.expect(
294-
storageRpcMock.create(EasyMock.eq(BLOB_INFO1.toPb()), EasyMock.capture(capturedStream),
295-
EasyMock.eq(BLOB_TARGET_OPTIONS_CREATE)))
320+
EasyMock.expect(storageRpcMock.create(
321+
EasyMock.eq(BLOB_INFO1.toBuilder()
322+
.md5(CONTENT_MD5)
323+
.crc32c(CONTENT_CRC32C)
324+
.build()
325+
.toPb()),
326+
EasyMock.capture(capturedStream),
327+
EasyMock.eq(BLOB_WRITE_OPTIONS_COMPLEX)))
296328
.andReturn(BLOB_INFO1.toPb());
297329
EasyMock.replay(optionsMock, storageRpcMock);
298330
storage = StorageFactory.instance().get(optionsMock);
@@ -787,12 +819,12 @@ public void testWriter() {
787819
@Test
788820
public void testWriterWithOptions() {
789821
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock).times(2);
790-
EasyMock.expect(storageRpcMock.open(BLOB_INFO1.toPb(), BLOB_TARGET_OPTIONS_CREATE))
822+
EasyMock.expect(storageRpcMock.open(BLOB_INFO1.toPb(), BLOB_WRITE_OPTIONS_COMPLEX))
791823
.andReturn("upload-id");
792824
EasyMock.replay(optionsMock, storageRpcMock);
793825
storage = StorageFactory.instance().get(optionsMock);
794-
BlobWriteChannel channel = storage.writer(BLOB_INFO1, BLOB_TARGET_METAGENERATION,
795-
BLOB_TARGET_NOT_EXIST, BLOB_TARGET_PREDEFINED_ACL);
826+
BlobWriteChannel channel = storage.writer(BLOB_INFO1, BLOB_WRITE_METAGENERATION,
827+
BLOB_WRITE_NOT_EXIST, BLOB_WRITE_PREDEFINED_ACL, BLOB_WRITE_CRC2C, BLOB_WRITE_MD5_HASH);
796828
assertNotNull(channel);
797829
assertTrue(channel.isOpen());
798830
}

0 commit comments

Comments
 (0)