Skip to content

Commit

Permalink
Fixed block issue and tier issue (#12978)
Browse files Browse the repository at this point in the history
  • Loading branch information
gapra-msft authored Jul 13, 2020
1 parent 65a33d3 commit acb8f13
Show file tree
Hide file tree
Showing 11 changed files with 931 additions and 47 deletions.
6 changes: 6 additions & 0 deletions sdk/storage/azure-storage-blob-cryptography/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,12 @@
<version>1.0.8</version> <!-- {x-version-update;com.azure:azure-identity;dependency} -->
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-security-keyvault-keys</artifactId>
<version>4.2.0-beta.5</version> <!-- {x-version-update;com.azure:azure-security-keyvault-keys;current} -->
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ public Mono<Response<BlockBlobItem>> uploadWithResponse(Flux<ByteBuffer> data,
AccessTier tier, BlobRequestConditions requestConditions) {
return this.uploadWithResponse(new BlobParallelUploadOptions(data)
.setParallelTransferOptions(parallelTransferOptions).setHeaders(headers).setMetadata(metadata)
.setTier(AccessTier.HOT).setRequestConditions(requestConditions));
.setTier(tier).setRequestConditions(requestConditions));
}

/**
Expand Down Expand Up @@ -443,58 +443,58 @@ Mono<EncryptedBlob> encryptBlob(Flux<ByteBuffer> plainTextFlux) throws InvalidKe
keyWrappingMetadata.put(CryptographyConstants.AGENT_METADATA_KEY,
CryptographyConstants.AGENT_METADATA_VALUE);

return this.keyWrapper.wrapKey(keyWrapAlgorithm, aesKey.getEncoded())
.map(encryptedKey -> {
WrappedKey wrappedKey = new WrappedKey(
this.keyWrapper.getKeyId().block(), encryptedKey, keyWrapAlgorithm);

// Build EncryptionData
EncryptionData encryptionData = new EncryptionData()
.setEncryptionMode(CryptographyConstants.ENCRYPTION_MODE)
.setEncryptionAgent(
new EncryptionAgent(CryptographyConstants.ENCRYPTION_PROTOCOL_V1,
EncryptionAlgorithm.AES_CBC_256))
.setKeyWrappingMetadata(keyWrappingMetadata)
.setContentEncryptionIV(cipher.getIV())
.setWrappedContentKey(wrappedKey);

// Encrypt plain text with content encryption key
Flux<ByteBuffer> encryptedTextFlux = plainTextFlux.map(plainTextBuffer -> {
int outputSize = cipher.getOutputSize(plainTextBuffer.remaining());
return this.keyWrapper.getKeyId().flatMap(keyId ->
this.keyWrapper.wrapKey(keyWrapAlgorithm, aesKey.getEncoded())
.map(encryptedKey -> {
WrappedKey wrappedKey = new WrappedKey(keyId, encryptedKey, keyWrapAlgorithm);

// Build EncryptionData
EncryptionData encryptionData = new EncryptionData()
.setEncryptionMode(CryptographyConstants.ENCRYPTION_MODE)
.setEncryptionAgent(
new EncryptionAgent(CryptographyConstants.ENCRYPTION_PROTOCOL_V1,
EncryptionAlgorithm.AES_CBC_256))
.setKeyWrappingMetadata(keyWrappingMetadata)
.setContentEncryptionIV(cipher.getIV())
.setWrappedContentKey(wrappedKey);

// Encrypt plain text with content encryption key
Flux<ByteBuffer> encryptedTextFlux = plainTextFlux.map(plainTextBuffer -> {
int outputSize = cipher.getOutputSize(plainTextBuffer.remaining());

/*
This should be the only place we allocate memory in encryptBlob(). Although there is an
overload that can encrypt in place that would save allocations, we do not want to overwrite
customer's memory, so we must allocate our own memory. If memory usage becomes unreasonable,
we should implement pooling.
*/
ByteBuffer encryptedTextBuffer = ByteBuffer.allocate(outputSize);

int encryptedBytes;
try {
encryptedBytes = cipher.update(plainTextBuffer, encryptedTextBuffer);
} catch (ShortBufferException e) {
throw logger.logExceptionAsError(Exceptions.propagate(e));
}
encryptedTextBuffer.position(0);
encryptedTextBuffer.limit(encryptedBytes);
return encryptedTextBuffer;
});
ByteBuffer encryptedTextBuffer = ByteBuffer.allocate(outputSize);

int encryptedBytes;
try {
encryptedBytes = cipher.update(plainTextBuffer, encryptedTextBuffer);
} catch (ShortBufferException e) {
throw logger.logExceptionAsError(Exceptions.propagate(e));
}
encryptedTextBuffer.position(0);
encryptedTextBuffer.limit(encryptedBytes);
return encryptedTextBuffer;
});

/*
Defer() ensures the contained code is not executed until the Flux is subscribed to, in
other words, cipher.doFinal() will not be called until the plainTextFlux has completed
and therefore all other data has been encrypted.
*/
encryptedTextFlux = Flux.concat(encryptedTextFlux, Flux.defer(() -> {
try {
return Flux.just(ByteBuffer.wrap(cipher.doFinal()));
} catch (GeneralSecurityException e) {
throw logger.logExceptionAsError(Exceptions.propagate(e));
}
encryptedTextFlux = Flux.concat(encryptedTextFlux, Flux.defer(() -> {
try {
return Flux.just(ByteBuffer.wrap(cipher.doFinal()));
} catch (GeneralSecurityException e) {
throw logger.logExceptionAsError(Exceptions.propagate(e));
}
}));
return new EncryptedBlob(encryptionData, encryptedTextFlux);
}));
return new EncryptedBlob(encryptionData, encryptedTextFlux);
});
} catch (NoSuchAlgorithmException | NoSuchPaddingException e) {
// These are hardcoded and guaranteed to work. There is no reason to propogate a checked exception.
throw logger.logExceptionAsError(new RuntimeException(e));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import com.azure.core.test.utils.TestResourceNamer
import com.azure.core.util.Configuration
import com.azure.core.util.FluxUtil
import com.azure.core.util.logging.ClientLogger
import com.azure.security.keyvault.keys.cryptography.models.KeyWrapAlgorithm
import com.azure.storage.blob.BlobAsyncClient
import com.azure.storage.blob.BlobClient
import com.azure.storage.blob.BlobServiceClientBuilder
Expand Down Expand Up @@ -88,7 +89,7 @@ class APISpec extends Specification {
static def PREMIUM_STORAGE = "PREMIUM_STORAGE_"

TestResourceNamer resourceNamer
private InterceptorManager interceptorManager
def InterceptorManager interceptorManager
protected String testName

// Fields used for conveniently creating blobs with data.
Expand Down Expand Up @@ -229,8 +230,10 @@ class APISpec extends Specification {
AsyncKeyEncryptionKeyResolver keyResolver,
StorageSharedKeyCredential credential, String endpoint,
HttpPipelinePolicy... policies) {

KeyWrapAlgorithm algorithm = key != null && key.getKeyId().block() == "local" ? KeyWrapAlgorithm.A256KW : KeyWrapAlgorithm.RSA_OAEP_256
EncryptedBlobClientBuilder builder = new EncryptedBlobClientBuilder()
.key(key, "keyWrapAlgorithm")
.key(key, algorithm.toString())
.keyResolver(keyResolver)
.endpoint(endpoint)
.httpClient(getHttpClient())
Expand Down Expand Up @@ -277,7 +280,7 @@ class APISpec extends Specification {
generateResourceName(containerPrefix, entityNo++)
}

private String generateResourceName(String prefix, int entityNo) {
def String generateResourceName(String prefix, int entityNo) {
return resourceNamer.randomName(prefix + testName + entityNo, 63)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1201,10 +1201,8 @@ class EncyptedBlockBlobAPITest extends APISpec {
Files.deleteIfExists(file.toPath())

expect:
def bac = new EncryptedBlobClientBuilder()
.key(fakeKey, "keyWrapAlgorithm")
.pipeline(ebc.getHttpPipeline())
.endpoint(ebc.getBlobUrl())
def bac = getEncryptedClientBuilder(fakeKey, null, primaryCredential,
ebc.getBlobUrl().toString())
.buildEncryptedBlobAsyncClient()

/*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package com.azure.storage.blob.specialized.cryptography


import com.azure.core.credential.TokenCredential
import com.azure.core.cryptography.AsyncKeyEncryptionKey
import com.azure.core.http.HttpClient
import com.azure.core.http.HttpPipeline
import com.azure.core.http.HttpPipelineBuilder
import com.azure.core.http.policy.*
import com.azure.core.test.TestMode
import com.azure.core.util.Configuration
import com.azure.identity.ClientSecretCredentialBuilder
import com.azure.security.keyvault.keys.KeyClient
import com.azure.security.keyvault.keys.KeyClientBuilder
import com.azure.security.keyvault.keys.KeyServiceVersion
import com.azure.security.keyvault.keys.cryptography.KeyEncryptionKeyClientBuilder
import com.azure.security.keyvault.keys.models.CreateRsaKeyOptions
import com.azure.security.keyvault.keys.models.KeyVaultKey
import com.azure.storage.blob.BlobContainerClient
import com.azure.storage.common.implementation.Constants

import java.time.Duration
import java.time.OffsetDateTime

class KeyvaultKeyTest extends APISpec {

BlobContainerClient cc
EncryptedBlobClient bec // encrypted client for download
KeyClient keyClient
String keyId

def setup() {
def keyVaultUrl = "https://azstoragesdkvault.vault.azure.net/"
if (testMode != TestMode.PLAYBACK) {
keyVaultUrl = Configuration.getGlobalConfiguration().get("KEYVAULT_URL")
}

keyClient = new KeyClientBuilder()
.pipeline(getHttpPipeline(getHttpClient(), KeyServiceVersion.V7_0))
.httpClient(getHttpClient())
.vaultUrl(keyVaultUrl)
.buildClient()

keyId = generateResourceName("keyId", entityNo++)

KeyVaultKey keyVaultKey = keyClient.createRsaKey(new CreateRsaKeyOptions(keyId)
.setExpiresOn(OffsetDateTime.now().plusYears(1))
.setKeySize(2048))

AsyncKeyEncryptionKey akek = new KeyEncryptionKeyClientBuilder()
.pipeline(getHttpPipeline(getHttpClient(), KeyServiceVersion.V7_0))
.httpClient(getHttpClient())
.buildAsyncKeyEncryptionKey(keyVaultKey.getId())
.block()

cc = getServiceClientBuilder(primaryCredential,
String.format(defaultEndpointTemplate, primaryCredential.getAccountName()))
.buildClient()
.getBlobContainerClient(generateContainerName())
cc.create()

bec = getEncryptedClientBuilder(akek, null, primaryCredential,
cc.getBlobContainerUrl().toString())
.blobName(generateBlobName())
.buildEncryptedBlobClient()
}

def cleanup() {
keyClient.beginDeleteKey(keyId)
}

def "upload download"() {
setup:
def inputArray = getRandomByteArray(Constants.KB)
InputStream stream = new ByteArrayInputStream(inputArray)
def os = new ByteArrayOutputStream()

when:
bec.upload(stream, Constants.KB)
bec.download(os)

then:
inputArray == os.toByteArray()
}


def "encryption not a noop"() {
setup:
def inputArray = getRandomByteArray(Constants.KB)
InputStream stream = new ByteArrayInputStream(inputArray)
def os = new ByteArrayOutputStream()

when:
bec.upload(stream, Constants.KB)
cc.getBlobClient(bec.getBlobName()).download(os)

then:
inputArray != os.toByteArray()
}

HttpPipeline getHttpPipeline(HttpClient httpClient, KeyServiceVersion serviceVersion) {
TokenCredential credential = null;

if (!interceptorManager.isPlaybackMode()) {
String clientId = System.getenv("AZURE_CLIENT_ID");
String clientKey = System.getenv("AZURE_CLIENT_SECRET");
String tenantId = System.getenv("AZURE_TENANT_ID");
Objects.requireNonNull(clientId, "The client id cannot be null");
Objects.requireNonNull(clientKey, "The client key cannot be null");
Objects.requireNonNull(tenantId, "The tenant id cannot be null");
credential = new ClientSecretCredentialBuilder()
.clientSecret(clientKey)
.clientId(clientId)
.tenantId(tenantId)
.build();
}

// Closest to API goes first, closest to wire goes last.
final List<HttpPipelinePolicy> policies = new ArrayList<>();
policies.add(new UserAgentPolicy("client_name", "client_version", Configuration.getGlobalConfiguration().clone(), serviceVersion));
HttpPolicyProviders.addBeforeRetryPolicies(policies);
RetryStrategy strategy = new ExponentialBackoff(5, Duration.ofSeconds(2), Duration.ofSeconds(16));
policies.add(new RetryPolicy(strategy));
if (credential != null) {
policies.add(new BearerTokenAuthenticationPolicy(credential, "https://vault.azure.net/.default"));
}
HttpPolicyProviders.addAfterRetryPolicies(policies);
policies.add(new HttpLoggingPolicy(new HttpLogOptions().setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS)));

if (!interceptorManager.isPlaybackMode()) {
if (testMode == TestMode.RECORD) {
policies.add(interceptorManager.getRecordPolicy());
}
}

HttpPipeline pipeline = new HttpPipelineBuilder()
.policies(policies.toArray(new HttpPipelinePolicy[0]))
.httpClient(httpClient == null ? interceptorManager.getPlaybackClient() : httpClient)
.build()

return pipeline;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package com.azure.storage.blob.specialized.cryptography

import com.azure.core.cryptography.AsyncKeyEncryptionKey
import com.azure.security.keyvault.keys.cryptography.LocalKeyEncryptionKeyClientBuilder
import com.azure.security.keyvault.keys.models.JsonWebKey
import com.azure.security.keyvault.keys.models.KeyOperation
import com.azure.storage.blob.BlobContainerClient
import com.azure.storage.common.implementation.Constants

import javax.crypto.spec.SecretKeySpec

class LocalKeyTest extends APISpec {

BlobContainerClient cc
EncryptedBlobClient bec // encrypted client for download

def setup() {

/* Insecurely generate a local key*/
def byteKey = getRandomByteArray(256)

JsonWebKey localKey = JsonWebKey.fromAes(new SecretKeySpec(byteKey, "AES"),
Arrays.asList(KeyOperation.WRAP_KEY, KeyOperation.UNWRAP_KEY))
.setId("local")
AsyncKeyEncryptionKey akek = new LocalKeyEncryptionKeyClientBuilder()
.buildAsyncKeyEncryptionKey(localKey)
.block();

cc = getServiceClientBuilder(primaryCredential,
String.format(defaultEndpointTemplate, primaryCredential.getAccountName()))
.buildClient()
.getBlobContainerClient(generateContainerName())
cc.create()

bec = getEncryptedClientBuilder(akek, null, primaryCredential,
cc.getBlobContainerUrl().toString())
.blobName(generateBlobName())
.buildEncryptedBlobClient()
}

def "upload download"() {
setup:
def inputArray = getRandomByteArray(Constants.KB)
InputStream stream = new ByteArrayInputStream(inputArray)
def os = new ByteArrayOutputStream()

when:
bec.upload(stream, Constants.KB)
bec.download(os)

then:
inputArray == os.toByteArray()
}


def "encryption not a noop"() {
setup:
def inputArray = getRandomByteArray(Constants.KB)
InputStream stream = new ByteArrayInputStream(inputArray)
def os = new ByteArrayOutputStream()

when:
bec.upload(stream, Constants.KB)
cc.getBlobClient(bec.getBlobName()).download(os)

then:
inputArray != os.toByteArray()
}

}
Loading

0 comments on commit acb8f13

Please sign in to comment.