Skip to content

Commit

Permalink
[Storage] Queues - move resource naming into common test, enable para…
Browse files Browse the repository at this point in the history
…llelization. (#21143)

* draft.

* fix pom.

* fix readme.

* blobs.

* batch.

* changefeed.

* crypto

* nio

* datalake

* files

* how about this?

* that isn't necessary.

* let's see.

* excess plugin.

* Queues: Move resource naming to common test. Enable queue parallelization at record and playback.

* recordings.

* fix build.

* checkstyle.

* make sure we can record other modules...

* that check isn't needed.

* pr feedback.

* avoid name collision...
  • Loading branch information
kasobol-msft authored May 4, 2021
1 parent ff88711 commit e78062b
Show file tree
Hide file tree
Showing 313 changed files with 6,758 additions and 8,068 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class APISpec extends StorageSpec {
static def BLOB_STORAGE = "BLOB_STORAGE_"
static def PREMIUM_STORAGE = "PREMIUM_STORAGE_"

TestResourceNamer resourceNamer
protected TestResourceNamer resourceNamer
def InterceptorManager interceptorManager
protected String testName

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.azure.storage.common.test.shared;

import com.azure.core.test.TestMode;
import com.azure.core.test.models.RecordedData;
import com.azure.core.test.utils.TestResourceNamer;

import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Locale;
import java.util.Objects;
import java.util.zip.CRC32;

public class StorageResourceNamer {
private final TestResourceNamer testResourceNamer;
private final String resourcePrefix;

public StorageResourceNamer(String testName, TestMode testMode, RecordedData recordedData) {
Objects.requireNonNull(testName);
Objects.requireNonNull(testMode);
resourcePrefix = getCrc32(testName);
testResourceNamer = new TestResourceNamer(resourcePrefix, testMode, recordedData);
}

public String getResourcePrefix() {
return resourcePrefix;
}

public String getRandomName(int maxLength) {
return testResourceNamer.randomName(getResourcePrefix(), maxLength);
}

public String getRandomName(String prefix, int maxLength) {
Objects.requireNonNull(prefix);
return testResourceNamer.randomName(prefix, maxLength);
}

public OffsetDateTime getUtcNow() {
return testResourceNamer.now();
}

public String getRandomUuid() {
return testResourceNamer.randomUuid();
}

private static String getCrc32(String input) {
CRC32 crc32 = new CRC32();
crc32.update(input.getBytes(StandardCharsets.UTF_8));
return String.format(Locale.US, "%08X", crc32.getValue()).toLowerCase();
}
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,76 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.azure.storage.common.test.shared;
package com.azure.storage.common.test.shared

import spock.lang.Specification;
import com.azure.core.http.HttpClient
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder
import com.azure.core.http.policy.HttpPipelinePolicy
import com.azure.core.test.InterceptorManager
import com.azure.core.test.TestMode
import spock.lang.Specification

class StorageSpec extends Specification {
protected static final TestEnvironment ENVIRONMENT = new TestEnvironment();

private String testName
private InterceptorManager interceptorManager
private StorageResourceNamer namer

def setup() {
testName = getTestName()
if (shouldUseThisToRecord()) {
interceptorManager = new InterceptorManager(testName, ENVIRONMENT.testMode)
namer = new StorageResourceNamer(testName, ENVIRONMENT.testMode, interceptorManager.getRecordedData())
}
System.out.printf("========================= %s =========================%n", testName)
}

def cleanup() {
if (shouldUseThisToRecord()) {
interceptorManager.close()
}
}

// TODO (kasobol-msft) Remove this when all modules are migrated.
protected shouldUseThisToRecord() {
return false
}

protected StorageResourceNamer getNamer() {
Objects.requireNonNull(namer, "namer has not been initialized yet")
return namer
}

protected HttpPipelinePolicy getRecordPolicy() {
if (ENVIRONMENT.testMode == TestMode.RECORD) {
return interceptorManager.getRecordPolicy()
} else {
return { context, next -> return next.process() }
}
}

protected HttpClient getHttpClient() {
if (ENVIRONMENT.testMode != TestMode.PLAYBACK) {
NettyAsyncHttpClientBuilder builder = new NettyAsyncHttpClientBuilder()
return builder.build()
} else {
return interceptorManager.getPlaybackClient()
}
}

private String getTestName() {
def iterationInfo = specificationContext.currentIteration
def featureInfo = iterationInfo.getParent()
def specInfo = featureInfo.getParent()
def fullName = specInfo.getName() + featureInfo.getName().split(" ").collect { it.capitalize() }.join("")

if (iterationInfo.getDataValues().length == 0) {
return fullName
}
def prefix = fullName
def suffix = "[" + iterationInfo.getIterationIndex() + "]"

return prefix + suffix
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,14 @@

package com.azure.storage.queue

import com.azure.core.http.HttpClient
import com.azure.core.http.HttpPipelineCallContext
import com.azure.core.http.HttpPipelineNextPolicy
import com.azure.core.http.HttpPipelinePosition
import com.azure.core.http.HttpResponse
import com.azure.core.http.ProxyOptions
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder
import com.azure.core.http.policy.HttpLogDetailLevel
import com.azure.core.http.policy.HttpLogOptions
import com.azure.core.http.policy.HttpPipelinePolicy
import com.azure.core.test.InterceptorManager
import com.azure.core.test.TestMode
import com.azure.core.test.utils.TestResourceNamer
import com.azure.core.util.Configuration
import com.azure.core.util.Context
import com.azure.core.util.logging.ClientLogger
Expand All @@ -24,7 +19,6 @@ import com.azure.storage.common.policy.RequestRetryOptions
import com.azure.storage.common.policy.RetryPolicyType
import com.azure.storage.common.test.shared.StorageSpec
import com.azure.storage.queue.models.QueuesSegmentOptions
import org.spockframework.runtime.model.IterationInfo
import reactor.core.publisher.Mono

import java.time.Duration
Expand All @@ -33,8 +27,6 @@ import java.time.OffsetDateTime
class APISpec extends StorageSpec {
// Field common used for all APIs.
def logger = new ClientLogger(APISpec.class)
InterceptorManager interceptorManager
TestResourceNamer testResourceName

// Clients for API tests
QueueServiceClient primaryQueueServiceClient
Expand All @@ -44,48 +36,43 @@ class APISpec extends StorageSpec {
protected static StorageSharedKeyCredential primaryCredential

// Test name for test method name.
String methodName
String connectionString

/**
* Setup the QueueServiceClient and QueueClient common used for the API tests.
*/
def setup() {
primaryCredential = getCredential(PRIMARY_STORAGE)
String testName = getFullTestName(specificationContext.currentIteration)
String className = specificationContext.getCurrentSpec().getName()
methodName = className + testName
logger.info("Test Mode: {}, Name: {}", ENVIRONMENT.testMode, methodName)
interceptorManager = new InterceptorManager(methodName, ENVIRONMENT.testMode)
testResourceName = new TestResourceNamer(methodName, ENVIRONMENT.testMode, interceptorManager.getRecordedData())
if (ENVIRONMENT.testMode != TestMode.PLAYBACK) {
connectionString = Configuration.getGlobalConfiguration().get("AZURE_STORAGE_QUEUE_CONNECTION_STRING")
} else {
connectionString = "DefaultEndpointsProtocol=https;AccountName=teststorage;AccountKey=atestaccountkey;" +
"EndpointSuffix=core.windows.net"
}

// Print out the test name to create breadcrumbs in our test logging in case anything hangs.
System.out.printf("========================= %s.%s =========================%n", className, testName)
}

/**
* Clean up the test queues and messages for the account.
*/
def cleanup() {
interceptorManager.close()
if (ENVIRONMENT.testMode != TestMode.PLAYBACK) {
def cleanupQueueServiceClient = new QueueServiceClientBuilder()
.retryOptions(new RequestRetryOptions(RetryPolicyType.FIXED, 3, 60, 1000, 1000, null))
.connectionString(connectionString)
.buildClient()
cleanupQueueServiceClient.listQueues(new QueuesSegmentOptions().setPrefix(methodName.toLowerCase()),
cleanupQueueServiceClient.listQueues(new QueuesSegmentOptions().setPrefix(namer.getResourcePrefix()),
null, Context.NONE).each {
queueItem -> cleanupQueueServiceClient.deleteQueue(queueItem.getName())
}
}
}

// TODO (kasobol-msft) remove this when all modules are migrated
@Override
protected shouldUseThisToRecord() {
return true
}

private StorageSharedKeyCredential getCredential(String accountType) {
String accountName
String accountKey
Expand All @@ -106,25 +93,21 @@ class APISpec extends StorageSpec {
return new StorageSharedKeyCredential(accountName, accountKey)
}

def queueServiceBuilderHelper(final InterceptorManager interceptorManager) {
def queueServiceBuilderHelper() {
QueueServiceClientBuilder builder = new QueueServiceClientBuilder()
if (ENVIRONMENT.testMode == TestMode.RECORD) {
builder.addPolicy(interceptorManager.getRecordPolicy())
}
return builder
.connectionString(connectionString)
.addPolicy(getRecordPolicy())
.httpClient(getHttpClient())
}

def queueBuilderHelper(final InterceptorManager interceptorManager) {
def queueName = testResourceName.randomName("queue", 16)
def queueBuilderHelper() {
def queueName = namer.getRandomName(60)
QueueClientBuilder builder = new QueueClientBuilder()
if (ENVIRONMENT.testMode == TestMode.RECORD) {
builder.addPolicy(interceptorManager.getRecordPolicy())
}
return builder
.connectionString(connectionString)
.queueName(queueName)
.addPolicy(getRecordPolicy())
.httpClient(getHttpClient())
}

Expand All @@ -139,9 +122,7 @@ class APISpec extends StorageSpec {
builder.addPolicy(policy)
}

if (ENVIRONMENT.testMode == TestMode.RECORD) {
builder.addPolicy(interceptorManager.getRecordPolicy())
}
builder.addPolicy(getRecordPolicy())

if (credential != null) {
builder.credential(credential)
Expand All @@ -154,47 +135,11 @@ class APISpec extends StorageSpec {
QueueClientBuilder builder = new QueueClientBuilder()
.endpoint(endpoint)
.httpClient(getHttpClient())
.addPolicy(getRecordPolicy())
.httpLogOptions(new HttpLogOptions().setLogLevel(HttpLogDetailLevel.BODY_AND_HEADERS))

if (ENVIRONMENT.testMode == TestMode.RECORD) {
builder.addPolicy(interceptorManager.getRecordPolicy())
}

return builder
}

private def getFullTestName(IterationInfo iterationInfo) {
def fullName = iterationInfo.getParent().getName().split(" ").collect { it.capitalize() }.join("")

if (iterationInfo.getDataValues().length == 0) {
return fullName
}
def prefix = fullName
def suffix = iterationInfo.getIterationIndex()

return prefix + suffix
}


OffsetDateTime getUTCNow() {
return testResourceName.now()
}

HttpClient getHttpClient() {
NettyAsyncHttpClientBuilder builder = new NettyAsyncHttpClientBuilder()
if (ENVIRONMENT.testMode != TestMode.PLAYBACK) {
builder.wiretap(true)

if (Boolean.parseBoolean(Configuration.getGlobalConfiguration().get("AZURE_TEST_DEBUGGING"))) {
builder.proxy(new ProxyOptions(ProxyOptions.Type.HTTP, new InetSocketAddress("localhost", 8888)))
}

return builder.build()
} else {
return interceptorManager.getPlaybackClient()
}
}

def sleepIfLive(long milliseconds) {
if (ENVIRONMENT.testMode == TestMode.PLAYBACK) {
return
Expand All @@ -203,10 +148,6 @@ class APISpec extends StorageSpec {
sleep(milliseconds)
}

boolean liveMode() {
return ENVIRONMENT.testMode == TestMode.RECORD
}

def getMessageUpdateDelay(long liveTestDurationInMillis) {
return (ENVIRONMENT.testMode == TestMode.PLAYBACK) ? Duration.ofMillis(10) : Duration.ofMillis(liveTestDurationInMillis)
}
Expand Down
Loading

0 comments on commit e78062b

Please sign in to comment.