Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add support for setting throughput on database creation #24456

Merged
merged 5 commits into from
Oct 4, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package com.azure.spring.data.cosmos.core;

import com.azure.cosmos.CosmosAsyncClient;
import com.azure.cosmos.CosmosAsyncDatabase;
import com.azure.cosmos.CosmosClientBuilder;
import com.azure.cosmos.CosmosException;
import com.azure.cosmos.implementation.ConflictException;
Expand Down Expand Up @@ -117,18 +118,9 @@ public class CosmosTemplateIT {
public void setUp() throws ClassNotFoundException {
if (cosmosTemplate == null) {
client = CosmosFactory.createCosmosAsyncClient(cosmosClientBuilder);
final CosmosFactory cosmosFactory = new CosmosFactory(client, TestConstants.DB_NAME);

final CosmosMappingContext mappingContext = new CosmosMappingContext();
personInfo = new CosmosEntityInformation<>(Person.class);
containerName = personInfo.getContainerName();

mappingContext.setInitialEntitySet(new EntityScanner(this.applicationContext).scan(Persistent.class));

final MappingCosmosConverter cosmosConverter = new MappingCosmosConverter(mappingContext,
null);

cosmosTemplate = new CosmosTemplate(cosmosFactory, cosmosConfig, cosmosConverter);
cosmosTemplate = createCosmosTemplate(cosmosConfig, TestConstants.DB_NAME);
}

collectionManager.ensureContainersCreatedAndEmpty(cosmosTemplate, Person.class,
Expand All @@ -137,6 +129,14 @@ public void setUp() throws ClassNotFoundException {
new PartitionKey(TEST_PERSON.getLastName()));
}

private CosmosTemplate createCosmosTemplate(CosmosConfig config, String dbName) throws ClassNotFoundException {
final CosmosFactory cosmosFactory = new CosmosFactory(client, dbName);
final CosmosMappingContext mappingContext = new CosmosMappingContext();
mappingContext.setInitialEntitySet(new EntityScanner(this.applicationContext).scan(Persistent.class));
final MappingCosmosConverter cosmosConverter = new MappingCosmosConverter(mappingContext, null);
return new CosmosTemplate(cosmosFactory, config, cosmosConverter);
}

private void insertPerson(Person person) {
cosmosTemplate.insert(person,
new PartitionKey(personInfo.getPartitionKeyFieldValue(person)));
Expand Down Expand Up @@ -660,4 +660,34 @@ public void createWithAutoscale() throws ClassNotFoundException {
assertEquals(Integer.parseInt(TestConstants.AUTOSCALE_MAX_THROUGHPUT),
throughput.getProperties().getAutoscaleMaxThroughput());
}

@Test
public void createDatabaseWithThroughput() throws ClassNotFoundException {
final String configuredThroughputDbName = TestConstants.DB_NAME + "-configured-throughput";
deleteDatabaseIfExists(configuredThroughputDbName);

Integer expectedRequestUnits = 700;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why 700 here ?
Where are we setting this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No particular reason for 700, the test is simply intended to verify that if the database throughput is provided, the database is created with that specific throughput. When defining the database throughput, some value for the request units needs to be provided - i just pulled 700 out of the air but it can be whatever. If you'd prefer another value, let me know and I can put that in.

Copy link
Contributor Author

@Blackbaud-MikeLueders Blackbaud-MikeLueders Sep 30, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in answer to your question, it's used just below, when the CosmosConfig is created...

final CosmosConfig config = CosmosConfig.builder()
            .enableDatabaseThroughput(false, expectedRequestUnits)
            .build();

and then again when comparing the value of the created database actual throughput...

assertEquals(expectedRequestUnits, response.getProperties().getManualThroughput());

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to change the value, I was just curious about 700 :)
The test looks good to me.

final CosmosConfig config = CosmosConfig.builder()
.enableDatabaseThroughput(false, expectedRequestUnits)
.build();
final CosmosTemplate configuredThroughputCosmosTemplate = createCosmosTemplate(config, configuredThroughputDbName);

final CosmosEntityInformation<Person, String> personInfo =
new CosmosEntityInformation<>(Person.class);
configuredThroughputCosmosTemplate.createContainerIfNotExists(personInfo);

final CosmosAsyncDatabase database = client.getDatabase(configuredThroughputDbName);
final ThroughputResponse response = database.readThroughput().block();
assertEquals(expectedRequestUnits, response.getProperties().getManualThroughput());
}

private void deleteDatabaseIfExists(String dbName) {
CosmosAsyncDatabase database = client.getDatabase(dbName);
try {
database.delete().block();
} catch (CosmosException ex) {
assertEquals(ex.getStatusCode(), 404);
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import com.azure.core.credential.AzureKeyCredential;
import com.azure.cosmos.CosmosAsyncClient;
import com.azure.cosmos.CosmosAsyncDatabase;
import com.azure.cosmos.CosmosClientBuilder;
import com.azure.cosmos.CosmosException;
import com.azure.cosmos.implementation.ConflictException;
Expand Down Expand Up @@ -119,17 +120,9 @@ public void setUp() throws ClassNotFoundException {
azureKeyCredential = new AzureKeyCredential(cosmosDbKey);
cosmosClientBuilder.credential(azureKeyCredential);
client = CosmosFactory.createCosmosAsyncClient(cosmosClientBuilder);
final CosmosFactory dbFactory = new CosmosFactory(client, TestConstants.DB_NAME);

final CosmosMappingContext mappingContext = new CosmosMappingContext();
personInfo = new CosmosEntityInformation<>(Person.class);
containerName = personInfo.getContainerName();

mappingContext.setInitialEntitySet(new EntityScanner(this.applicationContext).scan(Persistent.class));

final MappingCosmosConverter dbConverter =
new MappingCosmosConverter(mappingContext, null);
cosmosTemplate = new ReactiveCosmosTemplate(dbFactory, cosmosConfig, dbConverter);
cosmosTemplate = createReactiveCosmosTemplate(cosmosConfig, TestConstants.DB_NAME);
}

collectionManager.ensureContainersCreatedAndEmpty(cosmosTemplate, Person.class, GenIdEntity.class, AuditableEntity.class);
Expand All @@ -138,6 +131,14 @@ public void setUp() throws ClassNotFoundException {
new PartitionKey(personInfo.getPartitionKeyFieldValue(TEST_PERSON))).block();
}

private ReactiveCosmosTemplate createReactiveCosmosTemplate(CosmosConfig config, String dbName) throws ClassNotFoundException {
final CosmosFactory cosmosFactory = new CosmosFactory(client, dbName);
final CosmosMappingContext mappingContext = new CosmosMappingContext();
mappingContext.setInitialEntitySet(new EntityScanner(this.applicationContext).scan(Persistent.class));
final MappingCosmosConverter cosmosConverter = new MappingCosmosConverter(mappingContext, null);
return new ReactiveCosmosTemplate(cosmosFactory, config, cosmosConverter);
}

@After
public void cleanup() {
// Reset master key
Expand Down Expand Up @@ -545,4 +546,34 @@ public void createWithAutoscale() {
assertEquals(Integer.parseInt(TestConstants.AUTOSCALE_MAX_THROUGHPUT),
throughput.getProperties().getAutoscaleMaxThroughput());
}

@Test
public void createDatabaseWithThroughput() throws ClassNotFoundException {
final String configuredThroughputDbName = TestConstants.DB_NAME + "-other";
deleteDatabaseIfExists(configuredThroughputDbName);

Integer expectedRequestUnits = 700;
final CosmosConfig config = CosmosConfig.builder()
.enableDatabaseThroughput(false, expectedRequestUnits)
.build();
final ReactiveCosmosTemplate configuredThroughputCosmosTemplate = createReactiveCosmosTemplate(config, configuredThroughputDbName);

final CosmosEntityInformation<Person, String> personInfo =
new CosmosEntityInformation<>(Person.class);
configuredThroughputCosmosTemplate.createContainerIfNotExists(personInfo).block();

final CosmosAsyncDatabase database = client.getDatabase(configuredThroughputDbName);
final ThroughputResponse response = database.readThroughput().block();
assertEquals(expectedRequestUnits, response.getProperties().getManualThroughput());
}

private void deleteDatabaseIfExists(String dbName) {
CosmosAsyncDatabase database = client.getDatabase(dbName);
try {
database.delete().block();
} catch (CosmosException ex) {
assertEquals(ex.getStatusCode(), 404);
}
}

}
18 changes: 18 additions & 0 deletions sdk/cosmos/azure-spring-data-cosmos/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,24 @@ public CosmosConfig cosmosConfig() {
By default, `@EnableCosmosRepositories` will scan the current package for any interfaces that extend one of Spring Data's repository interfaces.
Use it to annotate your Configuration class to scan a different root package by `@EnableCosmosRepositories(basePackageClass=UserRepository.class)` if your project layout has multiple projects.

#### Using database provisioned throughput

Cosmos supports both [container](https://docs.microsoft.com/azure/cosmos-db/sql/how-to-provision-container-throughput)
and [database](https://docs.microsoft.com/azure/cosmos-db/sql/how-to-provision-database-throughput) provisioned
throughput. By default, spring-data-cosmos will provision throughput for each container created. If you prefer
to share throughput between containers, you can enable database provisioned throughput via CosmosConfig.

```java
@Override
public CosmosConfig cosmosConfig() {
int autoscale = false;
int initialRequestUnits = 400;
return CosmosConfig.builder()
.enableDatabaseThroughput(autoscale, initialRequestUnits)
.build();
}
```

### Define an entity
- Define a simple entity as item in Azure Cosmos DB.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public class CosmosConfig {

private final ResponseDiagnosticsProcessor responseDiagnosticsProcessor;

private final DatabaseThroughputConfig databaseThroughputConfig;

private final boolean queryMetricsEnabled;

/**
Expand All @@ -24,7 +26,22 @@ public class CosmosConfig {
@ConstructorProperties({"responseDiagnosticsProcessor", "queryMetricsEnabled"})
public CosmosConfig(ResponseDiagnosticsProcessor responseDiagnosticsProcessor,
boolean queryMetricsEnabled) {
this(responseDiagnosticsProcessor, null, queryMetricsEnabled);
}

/**
* Initialization
*
* @param responseDiagnosticsProcessor must not be {@literal null}
* @param databaseThroughputConfig may be @{literal null}
* @param queryMetricsEnabled must not be {@literal null}
*/
@ConstructorProperties({"responseDiagnosticsProcessor", "databaseThroughputConfig", "queryMetricsEnabled"})
public CosmosConfig(ResponseDiagnosticsProcessor responseDiagnosticsProcessor,
DatabaseThroughputConfig databaseThroughputConfig,
boolean queryMetricsEnabled) {
this.responseDiagnosticsProcessor = responseDiagnosticsProcessor;
this.databaseThroughputConfig = databaseThroughputConfig;
this.queryMetricsEnabled = queryMetricsEnabled;
}

Expand All @@ -46,6 +63,15 @@ public boolean isQueryMetricsEnabled() {
return queryMetricsEnabled;
}

/**
* Gets the database throughput configuration.
*
* @return DatabaseThroughputConfig, or null if no database throughput is configured
*/
public DatabaseThroughputConfig getDatabaseThroughputConfig() {
return databaseThroughputConfig;
}

/**
* Create a CosmosConfigBuilder instance
*
Expand All @@ -60,6 +86,7 @@ public static CosmosConfigBuilder builder() {
*/
public static class CosmosConfigBuilder {
private ResponseDiagnosticsProcessor responseDiagnosticsProcessor;
private DatabaseThroughputConfig databaseThroughputConfig;
private boolean queryMetricsEnabled;
CosmosConfigBuilder() {
}
Expand Down Expand Up @@ -88,19 +115,25 @@ public CosmosConfigBuilder enableQueryMetrics(boolean queryMetricsEnabled) {
return this;
}

public CosmosConfigBuilder enableDatabaseThroughput(boolean autoscale, int requestUnits) {
this.databaseThroughputConfig = new DatabaseThroughputConfig(autoscale, requestUnits);
return this;
}

/**
* Build a CosmosConfig instance
*
* @return CosmosConfig
*/
public CosmosConfig build() {
return new CosmosConfig(this.responseDiagnosticsProcessor, this.queryMetricsEnabled);
return new CosmosConfig(this.responseDiagnosticsProcessor, this.databaseThroughputConfig, this.queryMetricsEnabled);
}

@Override
public String toString() {
return "CosmosConfigBuilder{"
+ "responseDiagnosticsProcessor=" + responseDiagnosticsProcessor
+ ", databaseThroughputConfig=" + databaseThroughputConfig
+ ", queryMetricsEnabled=" + queryMetricsEnabled
+ '}';
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
package com.azure.spring.data.cosmos.config;

/**
* Throughput config for database creation
*/
public class DatabaseThroughputConfig {

private final boolean autoScale;
private final int requestUnits;

public DatabaseThroughputConfig(boolean autoScale, int requestUnits) {
this.autoScale = autoScale;
this.requestUnits = requestUnits;
}

public boolean isAutoScale() {
return autoScale;
}

public int getRequestUnits() {
return requestUnits;
}

@Override
public String toString() {
return "DatabaseThroughputConfig{"
+ "autoScale=" + autoScale
+ ", requestUnits=" + requestUnits
+ '}';
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import com.azure.cosmos.CosmosAsyncDatabase;
import com.azure.cosmos.models.CosmosContainerProperties;
import com.azure.cosmos.models.CosmosContainerResponse;
import com.azure.cosmos.models.CosmosDatabaseResponse;
import com.azure.cosmos.models.CosmosItemRequestOptions;
import com.azure.cosmos.models.CosmosItemResponse;
import com.azure.cosmos.models.CosmosQueryRequestOptions;
Expand All @@ -20,6 +21,7 @@
import com.azure.spring.data.cosmos.CosmosFactory;
import com.azure.spring.data.cosmos.common.CosmosUtils;
import com.azure.spring.data.cosmos.config.CosmosConfig;
import com.azure.spring.data.cosmos.config.DatabaseThroughputConfig;
import com.azure.spring.data.cosmos.core.convert.MappingCosmosConverter;
import com.azure.spring.data.cosmos.core.generator.CountQueryGenerator;
import com.azure.spring.data.cosmos.core.generator.FindQuerySpecGenerator;
Expand Down Expand Up @@ -75,6 +77,7 @@ public class CosmosTemplate implements CosmosOperations, ApplicationContextAware
private final ResponseDiagnosticsProcessor responseDiagnosticsProcessor;
private final boolean queryMetricsEnabled;
private final CosmosAsyncClient cosmosAsyncClient;
private final DatabaseThroughputConfig databaseThroughputConfig;

private ApplicationContext applicationContext;

Expand Down Expand Up @@ -126,6 +129,7 @@ public CosmosTemplate(CosmosFactory cosmosFactory,
this.databaseName = cosmosFactory.getDatabaseName();
this.responseDiagnosticsProcessor = cosmosConfig.getResponseDiagnosticsProcessor();
this.queryMetricsEnabled = cosmosConfig.isQueryMetricsEnabled();
this.databaseThroughputConfig = cosmosConfig.getDatabaseThroughputConfig();
}

/**
Expand Down Expand Up @@ -458,8 +462,7 @@ public String getContainerName(Class<?> domainType) {
@Override
public CosmosContainerProperties createContainerIfNotExists(CosmosEntityInformation<?, ?> information) {

final CosmosContainerResponse response = cosmosAsyncClient
.createDatabaseIfNotExists(this.databaseName)
final CosmosContainerResponse response = createDatabaseIfNotExists()
.publishOn(Schedulers.parallel())
.onErrorResume(throwable ->
CosmosExceptionUtils.exceptionHandler("Failed to create database", throwable))
Expand Down Expand Up @@ -501,6 +504,19 @@ public CosmosContainerProperties createContainerIfNotExists(CosmosEntityInformat
return response.getProperties();
}

private Mono<CosmosDatabaseResponse> createDatabaseIfNotExists() {
if (databaseThroughputConfig == null) {
return cosmosAsyncClient
.createDatabaseIfNotExists(this.databaseName);
} else {
ThroughputProperties throughputProperties = databaseThroughputConfig.isAutoScale()
? ThroughputProperties.createAutoscaledThroughput(databaseThroughputConfig.getRequestUnits())
: ThroughputProperties.createManualThroughput(databaseThroughputConfig.getRequestUnits());
return cosmosAsyncClient
.createDatabaseIfNotExists(this.databaseName, throughputProperties);
}
}

@Override
public CosmosContainerProperties getContainerProperties(String containerName) {
final CosmosContainerResponse response = cosmosAsyncClient.getDatabase(this.databaseName)
Expand Down
Loading