Skip to content

Commit

Permalink
Added k8s requests and limits for ephemeral-storage , hugepages-2Mi a…
Browse files Browse the repository at this point in the history 
…nd hugepages-1Gi (#443)

Fixes #441
  • Loading branch information
@corneil
corneil authored Mar 26, 2024
1 parent ff82f6f commit c827c35
Show file tree
Hide file tree
Showing 3 changed files with 189 additions and 36 deletions.
86 changes: 54 additions & 32 deletions .../java/org/springframework/cloud/deployer/spi/kubernetes/DeploymentPropertiesResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -147,6 +148,14 @@ List<Volume> getVolumes(Map<String, String> kubernetesDeployerProperties) {

return volumes;
}
private <T> String deducePropertyValue(Map<String, String> deployerProps, String propNameSansPrefix, Supplier<String> defaultValue) {
String propName = this.propertyPrefix + propNameSansPrefix;
String propValue = PropertyParserUtils.getDeploymentPropertyValue(deployerProps, propName);
if (!StringUtils.hasText(propValue)) {
propValue = defaultValue.get();
}
return propValue;
}

/**
* Get the resource limits for the deployment request. A Pod can define its maximum needed resources by setting the
Expand All @@ -159,33 +168,19 @@ List<Volume> getVolumes(Map<String, String> kubernetesDeployerProperties) {
* @return the resource limits to use
*/
Map<String, Quantity> deduceResourceLimits(Map<String, String> kubernetesDeployerProperties) {
String memory = PropertyParserUtils.getDeploymentPropertyValue(kubernetesDeployerProperties,
this.propertyPrefix + ".limits.memory");

if (!StringUtils.hasText(memory)) {
memory = properties.getLimits().getMemory();
}
String memory = deducePropertyValue(kubernetesDeployerProperties, ".limits.memory", () -> properties.getLimits().getMemory());

String cpu = PropertyParserUtils.getDeploymentPropertyValue(kubernetesDeployerProperties,
this.propertyPrefix + ".limits.cpu");
String cpu = deducePropertyValue(kubernetesDeployerProperties, ".limits.cpu", () -> properties.getLimits().getCpu());

if (!StringUtils.hasText(cpu)) {
cpu = properties.getLimits().getCpu();
}
String ephemeralStorage = deducePropertyValue(kubernetesDeployerProperties, ".limits.ephemeral-storage", () -> properties.getLimits().getEphemeralStorage());

String gpuVendor = PropertyParserUtils.getDeploymentPropertyValue(kubernetesDeployerProperties,
this.propertyPrefix + ".limits.gpuVendor");
String hugePages2Mi = deducePropertyValue(kubernetesDeployerProperties, ".limits.hugepages-2Mi", () -> properties.getLimits().getHugepages2Mi());

if (!StringUtils.hasText(gpuVendor)) {
gpuVendor = properties.getLimits().getGpuVendor();
}
String hugePages1Gi = deducePropertyValue(kubernetesDeployerProperties, ".limits.hugepages-1Gi", () -> properties.getLimits().getHugepages1Gi());

String gpuCount = PropertyParserUtils.getDeploymentPropertyValue(kubernetesDeployerProperties,
this.propertyPrefix + ".limits.gpuCount");
String gpuVendor = deducePropertyValue(kubernetesDeployerProperties, ".limits.gpuVendor", () -> properties.getLimits().getGpuVendor());

if (!StringUtils.hasText(gpuCount)) {
gpuCount = properties.getLimits().getGpuCount();
}
String gpuCount = deducePropertyValue(kubernetesDeployerProperties, ".limits.gpuCount", () -> properties.getLimits().getGpuCount());

Map<String,Quantity> limits = new HashMap<String,Quantity>();

Expand All @@ -197,10 +192,25 @@ Map<String, Quantity> deduceResourceLimits(Map<String, String> kubernetesDeploye
limits.put("cpu", new Quantity(cpu));
}

if(StringUtils.hasText(ephemeralStorage)) {
limits.put("ephemeral-storage", new Quantity(ephemeralStorage));
}

if(StringUtils.hasText(hugePages2Mi)) {
limits.put("hugepages-2Mi", new Quantity(hugePages2Mi));
}

if(StringUtils.hasText(hugePages1Gi)) {
limits.put("hugepages-1Gi", new Quantity(hugePages1Gi));
}

if (StringUtils.hasText(gpuVendor) && StringUtils.hasText(gpuCount)) {
limits.put(gpuVendor + "/gpu", new Quantity(gpuCount));
}

if(logger.isDebugEnabled()) {
logger.debug("limits:" + limits);
}
return limits;
}

Expand Down Expand Up @@ -240,22 +250,19 @@ ImagePullPolicy deduceImagePullPolicy(Map<String, String> kubernetesDeployerProp
* @return the resource requests to use
*/
Map<String, Quantity> deduceResourceRequests(Map<String, String> kubernetesDeployerProperties) {
String memOverride = PropertyParserUtils.getDeploymentPropertyValue(kubernetesDeployerProperties,
this.propertyPrefix + ".requests.memory");
String memOverride = deducePropertyValue(kubernetesDeployerProperties, ".requests.memory", () -> properties.getRequests().getMemory());

if (memOverride == null) {
memOverride = properties.getRequests().getMemory();
}
String cpuOverride = deducePropertyValue(kubernetesDeployerProperties, ".requests.cpu", () -> properties.getRequests().getCpu());

String ephemeralStorage = deducePropertyValue(kubernetesDeployerProperties, ".requests.ephemeral-storage", () -> properties.getLimits().getEphemeralStorage());

String cpuOverride = PropertyParserUtils.getDeploymentPropertyValue(kubernetesDeployerProperties,
this.propertyPrefix + ".requests.cpu");
String hugePages2Mi = deducePropertyValue(kubernetesDeployerProperties, ".requests.hugepages-2Mi", () -> properties.getLimits().getHugepages2Mi());

if (cpuOverride == null) {
cpuOverride = properties.getRequests().getCpu();
}
String hugePages1Gi = deducePropertyValue(kubernetesDeployerProperties, ".requests.hugepages-1Gi", () -> properties.getLimits().getHugepages1Gi());

logger.debug("Using requests - cpu: " + cpuOverride + " mem: " + memOverride);
if(logger.isDebugEnabled()) {
logger.debug("Using requests - cpu: " + cpuOverride + " mem: " + memOverride + " ephemeral-storage:" + ephemeralStorage + " hugepages-2Mi:" + hugePages2Mi + " hugepages-1Gi:" + hugePages1Gi);
}

Map<String,Quantity> requests = new HashMap<String, Quantity>();

Expand All @@ -267,6 +274,21 @@ Map<String, Quantity> deduceResourceRequests(Map<String, String> kubernetesDeplo
requests.put("cpu", new Quantity(cpuOverride));
}

if(StringUtils.hasText(ephemeralStorage)) {
requests.put("ephemeral-storage", new Quantity(ephemeralStorage));
}

if(StringUtils.hasText(hugePages2Mi)) {
requests.put("hugepages-2Mi", new Quantity(hugePages2Mi));
}

if(StringUtils.hasText(hugePages1Gi)) {
requests.put("hugepages-1Gi", new Quantity(hugePages1Gi));
}

if(logger.isDebugEnabled()) {
logger.debug("requests:" + requests);
}
return requests;
}

Expand Down
91 changes: 87 additions & 4 deletions .../java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesDeployerProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,20 @@ public static class LimitsResources {
*/
private String memory;

/**
* Container resource ephemeral-storage limit.
*/
private String ephemeralStorage;

/**
* Container resource hugepages-2Mi limit.
*/
private String hugepages2Mi;

/**
* Container resource hugepages-1Gi limit.
*/
private String hugepages1Gi;
/**
* Container GPU vendor name for limit
*/
Expand All @@ -113,9 +127,12 @@ public LimitsResources() {
* Use the default constructor and set() methods instead.
*/
@Deprecated
public LimitsResources(String cpu, String memory) {
public LimitsResources(String cpu, String memory, String ephemeralStorage, String hugepages2Mi, String hugepages1Gi) {
this.cpu = cpu;
this.memory = memory;
this.ephemeralStorage = ephemeralStorage;
this.hugepages2Mi = hugepages2Mi;
this.hugepages1Gi = hugepages1Gi;
}

public String getCpu() {
Expand All @@ -134,6 +151,30 @@ public void setMemory(String memory) {
this.memory = memory;
}

public String getEphemeralStorage() {
return ephemeralStorage;
}

public void setEphemeralStorage(String ephemeralStorage) {
this.ephemeralStorage = ephemeralStorage;
}

public String getHugepages2Mi() {
return hugepages2Mi;
}

public void setHugepages2Mi(String hugepages2Mi) {
this.hugepages2Mi = hugepages2Mi;
}

public String getHugepages1Gi() {
return hugepages1Gi;
}

public void setHugepages1Gi(String hugepages1Gi) {
this.hugepages1Gi = hugepages1Gi;
}

public String getGpuVendor() {
return gpuVendor;
}
Expand All @@ -157,21 +198,39 @@ public void setGpuCount(String gpuCount) {
public static class RequestsResources {

/**
* Container request limit.
* Container cpu request.
*/
private String cpu;

/**
* Container memory limit.
* Container memory request.
*/
private String memory;

/**
* Container resource ephemeral-storage request.
*/
private String ephemeralStorage;

/**
* Container resource hugepages-2Mi request.
*/
private String hugepages2Mi;

/**
* Container resource hugepages-1Gi request.
*/
private String hugepages1Gi;

public RequestsResources() {
}

public RequestsResources(String cpu, String memory) {
public RequestsResources(String cpu, String memory, String ephemeralStorage, String hugepages2Mi, String hugepages1Gi) {
this.cpu = cpu;
this.memory = memory;
this.ephemeralStorage = ephemeralStorage;
this.hugepages2Mi = hugepages2Mi;
this.hugepages1Gi = hugepages1Gi;
}

public String getCpu() {
Expand All @@ -189,6 +248,30 @@ public String getMemory() {
public void setMemory(String memory) {
this.memory = memory;
}

public String getEphemeralStorage() {
return ephemeralStorage;
}

public void setEphemeralStorage(String ephemeralStorage) {
this.ephemeralStorage = ephemeralStorage;
}

public String getHugepages2Mi() {
return hugepages2Mi;
}

public void setHugepages2Mi(String hugepages2Mi) {
this.hugepages2Mi = hugepages2Mi;
}

public String getHugepages1Gi() {
return hugepages1Gi;
}

public void setHugepages1Gi(String hugepages1Gi) {
this.hugepages1Gi = hugepages1Gi;
}
}

public static class StatefulSet {
Expand Down
48 changes: 48 additions & 0 deletions ...org/springframework/cloud/deployer/spi/kubernetes/RunAbstractKubernetesDeployerTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,4 +170,52 @@ public void requestMemory_deploymentProperty_usesDeploymentProperty() {
Map<String, Quantity> requests = this.deploymentPropertiesResolver.deduceResourceRequests(deploymentRequest.getDeploymentProperties());
assertThat(requests.get("memory")).isEqualTo(new Quantity("256Mi"));
}

@Test
public void requestEphemeralStorage_deploymentProperty_usesDeploymentProperty() {
kubernetesDeployerProperties.getRequests().setEphemeralStorage("2Gi");
deploymentProperties.put("spring.cloud.deployer.kubernetes.requests.ephemeral-storage", "2Gi");
Map<String, Quantity> requests = this.deploymentPropertiesResolver.deduceResourceRequests(deploymentRequest.getDeploymentProperties());
assertThat(requests.get("ephemeral-storage")).isEqualTo(new Quantity("2Gi"));
}

@Test
public void limitEphemeralStorage_deploymentProperty_usesDeploymentProperty() {
kubernetesDeployerProperties.getLimits().setEphemeralStorage("2Gi");
deploymentProperties.put("spring.cloud.deployer.kubernetes.limits.ephemeral-storage", "2Gi");
Map<String, Quantity> limits = this.deploymentPropertiesResolver.deduceResourceLimits(deploymentRequest.getDeploymentProperties());
assertThat(limits.get("ephemeral-storage")).isEqualTo(new Quantity("2Gi"));
}

@Test
public void requestHugepages1Gi_deploymentProperty_usesDeploymentProperty() {
kubernetesDeployerProperties.getRequests().setHugepages1Gi("4");
deploymentProperties.put("spring.cloud.deployer.kubernetes.requests.hugepages-1Gi", "4");
Map<String, Quantity> requests = this.deploymentPropertiesResolver.deduceResourceRequests(deploymentRequest.getDeploymentProperties());
assertThat(requests.get("hugepages-1Gi")).isEqualTo(new Quantity("4"));
}

@Test
public void limitHugepages1Gi_deploymentProperty_usesDeploymentProperty() {
kubernetesDeployerProperties.getLimits().setHugepages1Gi("4");
deploymentProperties.put("spring.cloud.deployer.kubernetes.limits.hugepages-1Gi", "4");
Map<String, Quantity> limits = this.deploymentPropertiesResolver.deduceResourceLimits(deploymentRequest.getDeploymentProperties());
assertThat(limits.get("hugepages-1Gi")).isEqualTo(new Quantity("4"));
}

@Test
public void requestHugepages2Mi_deploymentProperty_usesDeploymentProperty() {
kubernetesDeployerProperties.getRequests().setHugepages2Mi("40");
deploymentProperties.put("spring.cloud.deployer.kubernetes.requests.hugepages-2Mi", "40");
Map<String, Quantity> requests = this.deploymentPropertiesResolver.deduceResourceRequests(deploymentRequest.getDeploymentProperties());
assertThat(requests.get("hugepages-2Mi")).isEqualTo(new Quantity("40"));
}

@Test
public void limitHugepages2Mi_deploymentProperty_usesDeploymentProperty() {
kubernetesDeployerProperties.getLimits().setHugepages2Mi("40");
deploymentProperties.put("spring.cloud.deployer.kubernetes.limits.hugepages-2Mi", "40");
Map<String, Quantity> limits = this.deploymentPropertiesResolver.deduceResourceLimits(deploymentRequest.getDeploymentProperties());
assertThat(limits.get("hugepages-2Mi")).isEqualTo(new Quantity("40"));
}
}

0 comments on commit c827c35

Please sign in to comment.