Skip to content

Commit

Permalink
YARN-10278: CapacityScheduler test framework ProportionalCapacityPree…
Browse files Browse the repository at this point in the history 
…mptionPolicyMockFramework. Contributed by Szilard Nemeth (snemeth)
  • Loading branch information
@erichadoop
erichadoop committed Dec 2, 2020
1 parent 625f85f commit 1184284
Show file tree
Hide file tree
Showing 20 changed files with 1,724 additions and 1,163 deletions.
976 changes: 0 additions & 976 deletions ...r/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
View file

This file was deleted.

67 changes: 34 additions & 33 deletions ...op/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
Expand All @@ -39,7 +40,7 @@ public class TestPreemptionForQueueWithPriorities
extends ProportionalCapacityPreemptionPolicyMockFramework {
@Before
public void setup() {
rc = new DefaultResourceCalculator();
resourceCalculator = new DefaultResourceCalculator();
super.setup();
policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock);
}
Expand Down Expand Up @@ -82,13 +83,13 @@ public void testPreemptionForHighestPriorityUnderutilizedQueue()

// 10 preempted from app1, 15 preempted from app2, and nothing preempted
// from app3
verify(mDisp, times(10)).handle(argThat(
verify(eventHandler, times(10)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, times(15)).handle(argThat(
verify(eventHandler, times(15)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
}
Expand Down Expand Up @@ -135,13 +136,13 @@ public void testPreemptionForLowestPriorityUnderutilizedQueue()

// 10 preempted from app1, 15 preempted from app2, and nothing preempted
// from app3
verify(mDisp, times(10)).handle(argThat(
verify(eventHandler, times(10)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, times(15)).handle(argThat(
verify(eventHandler, times(15)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
}
Expand Down Expand Up @@ -185,10 +186,10 @@ public void testPreemptionWontHappenBetweenSatisfiedQueues()
policy.editSchedule();

// Nothing preempted
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
}
Expand Down Expand Up @@ -236,16 +237,16 @@ public void testPreemptionForMultipleQueuesInTheSamePriorityBuckets()
// from app3/app4
// (After preemption, a has 35 - 23 = 12, b has 25 - 6 = 19, so a:b after
// preemption is 1.58, close to 1.50)
verify(mDisp, times(23)).handle(argThat(
verify(eventHandler, times(23)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, times(6)).handle(argThat(
verify(eventHandler, times(6)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
}
Expand Down Expand Up @@ -295,16 +296,16 @@ public void testPreemptionForPriorityAndDisablePreemption()
// We suppose to preempt some resource from A, but now since queueA
// disables preemption, so we need to preempt some resource from B and
// some from C even if C has higher priority than A
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, times(9)).handle(argThat(
verify(eventHandler, times(9)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
verify(mDisp, times(19)).handle(argThat(
verify(eventHandler, times(19)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
}
Expand Down Expand Up @@ -352,16 +353,16 @@ public void testPriorityPreemptionForHierarchicalOfQueues()

// Preemption should first divide capacities between a / b, and b2 should
// get less preemption than b1 (because b2 has higher priority)
verify(mDisp, times(6)).handle(argThat(
verify(eventHandler, times(6)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, times(1)).handle(argThat(
verify(eventHandler, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
verify(mDisp, times(13)).handle(argThat(
verify(eventHandler, times(13)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
verify(mDisp, times(10)).handle(argThat(
verify(eventHandler, times(10)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
}
Expand Down Expand Up @@ -408,16 +409,16 @@ public void testPriorityPreemptionWithMandatoryResourceForHierarchicalOfQueues()

// Preemption should first divide capacities between a / b, and b1 should
// get less preemption than b2 (because b1 has higher priority)
verify(mDisp, times(3)).handle(argThat(
verify(eventHandler, times(3)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
verify(mDisp, times(2)).handle(argThat(
verify(eventHandler, times(2)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
}
Expand Down Expand Up @@ -474,23 +475,23 @@ public void testPriorityPreemptionWithMultipleResource()
// get less preemption than a1 (because a2 has higher priority). More
// specifically, a2 will not get preempted since the resource preempted
// from a1 can satisfy b already.
verify(mDisp, times(7)).handle(argThat(
verify(eventHandler, times(7)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));

verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));

verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
}

@Test
public void test3ResourceTypesInterQueuePreemption() throws IOException {
rc = new DominantResourceCalculator();
when(cs.getResourceCalculator()).thenReturn(rc);
resourceCalculator = new DominantResourceCalculator();
when(cs.getResourceCalculator()).thenReturn(resourceCalculator);

// Initialize resource map
String RESOURCE_1 = "res1";
Expand Down Expand Up @@ -535,7 +536,7 @@ public void test3ResourceTypesInterQueuePreemption() throws IOException {
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
policy.editSchedule();

verify(mDisp, times(1)).handle(argThat(
verify(eventHandler, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
}
Expand Down Expand Up @@ -589,10 +590,10 @@ public void testPriorityPreemptionForBalanceBetweenSatisfiedQueues()
// b and c has same relativeAssigned=1.0f(idealAssigned / guaranteed),
// since c has higher priority, c will be put in mostUnderServedQueue and
// get all remain 30 capacity.
verify(mDisp, times(10)).handle(argThat(
verify(eventHandler, times(10)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, never()).handle(argThat(
verify(eventHandler, never()).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
}
Expand Down
71 changes: 36 additions & 35 deletions ...cemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Before;
import org.junit.Test;
Expand Down Expand Up @@ -90,13 +91,13 @@ public void testNodePartitionPreemptionRespectGuaranteedCapacity()

// 30 preempted from app1, 30 preempted from app4, and nothing preempted
// from app2/app3
verify(mDisp, times(30)).handle(
verify(eventHandler, times(30)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
verify(mDisp, times(30)).handle(
verify(eventHandler, times(30)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(4))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
}

Expand Down Expand Up @@ -146,9 +147,9 @@ public void testNodePartitionPreemptionNotHappenBetweenSatisfiedQueues()
policy.editSchedule();

// No preemption happens
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
}

Expand Down Expand Up @@ -195,9 +196,9 @@ public void testNodePartitionPreemptionOfIgnoreExclusivityAndRespectCapacity()

// 30 preempted from app1, 30 preempted from app4, and nothing preempted
// from app2/app3
verify(mDisp, times(20)).handle(
verify(eventHandler, times(20)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
verify(mDisp, times(30)).handle(
verify(eventHandler, times(30)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
}

Expand Down Expand Up @@ -247,16 +248,16 @@ public void testNodePartitionPreemptionOfSkippingAMContainer()
policy.editSchedule();

// 4 from app1
verify(mDisp, times(4)).handle(
verify(eventHandler, times(4)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
// 19 from app2-app5
verify(mDisp, times(19)).handle(
verify(eventHandler, times(19)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
verify(mDisp, times(19)).handle(
verify(eventHandler, times(19)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
verify(mDisp, times(19)).handle(
verify(eventHandler, times(19)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(4))));
verify(mDisp, times(19)).handle(
verify(eventHandler, times(19)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(5))));
}

Expand Down Expand Up @@ -308,16 +309,16 @@ public void testNodePartitionPreemptionOfAMContainer()
policy.editSchedule();

// 4 from app1
verify(mDisp, times(19)).handle(
verify(eventHandler, times(19)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
// 19 from app2-app5
verify(mDisp, times(19)).handle(
verify(eventHandler, times(19)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
verify(mDisp, times(19)).handle(
verify(eventHandler, times(19)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
verify(mDisp, times(20)).handle(
verify(eventHandler, times(20)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(4))));
verify(mDisp, times(20)).handle(
verify(eventHandler, times(20)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(5))));
}

Expand Down Expand Up @@ -370,11 +371,11 @@ public void testNodePartitionDisablePreemptionForSingleLevelQueue()
policy.editSchedule();

// 10 preempted from app1, nothing preempted from app2-app3
verify(mDisp, times(10)).handle(
verify(eventHandler, times(10)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
}

Expand Down Expand Up @@ -429,11 +430,11 @@ public void testNodePartitionNonAccessibleQueuesSharePartitionedResource()
policy.editSchedule();

// 15 will be preempted app2/app3
verify(mDisp, times(15)).handle(
verify(eventHandler, times(15)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
verify(mDisp, times(15)).handle(
verify(eventHandler, times(15)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
}

Expand Down Expand Up @@ -499,13 +500,13 @@ public void testHierarchyPreemptionForMultiplePartitions()
policy.editSchedule();

// 10 will be preempted from app1 (a1) /app4 (b2)
verify(mDisp, times(10)).handle(
verify(eventHandler, times(10)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
verify(mDisp, times(10)).handle(
verify(eventHandler, times(10)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(4))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
}

Expand Down Expand Up @@ -559,9 +560,9 @@ public void testHierarchyPreemptionForDifferenceAcessibility()
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
policy.editSchedule();

verify(mDisp, times(50)).handle(
verify(eventHandler, times(50)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
verify(mDisp, times(30)).handle(
verify(eventHandler, times(30)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
}

Expand Down Expand Up @@ -610,13 +611,13 @@ public void testNodePartitionPreemptionWithVCoreResource() throws IOException {

// 30 preempted from app1, 30 preempted from app4, and nothing preempted
// from app2/app3
verify(mDisp, times(30)).handle(
verify(eventHandler, times(30)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
verify(mDisp, times(30)).handle(
verify(eventHandler, times(30)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(4))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(2))));
verify(mDisp, never()).handle(
verify(eventHandler, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
}

Expand Down Expand Up @@ -683,7 +684,7 @@ public void testNormalizeGuaranteeWithMultipleResource() throws IOException {
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
policy.editSchedule();

verify(mDisp, times(7)).handle(
verify(eventHandler, times(7)).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));

riMap.remove(RESOURCE_1);
Expand Down
Loading

0 comments on commit 1184284

Please sign in to comment.