Skip to content

fix double counting when AWs dont have any pods running #415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Jun 28, 2023
Merged

fix double counting when AWs dont have any pods running #415

Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
714374e
fix double counting when AWs dont have any pods running
asm582 Jun 16, 2023
c81e05b
use proposed preemptions to dispatch
asm582 Jun 19, 2023
ff6c59a
perform histrogram checks prior to preemption or pending res eval
asm582 Jun 20, 2023
6f351aa
remove histograms
asm582 Jun 21, 2023
afe6397
negate nodechecks histogram
asm582 Jun 21, 2023
2017cfc
fix double counting for pending AWs
asm582 Jun 23, 2023
66a21c2
remove noisy logging
asm582 Jun 25, 2023
09c3852
print optimistic dispatch msg when preempt res acquired
asm582 Jun 25, 2023
0040e4c
release quota on completed AWs only when QM is enabled
asm582 Jun 25, 2023
adbe374
update CRDs
asm582 Jun 26, 2023
abbfe88
add all AW resources for status dispatched
asm582 Jun 27, 2023
93bb22a
add label to test dep
asm582 Jun 27, 2023
557f378
rollback test case change
asm582 Jun 27, 2023
d1c2352
reserve all res during dispatch
asm582 Jun 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix double counting for pending AWs
  • Loading branch information
@asm582
asm582 committed Jun 23, 2023
commit 2017cfc9a7594045de970708ec9632160fbfb4fa
43 changes: 19 additions & 24 deletions pkg/controller/queuejob/queuejob_controller_ex.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -869,6 +869,17 @@ func (qjm *XController) getDispatchedAppWrappers() (map[string]*clusterstateapi.
return awrRetVal, awsRetVal
}

func (qjm *XController) addTotalSnapshotResourcesConsumedByAw(totalgpu int64, totalcpu float64, totalmemory float64) *clusterstateapi.Resource {

totalResource := clusterstateapi.EmptyResource()
totalResource.GPU = totalgpu
totalResource.MilliCPU = totalcpu
totalResource.Memory = totalmemory

return totalResource

}

func (qjm *XController) getAggregatedAvailableResourcesPriority(unallocatedClusterResources *clusterstateapi.
Resource, targetpr float64, requestingJob *arbv1.AppWrapper, agentId string) (*clusterstateapi.Resource, []*arbv1.AppWrapper) {
r := unallocatedClusterResources.Clone()
Expand Down Expand Up @@ -915,10 +926,7 @@ func (qjm *XController) getAggregatedAvailableResourcesPriority(unallocatedClust
klog.V(10).Infof("[getAggAvaiResPri] %s: Added %s to candidate preemptable job with priority %f.", time.Now().String(), value.Name, value.Status.SystemPriority)
}

totalResource := clusterstateapi.EmptyResource()
totalResource.GPU = value.Status.TotalGPU
totalResource.MilliCPU = value.Status.TotalCPU
totalResource.Memory = value.Status.TotalMemory
totalResource := qjm.addTotalSnapshotResourcesConsumedByAw(value.Status.TotalGPU, value.Status.TotalCPU, value.Status.TotalMemory)
preemptable = preemptable.Add(totalResource)

continue
Expand All @@ -931,30 +939,17 @@ func (qjm *XController) getAggregatedAvailableResourcesPriority(unallocatedClust
continue
} else if value.Status.State == arbv1.AppWrapperStateEnqueued {
// Don't count the resources that can run but not yet realized (job orchestration pending or partially running).
for _, resctrl := range qjm.qjobResControls {
qjv := resctrl.GetAggregatedResources(value)
pending = pending.Add(qjv)
klog.V(10).Infof("[getAggAvaiResPri] Subtract all resources %+v in resctrlType=%T for job %s which can-run is set to: %v but state is still pending.", qjv, resctrl, value.Name, value.Status.CanRun)
}
for _, genericItem := range value.Spec.AggrResources.GenericItems {
qjv, _ := genericresource.GetResources(&genericItem)
pending = pending.Add(qjv)
klog.V(10).Infof("[getAggAvaiResPri] Subtract all resources %+v in resctrlType=%T for job %s which can-run is set to: %v but state is still pending.", qjv, genericItem, value.Name, value.Status.CanRun)
}

totalResource := qjm.addTotalSnapshotResourcesConsumedByAw(value.Status.TotalGPU, value.Status.TotalCPU, value.Status.TotalMemory)
pending = pending.Add(totalResource)

continue
} else if value.Status.State == arbv1.AppWrapperStateActive {
if value.Status.Pending > 0 {
//Don't count partially running jobs with pods still pending.
for _, resctrl := range qjm.qjobResControls {
qjv := resctrl.GetAggregatedResources(value)
pending = pending.Add(qjv)
klog.V(10).Infof("[getAggAvaiResPri] Subtract all resources %+v in resctrlType=%T for job %s which can-run is set to: %v and status set to: %s but %v pod(s) are pending.", qjv, resctrl, value.Name, value.Status.CanRun, value.Status.State, value.Status.Pending)
}
for _, genericItem := range value.Spec.AggrResources.GenericItems {
qjv, _ := genericresource.GetResources(&genericItem)
pending = pending.Add(qjv)
klog.V(10).Infof("[getAggAvaiResPri] Subtract all resources %+v in resctrlType=%T for job %s which can-run is set to: %v and status set to: %s but %v pod(s) are pending.", qjv, genericItem, value.Name, value.Status.CanRun, value.Status.State, value.Status.Pending)
}
totalResource := qjm.addTotalSnapshotResourcesConsumedByAw(value.Status.TotalGPU, value.Status.TotalCPU, value.Status.TotalMemory)
pending = pending.Add(totalResource)

} else {
// TODO: Hack to handle race condition when Running jobs have not yet updated the pod counts (In-Flight AW Jobs)
// This hack uses the golang struct implied behavior of defining the object without a value. In this case
Expand Down