Merge pull request #57 from rookie0080/dev/improve-job-level-affinity…

…-performance

improve job level affinity

pkg/framework/api/common_state.go

@@ -38,6 +38,7 @@ const (
 	MatchedPDBIndexesKey      = "MatchedPDBIndexes"
 	VictimCountOfDeployKey    = "VictimCountOfDeployKey"
 	IndexOfPDBKey             = "IndexOfPDBKey"
+	EverScheduledKey          = "EverScheduledKey"
 
 	// Error Message
 	NodePartitionTypeMissedErrorString = "failed to get NodePartitionType, supposed to be set in cycle state"
@@ -322,3 +323,20 @@ func GetNotScheduledPodKeysCountByTemplate(unitState *CycleState, ownerName stri
 	}
 	return 0, fmt.Errorf(MissedError, NotScheduledPodKeysByTemplateKey)
 }
+
+func SetEverScheduledState(everScheduled bool, state *CycleState) {
+	data := &stateData{data: everScheduled}
+	state.Write(EverScheduledKey, data)
+}
+
+func GetEverScheduledState(state *CycleState) (bool, error) {
+	if state == nil {
+		return false, fmt.Errorf("nil cycle state")
+	}
+	if data, err := state.Read(EverScheduledKey); err == nil {
+		if s, ok := data.(*stateData); ok {
+			return s.data.(bool), nil
+		}
+	}
+	return false, fmt.Errorf("everScheduled state not found")
+}

pkg/framework/api/fake/listers.go

@@ -147,7 +147,7 @@ func (pvcs PersistentVolumeClaimLister) PersistentVolumeClaims(namespace string)
 	}
 }
 
-// NodeInfoLister declares a framework.NodeInfo type for testing.
+// NodeInfoLister declares a []framework.NodeInfo type for testing.
 type NodeInfoLister []framework.NodeInfo
 
 // Get returns a fake node object in the fake nodes.
@@ -187,6 +187,10 @@ func (nodes NodeInfoLister) HavePodsWithRequiredAntiAffinityList() []framework.N
 	return nodes
 }
 
+func (nodes NodeInfoLister) Len() int {
+	return len(nodes)
+}
+
 // NewNodeInfoLister create a new fake NodeInfoLister from a slice of v1.Nodes.
 func NewNodeInfoLister(nodes []*v1.Node) framework.NodeInfoLister {
 	nodeInfoList := make([]framework.NodeInfo, 0, len(nodes))

pkg/framework/api/interface.go

@@ -790,7 +790,7 @@ type ScheduleUnit interface {
 	GetAffinityNodeSelector() (*v1.NodeSelector, error)
 	// GetSortRulesForAffinity return the rules that indicate how the nodeGroups are sorted.
 	// The rule's index in slice is the sort sequence.
-	GetSortRulesForAffinity() ([]SortRule, error)
+	GetSortRulesForAffinity() []SortRule
 	// IsDebugModeOn checks whether the debug mode is set to on
 	IsDebugModeOn() bool
 	// SetEnqueuedTimeStamp set the timestamp when the unit is added to pending queue

pkg/framework/api/listers.go

@@ -18,28 +18,29 @@ package api
 
 // NodeInfoLister interface represents anything that can list/get NodeInfo objects from node name.
 type NodeInfoLister interface {
+	ClusterNodeInfoGetter
 	ClusterNodeInfoLister
-	// InPartitionList Returns the list of NodeInfos in the partition of the scheduler
-	InPartitionList() []NodeInfo
-	// OutOfPartitionList Returns the list of NodeInfos out of the partition of the scheduler
-	OutOfPartitionList() []NodeInfo
 }
 
 // SharedLister groups scheduler-specific listers.
 type SharedLister interface {
 	NodeInfos() NodeInfoLister
 }
 
-// ClusterNodeInfoLister interface represents anything that can list/get NodeInfo objects from node name.
+// ClusterNodeInfoLister interface represents anything that can list NodeInfo objects.
 type ClusterNodeInfoLister interface {
 	// List Returns the list of NodeInfos.
 	List() []NodeInfo
+	// Returns the list of NodeInfos in the partition of the scheduler
+	InPartitionList() []NodeInfo
+	// Returns the list of NodeInfos out of the partition of the scheduler
+	OutOfPartitionList() []NodeInfo
 	// HavePodsWithAffinityList Returns the list of NodeInfos of nodes with pods with affinity terms.
 	HavePodsWithAffinityList() []NodeInfo
 	// HavePodsWithRequiredAntiAffinityList Returns the list of NodeInfos of nodes with pods with required anti-affinity terms.
 	HavePodsWithRequiredAntiAffinityList() []NodeInfo
-	// Get Returns the NodeInfo of the given node name.
-	Get(nodeName string) (NodeInfo, error)
+
+	Len() int
 }
 
 type ClusterNodeInfoGetter interface {

pkg/framework/api/node_group.go

@@ -31,7 +31,7 @@ const (
 )
 
 type NodeCircle interface {
-	NodeInfoLister
+	ClusterNodeInfoLister
 	GetKey() string
 	Validate() error
 }
@@ -96,27 +96,27 @@ type NodeGroup interface {
 
 type NodeCircleImpl struct {
 	key string
-	NodeInfoLister
+	ClusterNodeInfoLister
 }
 
 var _ NodeCircle = &NodeCircleImpl{}
 
-func NewNodeCircle(key string, lister NodeInfoLister) NodeCircle {
+func NewNodeCircle(key string, lister ClusterNodeInfoLister) NodeCircle {
 	if lister == nil {
-		lister = NewNodeInfoLister()
+		lister = NewClusterNodeInfoLister()
 	}
-	return &NodeCircleImpl{key: key, NodeInfoLister: lister}
+	return &NodeCircleImpl{key: key, ClusterNodeInfoLister: lister}
 }
 
 func (nc *NodeCircleImpl) GetKey() string {
 	return GenerateReadableKey(nc.key)
 }
 
 func (nc *NodeCircleImpl) Validate() error {
-	if nc.NodeInfoLister == nil {
+	if nc.ClusterNodeInfoLister == nil {
 		return fmt.Errorf("lister is nil")
 	}
-	nodes := nc.NodeInfoLister.List()
+	nodes := nc.ClusterNodeInfoLister.List()
 	if len(nodes) == 0 {
 		return fmt.Errorf("no nodes in this node circle")
 	}
@@ -157,15 +157,17 @@ func (i *PreferredNodesImpl) List() []NodeInfo {
 
 // ------------------------------------------------------------------------------------------
 
+var _ NodeGroup = &NodeGroupImpl{}
+
 type NodeGroupImpl struct {
 	Key string
 	ClusterNodeInfoGetter
 	NodeCircles    []NodeCircle
 	PreferredNodes PreferredNodes
 }
 
-var _ NodeGroup = &NodeGroupImpl{}
-
+// NewNodeGroup creates and returns a new NodeGroup. A non-nil `getter` is passed usually when the
+// node set is large but unchanged, for example, when we get a basic NodeGroup from the snapshot.
 func NewNodeGroup(key string, getter ClusterNodeInfoGetter, nodeCircles []NodeCircle) NodeGroup {
 	if getter == nil {
 		getterImpl := &NodeInfoGetterImpl{NodeInfoMap: make(map[string]NodeInfo)}
@@ -189,6 +191,10 @@ func (ng *NodeGroupImpl) GetKey() string {
 }
 
 func (ng *NodeGroupImpl) Validate() error {
+	if ng.ClusterNodeInfoGetter == nil {
+		return fmt.Errorf("getter is nil")
+	}
+
 	var hasAvailableNodes bool
 	if preferredNodes := ng.PreferredNodes; preferredNodes != nil && len(preferredNodes.List()) > 0 {
 		hasAvailableNodes = true
@@ -241,10 +247,8 @@ func (g *NodeInfoGetterImpl) Get(nodeName string) (NodeInfo, error) {
 
 // ------------------------------------------------------------------------------------------
 
-// NodeInfoListerImpl implements NodeInfoLister interface.
+// NodeInfoListerImpl implements ClusterNodeInfoLister interface.
 type NodeInfoListerImpl struct {
-	// nodeInfoMap is a map of node name to its NodeInfo.
-	NodeInfoMap map[string]NodeInfo
 	// InPartitionNodes is the list of nodes in the partition of the scheduler.
 	InPartitionNodes []NodeInfo
 	// OutOfPartitionNodes is the list of nodes out of the partition of the scheduler.
@@ -256,20 +260,34 @@ type NodeInfoListerImpl struct {
 	HavePodsWithRequiredAntiAffinityNodes []NodeInfo
 }
 
-var _ NodeInfoLister = &NodeInfoListerImpl{}
+var _ ClusterNodeInfoLister = &NodeInfoListerImpl{}
 
-// NewNodeInfoLister creates a new NodeInfoLister object.
-func NewNodeInfoLister() NodeInfoLister {
-	return &NodeInfoListerImpl{
-		NodeInfoMap: make(map[string]NodeInfo),
-	}
+// NewClusterNodeInfoLister creates a new NodeInfoLister object.
+func NewClusterNodeInfoLister() ClusterNodeInfoLister {
+	return &NodeInfoListerImpl{}
 }
 
-// List returns the list of NodeInfos.
 func (i *NodeInfoListerImpl) List() []NodeInfo {
+	if i == nil {
+		return nil
+	}
 	return append(i.InPartitionNodes, i.OutOfPartitionNodes...)
 }
 
+func (i *NodeInfoListerImpl) InPartitionList() []NodeInfo {
+	if i == nil {
+		return nil
+	}
+	return i.InPartitionNodes
+}
+
+func (i *NodeInfoListerImpl) OutOfPartitionList() []NodeInfo {
+	if i == nil {
+		return nil
+	}
+	return i.OutOfPartitionNodes
+}
+
 // HavePodsWithAffinityList returns the list of NodeInfos of nodes with pods with affinity terms.
 func (i *NodeInfoListerImpl) HavePodsWithAffinityList() []NodeInfo {
 	return i.HavePodsWithAffinityNodes
@@ -280,33 +298,20 @@ func (i *NodeInfoListerImpl) HavePodsWithRequiredAntiAffinityList() []NodeInfo {
 	return i.HavePodsWithRequiredAntiAffinityNodes
 }
 
-// Get returns the NodeInfo of the given node name.
-func (i *NodeInfoListerImpl) Get(nodeName string) (NodeInfo, error) {
-	if v, ok := i.NodeInfoMap[nodeName]; ok && (v.GetNode() != nil || v.GetNMNode() != nil) {
-		return v, nil
+func (i *NodeInfoListerImpl) Len() int {
+	if i == nil {
+		return 0
 	}
-	return nil, fmt.Errorf("nodeinfo not found for node name %q", nodeName)
-}
-
-func (i *NodeInfoListerImpl) InPartitionList() []NodeInfo {
-	return i.InPartitionNodes
-}
-
-func (i *NodeInfoListerImpl) OutOfPartitionList() []NodeInfo {
-	return i.OutOfPartitionNodes
+	return len(i.InPartitionNodes) + len(i.OutOfPartitionNodes)
 }
 
 func (i *NodeInfoListerImpl) AddNodeInfo(nodeInfo NodeInfo) {
-	nodeName := nodeInfo.GetNodeName()
-	if _, ok := i.NodeInfoMap[nodeName]; ok {
-		return
-	}
-	i.NodeInfoMap[nodeName] = nodeInfo
 	if nodeInfo.GetNodeInSchedulerPartition() || nodeInfo.GetNMNodeInSchedulerPartition() {
 		i.InPartitionNodes = append(i.InPartitionNodes, nodeInfo)
 	} else {
 		i.OutOfPartitionNodes = append(i.OutOfPartitionNodes, nodeInfo)
 	}
+
 	if len(nodeInfo.GetPodsWithAffinity()) > 0 {
 		i.HavePodsWithAffinityNodes = append(i.HavePodsWithAffinityNodes, nodeInfo)
 	}
@@ -317,9 +322,9 @@ func (i *NodeInfoListerImpl) AddNodeInfo(nodeInfo NodeInfo) {
 
 // ------------------------------------------------------------------------------------------
 
-func FilterNodeInfoLister(lister NodeInfoLister, filterFunc func(NodeInfo) bool) NodeInfoLister {
+func FilterNodeInfoLister(lister ClusterNodeInfoLister, filterFunc func(NodeInfo) bool) ClusterNodeInfoLister {
 	nodes := lister.List()
-	ret := &NodeInfoListerImpl{NodeInfoMap: make(map[string]NodeInfo)}
+	ret := &NodeInfoListerImpl{}
 	for _, node := range nodes {
 		if !filterFunc(node) {
 			continue
@@ -342,29 +347,26 @@ func FilterPreferredNodes(preferredNodes PreferredNodes, filterFunc func(NodeInf
 }
 
 func FilterNodeGroup(nodeGroup NodeGroup, filterFunc func(NodeInfo) bool) NodeGroup {
-	newNodeGroup := NewNodeGroup(nodeGroup.GetKey(), nil, nil)
-	{
-		nodeCircles := nodeGroup.GetNodeCircles()
-		newNodeCircles := make([]NodeCircle, 0, len(nodeCircles))
-		for _, nodeCircle := range nodeCircles {
-			newNodeCircle := NewNodeCircle(
-				nodeCircle.GetKey(),
-				FilterNodeInfoLister(nodeCircle, func(nodeInfo NodeInfo) bool {
-					return filterFunc(nodeInfo)
-				}),
-			)
-			if len(newNodeCircle.List()) > 0 {
-				newNodeCircles = append(newNodeCircles, newNodeCircle)
-			}
+	nodeCircles := nodeGroup.GetNodeCircles()
+	newNodeCircles := make([]NodeCircle, 0, len(nodeCircles))
+	for _, nodeCircle := range nodeCircles {
+		newNodeCircle := NewNodeCircle(
+			nodeCircle.GetKey(),
+			FilterNodeInfoLister(nodeCircle, func(nodeInfo NodeInfo) bool {
+				return filterFunc(nodeInfo)
+			}),
+		)
+		if len(newNodeCircle.List()) > 0 {
+			newNodeCircles = append(newNodeCircles, newNodeCircle)
 		}
-		newNodeGroup.SetNodeCircles(newNodeCircles)
 	}
-	{
-		preferredNodes := nodeGroup.GetPreferredNodes()
-		if preferredNodes != nil {
-			newNodeGroup.SetPreferredNodes(FilterPreferredNodes(preferredNodes, filterFunc))
-		}
+	newNodeGroup := NewNodeGroup(nodeGroup.GetKey(), nil, newNodeCircles)
+
+	preferredNodes := nodeGroup.GetPreferredNodes()
+	if preferredNodes != nil {
+		newNodeGroup.SetPreferredNodes(FilterPreferredNodes(preferredNodes, filterFunc))
 	}
+
 	return newNodeGroup
 }
 

pkg/framework/api/unit.go

@@ -279,10 +279,10 @@ func (p *PodGroupUnit) GetAffinityNodeSelector() (*v1.NodeSelector, error) {
 	return podGroupAffinity.NodeSelector, nil
 }
 
-func (p *PodGroupUnit) GetSortRulesForAffinity() ([]SortRule, error) {
+func (p *PodGroupUnit) GetSortRulesForAffinity() []SortRule {
 	if p.podGroup.Spec.Affinity == nil ||
 		p.podGroup.Spec.Affinity.PodGroupAffinity == nil {
-		return nil, nil
+		return nil
 	}
 	var rules []SortRule
 	for _, r := range p.podGroup.Spec.Affinity.PodGroupAffinity.SortRules {
@@ -299,7 +299,7 @@ func (p *PodGroupUnit) GetSortRulesForAffinity() ([]SortRule, error) {
 			Order:     SortOrder(r.Order),
 		})
 	}
-	return rules, nil
+	return rules
 }
 
 func (p *PodGroupUnit) IsDebugModeOn() bool {
@@ -541,8 +541,8 @@ func (s *SinglePodUnit) GetPreferredAffinity() ([]UnitAffinityTerm, error) {
 	return nil, nil
 }
 
-func (s *SinglePodUnit) GetSortRulesForAffinity() ([]SortRule, error) {
-	return nil, nil
+func (s *SinglePodUnit) GetSortRulesForAffinity() []SortRule {
+	return nil
 }
 
 func (s *SinglePodUnit) IsDebugModeOn() bool {

pkg/scheduler/cache/snapshot.go

@@ -63,10 +63,9 @@ func NewEmptySnapshot(handler commoncache.CacheHandler) *Snapshot {
 }
 
 func (s *Snapshot) MakeBasicNodeGroup() framework.NodeGroup {
-	nodeGroup := framework.NewNodeGroup(
-		framework.DefaultNodeGroupName,
-		s,
-		[]framework.NodeCircle{framework.NewNodeCircle(framework.DefaultNodeCircleName, s)})
+	nodeCircle := framework.NewNodeCircle(framework.DefaultNodeCircleName, s)
+	nodeGroup := framework.NewNodeGroup(framework.DefaultNodeGroupName, s, []framework.NodeCircle{nodeCircle})
+	nodeGroup.SetPreferredNodes(framework.NewPreferredNodes())
 	return nodeGroup
 }
 
@@ -133,6 +132,10 @@ func (s *Snapshot) HavePodsWithRequiredAntiAffinityList() []framework.NodeInfo {
 	return s.nodeSlices.havePodsWithRequiredAntiAffinityNodeSlice.Nodes()
 }
 
+func (s *Snapshot) Len() int {
+	return len(s.nodeSlices.inPartitionNodeSlice.Nodes()) + len(s.nodeSlices.outOfPartitionNodeSlice.Nodes())
+}
+
 // Get returns the NodeInfo of the given node name.
 //
 // Note: Snapshot operations are lock-free. Our premise for removing lock: even if read operations

pkg/scheduler/core/pod_scheduler/pod_scheduler.go

@@ -380,7 +380,7 @@ func (gs *podScheduler) findNodesThatFitPod(
 	f framework.SchedulerFramework,
 	state *framework.CycleState,
 	pod *v1.Pod,
-	nodeLister framework.NodeInfoLister,
+	nodeLister framework.ClusterNodeInfoLister,
 	usr *framework.UnitSchedulingRequest,
 	cachedStatusMap framework.NodeToStatusMap,
 ) ([]framework.NodeInfo, framework.NodeToStatusMap, error) {
@@ -415,7 +415,7 @@ func (gs *podScheduler) findNodesThatPassFilters(
 	f framework.SchedulerFramework,
 	state *framework.CycleState,
 	pod *v1.Pod, statuses, cachedStatuses framework.NodeToStatusMap,
-	nodeLister framework.NodeInfoLister,
+	nodeLister framework.ClusterNodeInfoLister,
 	usr *framework.UnitSchedulingRequest,
 ) ([]framework.NodeInfo, error) {
 	beginCheckNode := time.Now()