From e4c7972f9418c7a5a06d8f95503ef3602fc39ea7 Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Tue, 7 Nov 2023 23:06:42 +0900
Subject: [PATCH 01/11] support plugin config

---
 cloud/scheduler/plugins/registry.go | 100 ++++++++++++++++++++++++++--
 cloud/scheduler/scheduler.go        |  31 ++++++---
 cloud/scheduler/scheduler_test.go   |  18 +++--
 cmd/main.go                         |  13 +++-
 config/manager/manager.yaml         |  37 ++++++++--
 5 files changed, 169 insertions(+), 30 deletions(-)

diff --git a/cloud/scheduler/plugins/registry.go b/cloud/scheduler/plugins/registry.go
index 1d728a0..96e6a1c 100644
--- a/cloud/scheduler/plugins/registry.go
+++ b/cloud/scheduler/plugins/registry.go
@@ -1,33 +1,119 @@
 package plugins
 
 import (
+	"os"
+
+	"gopkg.in/yaml.v3"
+
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/framework"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/idrange"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/nodename"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/noderesource"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/overcommit"
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/random"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/regex"
 )
 
-func NewNodeFilterPlugins() []framework.NodeFilterPlugin {
-	return []framework.NodeFilterPlugin{
+type PluginConfigs struct {
+	filterPlugins map[string]pluginConfig `yaml:"filters"`
+	scorePlugins  map[string]pluginConfig `yaml:"scores"`
+	vmidPlugins   map[string]pluginConfig `yaml:"vmids"`
+}
+
+type pluginConfig struct {
+	Disable bool                   `yaml:"disable,omitempty"`
+	Config  map[string]interface{} `yaml:"config,omitempty"`
+}
+
+type PluginRegistry struct {
+	filterPlugins []framework.NodeFilterPlugin
+	scorePlugins  []framework.NodeScorePlugin
+	vmidPlugins   []framework.VMIDPlugin
+}
+
+func (r *PluginRegistry) FilterPlugins() []framework.NodeFilterPlugin {
+	return r.filterPlugins
+}
+
+func (r *PluginRegistry) ScorePlugins() []framework.NodeScorePlugin {
+	return r.scorePlugins
+}
+
+func (r *PluginRegistry) VMIDPlugins() []framework.VMIDPlugin {
+	return r.vmidPlugins
+}
+
+func NewRegistry(configs PluginConfigs) PluginRegistry {
+	r := PluginRegistry{
+		filterPlugins: NewNodeFilterPlugins(configs.filterPlugins),
+		scorePlugins:  NewNodeScorePlugins(configs.scorePlugins),
+		vmidPlugins:   NewVMIDPlugins(configs.vmidPlugins),
+	}
+	return r
+}
+
+func NewNodeFilterPlugins(config map[string]pluginConfig) []framework.NodeFilterPlugin {
+	pls := []framework.NodeFilterPlugin{
 		&nodename.NodeName{},
 		&overcommit.CPUOvercommit{},
 		&overcommit.MemoryOvercommit{},
 		&regex.NodeRegex{},
 	}
+	plugins := []framework.NodeFilterPlugin{}
+	for _, pl := range pls {
+		c, ok := config[pl.Name()]
+		if ok && c.Disable {
+			continue
+		}
+		plugins = append(plugins, pl)
+	}
+	return plugins
 }
 
-func NewNodeScorePlugins() []framework.NodeScorePlugin {
-	return []framework.NodeScorePlugin{
-		// &random.Random{},
+func NewNodeScorePlugins(config map[string]pluginConfig) []framework.NodeScorePlugin {
+	pls := []framework.NodeScorePlugin{
+		&random.Random{},
 		&noderesource.NodeResource{},
 	}
+	plugins := []framework.NodeScorePlugin{}
+	for _, pl := range pls {
+		c, ok := config[pl.Name()]
+		if ok && c.Disable {
+			continue
+		}
+		plugins = append(plugins, pl)
+	}
+	return plugins
 }
 
-func NewVMIDPlugins() []framework.VMIDPlugin {
-	return []framework.VMIDPlugin{
+func NewVMIDPlugins(config map[string]pluginConfig) []framework.VMIDPlugin {
+	pls := []framework.VMIDPlugin{
 		&idrange.Range{},
 		&regex.Regex{},
 	}
+	plugins := []framework.VMIDPlugin{}
+	for _, pl := range pls {
+		c, ok := config[pl.Name()]
+		if ok && c.Disable {
+			continue
+		}
+		plugins = append(plugins, pl)
+	}
+	return plugins
+}
+
+// Read config file and unmarshal it to PluginConfig type
+func GetPluginConfigFromFile(path string) (PluginConfigs, error) {
+	config := PluginConfigs{}
+	if path == "" {
+		return config, nil
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return config, err
+	}
+	if err := yaml.Unmarshal(b, &config); err != nil {
+		return config, err
+	}
+	return config, nil
 }
diff --git a/cloud/scheduler/scheduler.go b/cloud/scheduler/scheduler.go
index 8dde00c..2531c5b 100644
--- a/cloud/scheduler/scheduler.go
+++ b/cloud/scheduler/scheduler.go
@@ -35,9 +35,14 @@ type Manager struct {
 }
 
 // return manager with initialized scheduler-table
-func NewManager(params SchedulerParams) *Manager {
+func NewManager(params SchedulerParams) (*Manager, error) {
 	table := make(map[schedulerID]*Scheduler)
-	return &Manager{ctx: context.Background(), params: params, table: table}
+	config, err := plugins.GetPluginConfigFromFile(params.PluginConfigFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read plugin config: %v", err)
+	}
+	params.pluginconfigs = config
+	return &Manager{ctx: context.Background(), params: params, table: table}, nil
 }
 
 // return new/existing scheduler
@@ -70,9 +75,7 @@ func (m *Manager) NewScheduler(client *proxmox.Service, opts ...SchedulerOption)
 		client:          client,
 		schedulingQueue: queue.New(),
 
-		filterPlugins: plugins.NewNodeFilterPlugins(),
-		scorePlugins:  plugins.NewNodeScorePlugins(),
-		vmidPlugins:   plugins.NewVMIDPlugins(),
+		registry: plugins.NewRegistry(m.params.PluginConfigs()),
 
 		resultMap: make(map[string]chan *framework.CycleState),
 		logger:    m.params.Logger.WithValues("Name", "qemu-scheduler"),
@@ -122,9 +125,7 @@ type Scheduler struct {
 	client          *proxmox.Service
 	schedulingQueue *queue.SchedulingQueue
 
-	filterPlugins []framework.NodeFilterPlugin
-	scorePlugins  []framework.NodeScorePlugin
-	vmidPlugins   []framework.VMIDPlugin
+	registry plugins.PluginRegistry
 
 	// to do : cache
 
@@ -144,6 +145,14 @@ type Scheduler struct {
 
 type SchedulerParams struct {
 	Logger logr.Logger
+
+	// file path for pluginConfig
+	PluginConfigFile string
+	pluginconfigs    plugins.PluginConfigs
+}
+
+func (p *SchedulerParams) PluginConfigs() plugins.PluginConfigs {
+	return p.pluginconfigs
 }
 
 type schedulerID struct {
@@ -320,7 +329,7 @@ func (s *Scheduler) RunFilterPlugins(ctx context.Context, state *framework.Cycle
 	}
 	for _, nodeInfo := range nodeInfos {
 		status := framework.NewStatus()
-		for _, pl := range s.filterPlugins {
+		for _, pl := range s.registry.FilterPlugins() {
 			status = pl.Filter(ctx, state, config, nodeInfo)
 			if !status.IsSuccess() {
 				status.SetFailedPlugin(pl.Name())
@@ -344,7 +353,7 @@ func (s *Scheduler) RunScorePlugins(ctx context.Context, state *framework.CycleS
 		return nil, status
 	}
 	for index, nodeInfo := range nodeInfos {
-		for _, pl := range s.scorePlugins {
+		for _, pl := range s.registry.ScorePlugins() {
 			score, status := pl.Score(ctx, state, config, nodeInfo)
 			if !status.IsSuccess() {
 				return nil, status
@@ -379,7 +388,7 @@ func selectHighestScoreNode(scoreList framework.NodeScoreList) (string, error) {
 }
 
 func (s *Scheduler) RunVMIDPlugins(ctx context.Context, state *framework.CycleState, config api.VirtualMachineCreateOptions, nextid int, usedID map[int]bool) (int, error) {
-	for _, pl := range s.vmidPlugins {
+	for _, pl := range s.registry.VMIDPlugins() {
 		key := pl.PluginKey()
 		value := ctx.Value(key)
 		if value != nil {
diff --git a/cloud/scheduler/scheduler_test.go b/cloud/scheduler/scheduler_test.go
index cdf9941..2ff9b37 100644
--- a/cloud/scheduler/scheduler_test.go
+++ b/cloud/scheduler/scheduler_test.go
@@ -16,13 +16,15 @@ import (
 var _ = Describe("NewManager", Label("unit", "scheduler"), func() {
 	It("should not error", func() {
 		params := scheduler.SchedulerParams{}
-		manager := scheduler.NewManager(params)
+		manager, err := scheduler.NewManager(params)
+		Expect(err).NotTo(HaveOccurred())
 		Expect(manager).NotTo(BeNil())
 	})
 })
 
 var _ = Describe("NewScheduler", Label("unit", "scheduler"), func() {
-	manager := scheduler.NewManager(scheduler.SchedulerParams{})
+	manager, err := scheduler.NewManager(scheduler.SchedulerParams{})
+	Expect(err).NotTo(HaveOccurred())
 
 	It("should not error", func() {
 		sched := manager.NewScheduler(proxmoxSvc)
@@ -31,7 +33,8 @@ var _ = Describe("NewScheduler", Label("unit", "scheduler"), func() {
 })
 
 var _ = Describe("GetOrCreateScheduler", Label("integration", "scheduler"), func() {
-	manager := scheduler.NewManager(scheduler.SchedulerParams{})
+	manager, err := scheduler.NewManager(scheduler.SchedulerParams{})
+	Expect(err).NotTo(HaveOccurred())
 
 	It("should not error", func() {
 		sched := manager.GetOrCreateScheduler(proxmoxSvc)
@@ -40,7 +43,8 @@ var _ = Describe("GetOrCreateScheduler", Label("integration", "scheduler"), func
 })
 
 var _ = Describe("Run (RunAsync) / IsRunning / Stop", Label("unit", "scheduler"), func() {
-	manager := scheduler.NewManager(scheduler.SchedulerParams{zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))})
+	manager, err := scheduler.NewManager(scheduler.SchedulerParams{Logger: zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))})
+	Expect(err).NotTo(HaveOccurred())
 
 	Context("with minimal scheduler", func() {
 		It("should not error", func() {
@@ -56,7 +60,8 @@ var _ = Describe("Run (RunAsync) / IsRunning / Stop", Label("unit", "scheduler")
 })
 
 var _ = Describe("WithTimeout", Label("integration", "scheduler"), func() {
-	manager := scheduler.NewManager(scheduler.SchedulerParams{})
+	manager, err := scheduler.NewManager(scheduler.SchedulerParams{})
+	Expect(err).NotTo(HaveOccurred())
 
 	It("should not error", func() {
 		sched := manager.NewScheduler(proxmoxSvc, scheduler.WithTimeout(2*time.Second))
@@ -70,7 +75,8 @@ var _ = Describe("WithTimeout", Label("integration", "scheduler"), func() {
 })
 
 var _ = Describe("CreateQEMU", Label("integration"), func() {
-	manager := scheduler.NewManager(scheduler.SchedulerParams{zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))})
+	manager, err := scheduler.NewManager(scheduler.SchedulerParams{Logger: zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))})
+	Expect(err).NotTo(HaveOccurred())
 	var result framework.SchedulerResult
 
 	AfterEach(func() {
diff --git a/cmd/main.go b/cmd/main.go
index 83bca3f..c8067fc 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -56,11 +56,13 @@ func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
+	var pluginConfig string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
+	flag.StringVar(&pluginConfig, "scheduler-plugin-config", "/etc/qemu-scheduler/plugin-config.yaml", "The config file path for qemu-scheduler plugins")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -92,11 +94,20 @@ func main() {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
+	schedManager, err := scheduler.NewManager(
+		scheduler.SchedulerParams{
+			Logger:           zap.New(zap.UseFlagOptions(&opts)),
+			PluginConfigFile: pluginConfig,
+		},
+	)
+	if err != nil {
+		setupLog.Error(err, "failed to start qemu-scheudler manager")
+	}
 
 	if err = (&controller.ProxmoxMachineReconciler{
 		Client:           mgr.GetClient(),
 		Scheme:           mgr.GetScheme(),
-		SchedulerManager: scheduler.NewManager(scheduler.SchedulerParams{Logger: zap.New(zap.UseFlagOptions(&opts))}),
+		SchedulerManager: schedManager,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ProxmoxMachine")
 		os.Exit(1)
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index b82b9cf..4f8faa4 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Namespace
 metadata:
   labels:
-    control-plane: capp-controller-manager
+    control-plane: cappx-controller-manager
     app.kubernetes.io/name: namespace
     app.kubernetes.io/instance: system
     app.kubernetes.io/component: manager
@@ -17,9 +17,9 @@ metadata:
   name: controller-manager
   namespace: system
   labels:
-    control-plane: capp-controller-manager
+    control-plane: cappx-controller-manager
     app.kubernetes.io/name: deployment
-    app.kubernetes.io/instance: capp-controller-manager
+    app.kubernetes.io/instance: cappx-controller-manager
     app.kubernetes.io/component: manager
     app.kubernetes.io/created-by: cluster-api-provider-proxmox
     app.kubernetes.io/part-of: cluster-api-provider-proxmox
@@ -27,14 +27,14 @@ metadata:
 spec:
   selector:
     matchLabels:
-      control-plane: capp-controller-manager
+      control-plane: cappx-controller-manager
   replicas: 1
   template:
     metadata:
       annotations:
         kubectl.kubernetes.io/default-container: manager
       labels:
-        control-plane: capp-controller-manager
+        control-plane: cappx-controller-manager
     spec:
       # TODO(user): Uncomment the following code to configure the nodeAffinity expression
       # according to the platforms which are supported by your solution.
@@ -70,6 +70,7 @@ spec:
         - /manager
         args:
         - --leader-elect
+        - --scheduler-plugin-config=/etc/qemu-scheduler/plugin-config.yaml
         image: controller:latest
         name: manager
         securityContext:
@@ -98,5 +99,31 @@ spec:
           requests:
             cpu: 10m
             memory: 64Mi
+        volumeMounts:
+          - name: scheduler-configs
+            mountPath: /etc/qemu-scheduler
+            readOnly: true
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
+      volumes:
+        - name: scheduler-configs
+          configMap:
+            name: qemu-scheduler-configs
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    control-plane: cappx-controller-manager
+    app.kubernetes.io/name: namespace
+    app.kubernetes.io/instance: system
+    app.kubernetes.io/component: manager
+    app.kubernetes.io/created-by: cluster-api-provider-proxmox
+    app.kubernetes.io/part-of: cluster-api-provider-proxmox
+    app.kubernetes.io/managed-by: kustomize
+  name: qemu-scheduler-configs
+  namespace: system
+data:
+  plugin-config.yaml: | 
+    scores:
+      - disable: true

From 133b41188c1fb584def27f2da8ef29c174f0bc23 Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Tue, 7 Nov 2023 23:29:46 +0900
Subject: [PATCH 02/11] use optional for hardware devices

---
 api/v1beta1/proxmoxmachine_types.go | 2 +-
 api/v1beta1/type.go                 | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/api/v1beta1/proxmoxmachine_types.go b/api/v1beta1/proxmoxmachine_types.go
index c83693d..07d4ffb 100644
--- a/api/v1beta1/proxmoxmachine_types.go
+++ b/api/v1beta1/proxmoxmachine_types.go
@@ -54,7 +54,7 @@ type ProxmoxMachineSpec struct {
 
 	// Hardware
 	// +kubebuilder:default:={cpu:2,disk:"50G",memory:4096,networkDevice:{model:virtio,bridge:vmbr0,firewall:true}}
-	Hardware Hardware `json:"hardware"`
+	Hardware Hardware `json:"hardware,omitempty"`
 
 	// Network
 	Network Network `json:"network,omitempty"`
diff --git a/api/v1beta1/type.go b/api/v1beta1/type.go
index 4d8f16b..355aa51 100644
--- a/api/v1beta1/type.go
+++ b/api/v1beta1/type.go
@@ -99,16 +99,16 @@ type Hardware struct {
 	// network devices
 	// to do: multiple devices
 	// +kubebuilder:default:={model:virtio,bridge:vmbr0,firewall:true}
-	NetworkDevice NetworkDevice `json:"networkDevice"`
+	NetworkDevice NetworkDevice `json:"networkDevice,omitempty"`
 }
 
 // Network Device
 type NetworkDevice struct {
 	// +kubebuilder:default:="virtio"
-	Model NetworkDeviceModel `json:"model"`
+	Model NetworkDeviceModel `json:"model,omitempty"`
 
 	// +kubebuilder:default:="vmbr0"
-	Bridge NetworkDeviceBridge `json:"bridge"`
+	Bridge NetworkDeviceBridge `json:"bridge,omitempty"`
 
 	// +kubebuilder:default:=true
 	Firewall bool `json:"firewall,omitempty"`

From 8ad62bb87fe4692b5eb7bfa8f7dd36217eb6a8ab Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Tue, 7 Nov 2023 23:30:47 +0900
Subject: [PATCH 03/11] set flags

---
 config/default/manager_auth_proxy_patch.yaml | 5 -----
 config/manager/manager.yaml                  | 4 +++-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml
index b751266..7945f64 100644
--- a/config/default/manager_auth_proxy_patch.yaml
+++ b/config/default/manager_auth_proxy_patch.yaml
@@ -48,8 +48,3 @@ spec:
           requests:
             cpu: 5m
             memory: 64Mi
-      - name: manager
-        args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index 4f8faa4..1c2c53c 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -69,7 +69,9 @@ spec:
       - command:
         - /manager
         args:
-        - --leader-elect
+        - "--health-probe-bind-address=:8081"
+        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--leader-elect"
         - --scheduler-plugin-config=/etc/qemu-scheduler/plugin-config.yaml
         image: controller:latest
         name: manager

From fa64f227feb8242f4c16b8368f308c5d10e5b9ba Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Tue, 7 Nov 2023 23:44:17 +0900
Subject: [PATCH 04/11] fix plugin-config

---
 cmd/main.go                 | 1 +
 config/manager/manager.yaml | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmd/main.go b/cmd/main.go
index c8067fc..93dc35d 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -102,6 +102,7 @@ func main() {
 	)
 	if err != nil {
 		setupLog.Error(err, "failed to start qemu-scheudler manager")
+		os.Exit(1)
 	}
 
 	if err = (&controller.ProxmoxMachineReconciler{
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index 1c2c53c..0863802 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -128,4 +128,10 @@ metadata:
 data:
   plugin-config.yaml: | 
     scores:
-      - disable: true
+      - Random:
+          disable: true
+    # filters:
+      # - CPUOvercommit:
+      #     disable: true
+      # - MemoryOvercommit:
+      #     disable: true

From 35ef1838c58b3af22d67434365227c4f13b74df9 Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Wed, 8 Nov 2023 20:38:24 +0900
Subject: [PATCH 05/11] use enable instead of disable for plugin config

---
 cloud/scheduler/README.md           | 32 +++++++++++++++++++++++++++--
 cloud/scheduler/plugins/registry.go | 10 ++++-----
 config/manager/manager.yaml         | 12 +++++------
 3 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/cloud/scheduler/README.md b/cloud/scheduler/README.md
index c825b73..abf9818 100644
--- a/cloud/scheduler/README.md
+++ b/cloud/scheduler/README.md
@@ -8,7 +8,12 @@ Basic flow of the node selection process is `filter => score => select one node
 
 ### Filter Plugins
 
-Filter plugins filter the node based on nodename, overcommit ratio etc.
+Filter plugins filter the node based on nodename, overcommit ratio etc. So that we can avoid to run qemus on not desired Proxmox nodes.
+
+- [NodeName plugin](./plugins/nodename/node_name.go) (pass the node matching specified node name)
+- [CPUOvercommit plugin](./plugins/overcommit/cpu_overcommit.go) (pass the node that has enough cpu against running vm)
+- [MemoryOvercommit plugin](./plugins/overcommit/memory_overcommit.go) (pass the node that has enough memory against running vm)
+- [NodeRegex plugin](./plugins/regex/node_regex.go) (pass the node matching specified regex)
 
 #### regex plugin
 
@@ -20,11 +25,17 @@ value(example): node[0-9]+
 
 ### Score Plugins
 
-Score plugins score the nodes based on resource etc.
+Score plugins score the nodes based on resource etc. So that we can run qemus on the most appropriate Proxmox node.
+
+- [NodeResource plugin](./plugins/noderesource/node_resrouce.go) (nodes with more resources have higher scores)
+- [Random plugin](./plugins/random/random.go) (just a reference implementation of score plugin)
 
 ## How to specify vmid
 qemu-scheduler reads context and find key registerd to scheduler. If the context has any value of the registerd key, qemu-scheduler uses the plugin that matchies the key.
 
+- [Range plugin](./plugins/idrange/idrange.go) (select minimum availabe vmid from the specified id range)
+- [VMIDRegex plugin](./plugins/regex/vmid_regex.go) (select minimum availabe vmid matching specified regex)
+
 ### Range Plugin
 You can specify vmid range with `(start id)-(end id)` format.
 ```sh
@@ -64,4 +75,21 @@ spec:
     metadata:
       annotations:
         node.qemu-scheduler/regex: node[0-9]+ # this annotation will be propagated to your ProxmoxMachine via MachineSet
+```
+
+## How to configure (or disable/enable) specific Plugins
+
+By default, all the plugins are enabled. You can disable specific plugins via plugin-config.
+```sh
+# example plugin-config.yaml
+
+# plugin type name (scores, filters, vmids)
+scores:
+  - Random: 
+      enable: false # disable
+filters:
+  - CPUOvercommit:
+      enable: false # disable
+  - MemoryOvercommit:
+      enable true   # enable (can be omitted)
 ```
\ No newline at end of file
diff --git a/cloud/scheduler/plugins/registry.go b/cloud/scheduler/plugins/registry.go
index 96e6a1c..ef66e81 100644
--- a/cloud/scheduler/plugins/registry.go
+++ b/cloud/scheduler/plugins/registry.go
@@ -21,8 +21,8 @@ type PluginConfigs struct {
 }
 
 type pluginConfig struct {
-	Disable bool                   `yaml:"disable,omitempty"`
-	Config  map[string]interface{} `yaml:"config,omitempty"`
+	Enable bool                   `yaml:"enable,omitempty"`
+	Config map[string]interface{} `yaml:"config,omitempty"`
 }
 
 type PluginRegistry struct {
@@ -62,7 +62,7 @@ func NewNodeFilterPlugins(config map[string]pluginConfig) []framework.NodeFilter
 	plugins := []framework.NodeFilterPlugin{}
 	for _, pl := range pls {
 		c, ok := config[pl.Name()]
-		if ok && c.Disable {
+		if ok && !c.Enable {
 			continue
 		}
 		plugins = append(plugins, pl)
@@ -78,7 +78,7 @@ func NewNodeScorePlugins(config map[string]pluginConfig) []framework.NodeScorePl
 	plugins := []framework.NodeScorePlugin{}
 	for _, pl := range pls {
 		c, ok := config[pl.Name()]
-		if ok && c.Disable {
+		if ok && !c.Enable {
 			continue
 		}
 		plugins = append(plugins, pl)
@@ -94,7 +94,7 @@ func NewVMIDPlugins(config map[string]pluginConfig) []framework.VMIDPlugin {
 	plugins := []framework.VMIDPlugin{}
 	for _, pl := range pls {
 		c, ok := config[pl.Name()]
-		if ok && c.Disable {
+		if ok && !c.Enable {
 			continue
 		}
 		plugins = append(plugins, pl)
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index 0863802..0048362 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -129,9 +129,9 @@ data:
   plugin-config.yaml: | 
     scores:
       - Random:
-          disable: true
-    # filters:
-      # - CPUOvercommit:
-      #     disable: true
-      # - MemoryOvercommit:
-      #     disable: true
+          enable: false
+    filters:
+      - CPUOvercommit:
+          enable: false
+      - MemoryOvercommit:
+          enable: false

From 48c2c5f5bf5578559c953f8938e238050f597c02 Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Thu, 9 Nov 2023 00:13:33 +0900
Subject: [PATCH 06/11] fix plugin-config

---
 cloud/scheduler/plugins/registry.go      | 22 ++++----
 cloud/scheduler/plugins/registry_test.go | 69 ++++++++++++++++++++++++
 cloud/scheduler/scheduler.go             |  1 +
 cloud/scheduler/scheduler_test.go        | 28 ++++++++--
 cmd/main.go                              |  2 +-
 5 files changed, 105 insertions(+), 17 deletions(-)
 create mode 100644 cloud/scheduler/plugins/registry_test.go

diff --git a/cloud/scheduler/plugins/registry.go b/cloud/scheduler/plugins/registry.go
index ef66e81..5c85d96 100644
--- a/cloud/scheduler/plugins/registry.go
+++ b/cloud/scheduler/plugins/registry.go
@@ -15,12 +15,12 @@ import (
 )
 
 type PluginConfigs struct {
-	filterPlugins map[string]pluginConfig `yaml:"filters"`
-	scorePlugins  map[string]pluginConfig `yaml:"scores"`
-	vmidPlugins   map[string]pluginConfig `yaml:"vmids"`
+	FilterPlugins map[string]PluginConfig `yaml:"filters,omitempty"`
+	ScorePlugins  map[string]PluginConfig `yaml:"scores,omitempty"`
+	VMIDPlugins   map[string]PluginConfig `yaml:"vmids,omitempty"`
 }
 
-type pluginConfig struct {
+type PluginConfig struct {
 	Enable bool                   `yaml:"enable,omitempty"`
 	Config map[string]interface{} `yaml:"config,omitempty"`
 }
@@ -45,14 +45,14 @@ func (r *PluginRegistry) VMIDPlugins() []framework.VMIDPlugin {
 
 func NewRegistry(configs PluginConfigs) PluginRegistry {
 	r := PluginRegistry{
-		filterPlugins: NewNodeFilterPlugins(configs.filterPlugins),
-		scorePlugins:  NewNodeScorePlugins(configs.scorePlugins),
-		vmidPlugins:   NewVMIDPlugins(configs.vmidPlugins),
+		filterPlugins: NewNodeFilterPlugins(configs.FilterPlugins),
+		scorePlugins:  NewNodeScorePlugins(configs.ScorePlugins),
+		vmidPlugins:   NewVMIDPlugins(configs.VMIDPlugins),
 	}
 	return r
 }
 
-func NewNodeFilterPlugins(config map[string]pluginConfig) []framework.NodeFilterPlugin {
+func NewNodeFilterPlugins(config map[string]PluginConfig) []framework.NodeFilterPlugin {
 	pls := []framework.NodeFilterPlugin{
 		&nodename.NodeName{},
 		&overcommit.CPUOvercommit{},
@@ -70,7 +70,7 @@ func NewNodeFilterPlugins(config map[string]pluginConfig) []framework.NodeFilter
 	return plugins
 }
 
-func NewNodeScorePlugins(config map[string]pluginConfig) []framework.NodeScorePlugin {
+func NewNodeScorePlugins(config map[string]PluginConfig) []framework.NodeScorePlugin {
 	pls := []framework.NodeScorePlugin{
 		&random.Random{},
 		&noderesource.NodeResource{},
@@ -86,7 +86,7 @@ func NewNodeScorePlugins(config map[string]pluginConfig) []framework.NodeScorePl
 	return plugins
 }
 
-func NewVMIDPlugins(config map[string]pluginConfig) []framework.VMIDPlugin {
+func NewVMIDPlugins(config map[string]PluginConfig) []framework.VMIDPlugin {
 	pls := []framework.VMIDPlugin{
 		&idrange.Range{},
 		&regex.Regex{},
@@ -104,7 +104,7 @@ func NewVMIDPlugins(config map[string]pluginConfig) []framework.VMIDPlugin {
 
 // Read config file and unmarshal it to PluginConfig type
 func GetPluginConfigFromFile(path string) (PluginConfigs, error) {
-	config := PluginConfigs{}
+	var config PluginConfigs
 	if path == "" {
 		return config, nil
 	}
diff --git a/cloud/scheduler/plugins/registry_test.go b/cloud/scheduler/plugins/registry_test.go
new file mode 100644
index 0000000..1705d98
--- /dev/null
+++ b/cloud/scheduler/plugins/registry_test.go
@@ -0,0 +1,69 @@
+package plugins_test
+
+import (
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins"
+)
+
+func TestPlugins(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Plugins Suite")
+}
+
+var _ = Describe("GetPluginConfigFromFile", Label("unit", "scheduler"), func() {
+	path := "./test-plugin-config.yaml"
+	BeforeEach(func() {
+		content := `scores:
+  Random:
+    enable: false`
+		err := stringToFile(content, path)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		err := rm(path)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	Context("with empty file path", func() {
+		path := ""
+		It("should not error", func() {
+			config, err := plugins.GetPluginConfigFromFile(path)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(config).To(Equal(plugins.PluginConfigs{}))
+		})
+	})
+
+	Context("with non-empty file path", func() {
+		It("should not error", func() {
+			config, err := plugins.GetPluginConfigFromFile(path)
+			Expect(err).NotTo(HaveOccurred())
+			scores := map[string]plugins.PluginConfig{}
+			scores["Random"] = plugins.PluginConfig{Enable: false}
+			Expect(config).To(Equal(plugins.PluginConfigs{ScorePlugins: scores}))
+		})
+	})
+
+	Context("with wrong file path", func() {
+		It("shold error", func() {
+			path := "./wrong-plugin-config.yaml"
+			config, err := plugins.GetPluginConfigFromFile(path)
+			Expect(err).To(HaveOccurred())
+			Expect(config).To(Equal(plugins.PluginConfigs{}))
+		})
+	})
+})
+
+func stringToFile(str string, path string) error {
+	b := []byte(str)
+	return os.WriteFile(path, b, 0666)
+}
+
+func rm(path string) error {
+	return os.Remove(path)
+}
diff --git a/cloud/scheduler/scheduler.go b/cloud/scheduler/scheduler.go
index 2531c5b..886274a 100644
--- a/cloud/scheduler/scheduler.go
+++ b/cloud/scheduler/scheduler.go
@@ -42,6 +42,7 @@ func NewManager(params SchedulerParams) (*Manager, error) {
 		return nil, fmt.Errorf("failed to read plugin config: %v", err)
 	}
 	params.pluginconfigs = config
+	params.Logger.Info(fmt.Sprintf("load plugin config: %v", config))
 	return &Manager{ctx: context.Background(), params: params, table: table}, nil
 }
 
diff --git a/cloud/scheduler/scheduler_test.go b/cloud/scheduler/scheduler_test.go
index 2ff9b37..9cf08b2 100644
--- a/cloud/scheduler/scheduler_test.go
+++ b/cloud/scheduler/scheduler_test.go
@@ -14,11 +14,29 @@ import (
 )
 
 var _ = Describe("NewManager", Label("unit", "scheduler"), func() {
-	It("should not error", func() {
-		params := scheduler.SchedulerParams{}
-		manager, err := scheduler.NewManager(params)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(manager).NotTo(BeNil())
+	Context("with empty params", func() {
+		It("should not error", func() {
+			params := scheduler.SchedulerParams{}
+			manager, err := scheduler.NewManager(params)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(manager).NotTo(BeNil())
+		})
+	})
+	Context("with only logger", func() {
+		It("should not error", func() {
+			params := scheduler.SchedulerParams{Logger: zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))}
+			manager, err := scheduler.NewManager(params)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(manager).NotTo(BeNil())
+		})
+	})
+	Context("with plugin-config", func() {
+		It("should not error", func() {
+			params := scheduler.SchedulerParams{PluginConfigFile: ""}
+			manager, err := scheduler.NewManager(params)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(manager).NotTo(BeNil())
+		})
 	})
 })
 
diff --git a/cmd/main.go b/cmd/main.go
index 93dc35d..ef61fad 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -62,7 +62,7 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.StringVar(&pluginConfig, "scheduler-plugin-config", "/etc/qemu-scheduler/plugin-config.yaml", "The config file path for qemu-scheduler plugins")
+	flag.StringVar(&pluginConfig, "scheduler-plugin-config", "", "The config file path for qemu-scheduler plugins")
 	opts := zap.Options{
 		Development: true,
 	}

From d3d5e9f5ad1280b0b1111eeab6374779088ed71f Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Thu, 9 Nov 2023 00:17:20 +0900
Subject: [PATCH 07/11] fix typo / add instruction

---
 cloud/scheduler/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cloud/scheduler/README.md b/cloud/scheduler/README.md
index abf9818..f571e2a 100644
--- a/cloud/scheduler/README.md
+++ b/cloud/scheduler/README.md
@@ -79,7 +79,7 @@ spec:
 
 ## How to configure (or disable/enable) specific Plugins
 
-By default, all the plugins are enabled. You can disable specific plugins via plugin-config.
+By default, all the plugins are enabled. You can disable specific plugins via plugin-config. for CAPPX, check example ConfigMap [here](../../config/manager/manager.yaml)
 ```sh
 # example plugin-config.yaml
 
@@ -91,5 +91,5 @@ filters:
   - CPUOvercommit:
       enable: false # disable
   - MemoryOvercommit:
-      enable true   # enable (can be omitted)
+      enable: true   # enable (can be omitted)
 ```
\ No newline at end of file

From 49918bb42f2e4d606cb014b47f871037065760e7 Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Thu, 9 Nov 2023 00:35:30 +0900
Subject: [PATCH 08/11] fix configmap

---
 config/manager/manager.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index 0048362..8d232ef 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -128,10 +128,10 @@ metadata:
 data:
   plugin-config.yaml: | 
     scores:
-      - Random:
-          enable: false
+      Random:
+        enable: false
     filters:
-      - CPUOvercommit:
-          enable: false
-      - MemoryOvercommit:
-          enable: false
+      CPUOvercommit:
+        enable: false
+      MemoryOvercommit:
+        enable: false

From 6530c1301b9b16f2fd505e86d4be6f9926121199 Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Thu, 9 Nov 2023 00:36:07 +0900
Subject: [PATCH 09/11] fix makefile

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index d4501c7..edc7c00 100644
--- a/Makefile
+++ b/Makefile
@@ -118,7 +118,7 @@ build-e2e-image: ## Build cappx image to be used for e2e test
 
 USE_EXISTING_CLUSTER := false
 .PHONY: e2e
-e2e: generate-e2e-templates build-e2e-image cleanup-e2e-artifacts ## Run e2e test
+e2e: generate-e2e-templates build-e2e-image cleanup-e2e-artifacts $(KUBECTL) ## Run e2e test
 	go test $(E2E_DIR)/... -v \
 	-timeout=$(GINKGO_TIMEOUT) \
 	--e2e.artifacts-folder=$(E2E_DIR) \
@@ -184,7 +184,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default | kubectl diff -f -
+	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.

From 5be2cfa12317c0c6364bb673f803fb0a33a1af7d Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Thu, 9 Nov 2023 00:39:59 +0900
Subject: [PATCH 10/11] remove random plugin

---
 cloud/scheduler/plugins/registry.go | 2 --
 config/manager/manager.yaml         | 3 ---
 2 files changed, 5 deletions(-)

diff --git a/cloud/scheduler/plugins/registry.go b/cloud/scheduler/plugins/registry.go
index 5c85d96..00fe773 100644
--- a/cloud/scheduler/plugins/registry.go
+++ b/cloud/scheduler/plugins/registry.go
@@ -10,7 +10,6 @@ import (
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/nodename"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/noderesource"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/overcommit"
-	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/random"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scheduler/plugins/regex"
 )
 
@@ -72,7 +71,6 @@ func NewNodeFilterPlugins(config map[string]PluginConfig) []framework.NodeFilter
 
 func NewNodeScorePlugins(config map[string]PluginConfig) []framework.NodeScorePlugin {
 	pls := []framework.NodeScorePlugin{
-		&random.Random{},
 		&noderesource.NodeResource{},
 	}
 	plugins := []framework.NodeScorePlugin{}
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
index 8d232ef..89036c0 100644
--- a/config/manager/manager.yaml
+++ b/config/manager/manager.yaml
@@ -127,9 +127,6 @@ metadata:
   namespace: system
 data:
   plugin-config.yaml: | 
-    scores:
-      Random:
-        enable: false
     filters:
       CPUOvercommit:
         enable: false

From 3693d0344bdaf765e55b38eeae1c4d01dbf1fc31 Mon Sep 17 00:00:00 2001
From: sp-yduck <sudo20.t.ab@gmail.com>
Date: Thu, 9 Nov 2023 00:56:16 +0900
Subject: [PATCH 11/11] update scheduler README

---
 cloud/scheduler/README.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/cloud/scheduler/README.md b/cloud/scheduler/README.md
index f571e2a..c474474 100644
--- a/cloud/scheduler/README.md
+++ b/cloud/scheduler/README.md
@@ -28,7 +28,7 @@ value(example): node[0-9]+
 Score plugins score the nodes based on resource etc. So that we can run qemus on the most appropriate Proxmox node.
 
 - [NodeResource plugin](./plugins/noderesource/node_resrouce.go) (nodes with more resources have higher scores)
-- [Random plugin](./plugins/random/random.go) (just a reference implementation of score plugin)
+- [Random plugin](./plugins/random/random.go) (diabled by default. just a reference implementation of score plugin)
 
 ## How to specify vmid
 qemu-scheduler reads context and find key registerd to scheduler. If the context has any value of the registerd key, qemu-scheduler uses the plugin that matchies the key.
@@ -84,12 +84,12 @@ By default, all the plugins are enabled. You can disable specific plugins via pl
 # example plugin-config.yaml
 
 # plugin type name (scores, filters, vmids)
-scores:
-  - Random: 
-      enable: false # disable
 filters:
-  - CPUOvercommit:
-      enable: false # disable
-  - MemoryOvercommit:
-      enable: true   # enable (can be omitted)
+  CPUOvercommit:
+    enable: false # disable
+  MemoryOvercommit:
+    enable: true   # enable (can be omitted)
+vmids:
+  Regex:
+    enable: false # disable
 ```
\ No newline at end of file