Enhance README

.gitignore

@@ -8,6 +8,8 @@
 *.so
 *.dylib
 
+bin/
+
 # Test binary, built with `go test -c`
 *.test
 
@@ -25,3 +27,5 @@ hack/tools/bin/
 
 # Go workspace file
 go.work
+
+.gitconfig

README.md

@@ -55,6 +55,64 @@ metadata:
   namespace: default
 ```
 
+The content of the ConfigMap can be either JSON or YAML. Same ConfigMap using YAML
+
+```yaml
+apiVersion: v1
+data:
+  config.yaml: |
+    resources:
+    - group: ""
+      version: v"
+      kind: Secret
+    - group: apps
+      version: v1
+      kind: Deployment
+    logs:
+    - namespace: kube-system
+      sinceSeconds:600
+kind: ConfigMap
+metadata:
+  name: k8s-collector
+  namespace: default
+```
+
+When collecting resources, specify the group/version/kind. That is mandatory. Anything can be passed, including CustomResourceDefinitions.
+You can filter resources by namespace and/or labels.
+For instance to collect all Secret instances but *only* Deployment instances:
+
+- in the __nginx__ namespace and
+- with labels app:nginx and version:latest
+
+use the following ConfigMap
+
+```yaml
+apiVersion: v1
+data:
+  config.yaml: |
+    resources:
+    - group: ""
+      version: v"
+      kind: Secret
+    - group: apps
+      version: v1
+      kind: Deployment
+      namespace: nginx
+      labelFilters:
+      - key: app
+        operation: Equal
+        value: nginx
+      - key: version
+        operation: Equal
+        value: latest
+kind: ConfigMap
+metadata:
+  name: k8s-collector
+  namespace: default
+```
+
+When collecting logs, you can select a subset of Pods by specifying the namespace and the label filters (same as for resources).
+
 ### Collection folders
 k8s-collector will create two folders:
 

go.mod

@@ -3,7 +3,6 @@ module github.com/gianlucam76/k8s_collector
 go 1.20
 
 require (
-	github.com/gdexlab/go-render v1.0.1
 	github.com/go-logr/logr v1.3.0
 	github.com/onsi/ginkgo/v2 v2.13.0
 	github.com/onsi/gomega v1.29.0
@@ -69,6 +68,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.28.3 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/kubectl v0.28.2 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/cluster-api v1.5.3 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -21,8 +21,6 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ
 github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
-github.com/gdexlab/go-render v1.0.1 h1:rxqB3vo5s4n1kF0ySmoNeSPRYkEsyHgln4jFIQY7v0U=
-github.com/gdexlab/go-render v1.0.1/go.mod h1:wRi5nW2qfjiGj4mPukH4UV0IknS1cHD4VgFTmJX5JzM=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
@@ -35,6 +33,7 @@ github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogB
 github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
@@ -206,6 +205,8 @@ k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
+k8s.io/kubectl v0.28.2 h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM=
+k8s.io/kubectl v0.28.2/go.mod h1:6EQWTPySF1fn7yKoQZHYf9TPwIl2AygHEcJoxFekr64=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/cluster-api v1.5.3 h1:TtxneDCps14sZ9bNr515ivBRMj9OwUE6mRUr6l7fSBA=

pkg/utils/collect.go

@@ -21,11 +21,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/projectsveltos/libsveltos/lib/utils"
 )
 
 func (a *Collector) CollectResouces(ctx context.Context, logger logr.Logger) error {
@@ -65,19 +70,53 @@ func (a *Collector) loadConfiguration(ctx context.Context, logger logr.Logger) (
 
 	for k := range configMap.Data {
 		var currentConfiguration Configuration
+
+		err = yaml.Unmarshal([]byte(configMap.Data[k]), &currentConfiguration)
+		if err == nil {
+			return &currentConfiguration, nil
+		}
+
 		err = json.Unmarshal([]byte(configMap.Data[k]), &currentConfiguration)
-		if err != nil {
-			logger.Info(fmt.Sprintf("content %q", configMap.Data[k]))
-			logger.Info(fmt.Sprintf("configMap key: %q does not contain a valid configuration instance: %v", k, err))
-			continue
+		if err == nil {
+			return &currentConfiguration, nil
 		}
 
-		return &currentConfiguration, nil
+		logger.Info(fmt.Sprintf("content %q", configMap.Data[k]))
+		logger.Info(fmt.Sprintf("configMap key: %q does not contain a valid configuration instance: %v", k, err))
 	}
 
 	return nil, nil
 }
 
+func getUnstructured(section []byte, logger logr.Logger) ([]*unstructured.Unstructured, error) {
+	policies := make([]*unstructured.Unstructured, 0)
+	elements := strings.Split(string(section), "---")
+
+	for i := range elements {
+		section := elements[i]
+		if section == "" {
+			continue
+		}
+
+		section = elements[i]
+
+		policy, err := utils.GetUnstructured([]byte(section))
+		if err != nil {
+			logger.Error(err, fmt.Sprintf("failed to get policy from Data %.100s", section))
+			return nil, err
+		}
+
+		if policy == nil {
+			logger.Error(err, fmt.Sprintf("failed to get policy from Data %.100s", section))
+			return nil, fmt.Errorf("failed to get policy from Data %.100s", section)
+		}
+
+		policies = append(policies, policy)
+	}
+
+	return policies, nil
+}
+
 func (a *Collector) collectData(ctx context.Context, configuration *Configuration, logger logr.Logger) error {
 	logger.Info("collecting logs")
 	var err error

pkg/utils/collect_test.go

@@ -7,6 +7,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v2"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -16,7 +17,7 @@ import (
 )
 
 var _ = Describe("Collect", func() {
-	It("loadConfiguration loads configuration from ConfigMap", func() {
+	It("loadConfiguration loads configuration from ConfigMap (YAML)", func() {
 		sinceSecond := int64(600)
 		collectorConfig := &utils.Configuration{
 			Logs: []utils.Log{
@@ -28,7 +29,7 @@ var _ = Describe("Collect", func() {
 			},
 		}
 
-		jsonBytes, err := json.Marshal(collectorConfig)
+		jsonBytes, err := yaml.Marshal(collectorConfig)
 		Expect(err).To(BeNil())
 
 		configMap := &corev1.ConfigMap{
@@ -54,4 +55,44 @@ var _ = Describe("Collect", func() {
 		Expect(err).To(BeNil())
 		Expect(collectorConfig).ToNot(BeNil())
 	})
+
+	It("loadConfiguration loads configuration from ConfigMap (JSON)", func() {
+		sinceSecond := int64(600)
+		collectorConfig := &utils.Configuration{
+			Logs: []utils.Log{
+				{Namespace: "kube-system", SinceSeconds: &sinceSecond},
+			},
+			Resources: []utils.Resource{
+				{Group: "", Version: "v1", Kind: "Service"},
+				{Group: "", Version: "v1", Kind: "Pod"},
+				{Group: "apps", Version: "v1", Kind: "Deployment"},
+			},
+		}
+
+		jsonBytes, err := json.Marshal(collectorConfig)
+		Expect(err).To(BeNil())
+
+		configMap := &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "default",
+				Name:      "bar",
+			},
+			Data: map[string]string{
+				"config": string(jsonBytes),
+			},
+		}
+
+		collector, err := utils.GetCollectorInstance(scheme, env.Config, "", configMap.Name)
+		Expect(err).To(BeNil())
+		Expect(k8sClient.Create(context.TODO(), configMap)).To(Succeed())
+
+		waitForObject(context.TODO(), k8sClient, configMap)
+
+		os.Setenv("COLLECTOR_NAMESPACE", configMap.Namespace)
+		config := textlogger.NewConfig(textlogger.Verbosity(1))
+		logger := textlogger.NewLogger(config)
+		collectorConfig, err = utils.LoadConfiguration(collector, context.TODO(), logger)
+		Expect(err).To(BeNil())
+		Expect(collectorConfig).ToNot(BeNil())
+	})
 })