Pr/exec

.github/workflows/test.yml

@@ -25,6 +25,8 @@ jobs:
           key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
           restore-keys: |
             cache-
+      - name: Install pyyaml for backstage test
+        run: pip3 install pyyaml
       - name: Test
         run: make test
 
@@ -104,6 +106,8 @@ jobs:
           key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
           restore-keys: |
             cache-
+      - name: Install pyyaml for backstage test
+        run: pip3 install pyyaml
       - name: Test
         run: make test-prod
 

.gitignore

@@ -26,3 +26,10 @@ external/diffgen/target
 # For AI agents
 TODO.md
 .envrc
+.db/
+bin/
+out/
+._DS_Store
+
+exec-checkout/
+__debug_bin*

Makefile

@@ -195,7 +195,7 @@ MODERNIZE ?= $(LOCALBIN)/modernize
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v3.8.7
 CONTROLLER_TOOLS_VERSION ?= v0.19.0
-GOLANGCI_LINT_VERSION ?= v2.1.6
+GOLANGCI_LINT_VERSION ?= v2.7.2
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 

api/v1/exec.go

@@ -0,0 +1,26 @@
+package v1
+
+import (
+	"github.com/flanksource/duty/connection"
+	"github.com/flanksource/duty/shell"
+	"github.com/flanksource/duty/types"
+)
+
+type Exec struct {
+	BaseScraper `json:",inline"`
+
+	// Script is an inline script to run
+	Script string `json:"script" yaml:"script" template:"true"`
+
+	// Connections for AWS/GCP/Azure/K8s credential injection
+	Connections *connection.ExecConnections `json:"connections,omitempty" yaml:"connections,omitempty" template:"true"`
+
+	// Git repository to checkout before running script
+	Checkout *connection.GitConnection `json:"checkout,omitempty" yaml:"checkout,omitempty"`
+
+	// Environment variables
+	Env []types.EnvVar `json:"env,omitempty" yaml:"env,omitempty"`
+
+	// Artifacts to collect after execution
+	Artifacts []shell.Artifact `json:"artifacts,omitempty" yaml:"artifacts,omitempty" template:"true"`
+}

api/v1/scrapeconfig_types.go

@@ -111,6 +111,9 @@ func (t *ScrapeConfig) Type() string {
 	if len(t.Spec.HTTP) != 0 {
 		return "http"
 	}
+	if len(t.Spec.Exec) != 0 {
+		return "exec"
+	}
 	return ""
 }
 

api/v1/types.go

@@ -16,6 +16,7 @@ var AllScraperConfigs = map[string]any{
 	"aws":            AWS{},
 	"azure":          Azure{},
 	"azuredevops":    AzureDevops{},
+	"exec":           Exec{},
 	"file":           File{},
 	"gcp":            GCP{},
 	"githubactions":  GitHubActions{},
@@ -68,6 +69,7 @@ type ScraperSpec struct {
 	Clickhouse     []Clickhouse     `json:"clickhouse,omitempty"`
 	Logs           []Logs           `json:"logs,omitempty"`
 	PubSub         []PubSub         `json:"pubsub,omitempty"`
+	Exec           []Exec           `json:"exec,omitempty" yaml:"exec,omitempty"`
 	System         bool             `json:"system,omitempty"`
 
 	// CRDSync when set to true, will create (or update) the corresponding database record
@@ -140,6 +142,10 @@ func (c ScraperSpec) ApplyPlugin(plugins []ScrapePluginSpec) ScraperSpec {
 		spec.Logs[i].BaseScraper = spec.Logs[i].BaseScraper.ApplyPlugins(plugins...)
 	}
 
+	for i := range spec.Exec {
+		spec.Exec[i].BaseScraper = spec.Exec[i].BaseScraper.ApplyPlugins(plugins...)
+	}
+
 	return *spec
 }