Pr/GitHub security

.gitignore

@@ -25,4 +25,8 @@ external/diffgen/target
 
 # For AI agents
 TODO.md
-.envrc
+.db/
+db/
+.DS_Store
+out/
+bin/

api/context.go

@@ -161,3 +161,7 @@ func (ctx ScrapeContext) Namespace() string {
 func (ctx ScrapeContext) IsTrace() bool {
 	return ctx.scrapeConfig.Spec.IsTrace()
 }
+
+func (ctx ScrapeContext) IsDebug() bool {
+	return ctx.scrapeConfig.Spec.IsDebug()
+}

api/v1/azure_ad.go

@@ -0,0 +1,34 @@
+package v1
+
+type AzureAD struct {
+	BaseScraper      `json:",inline"`
+	AzureConnection  `yaml:",inline" json:",inline"`
+	Users            AzureUsers            `yaml:"users,omitempty" json:"users,omitempty"`
+	Groups           AzureGroups           `yaml:"groups,omitempty" json:"groups,omitempty"`
+	AppRegistrations AzureAppRegistrations `yaml:"appRegistrations,omitempty" json:"appRegistrations,omitempty"`
+	Logins           AzureLogins           `yaml:"logins,omitempty" json:"logins,omitempty"`
+}
+
+type CELFilter string
+
+type MsGraphFilter struct {
+	Filter []CELFilter `yaml:"filter,omitempty" json:"filter,omitempty"`
+	// MS.Graph query string
+	Query string `yaml:"query,omitempty" json:"query,omitempty"`
+}
+
+type AzureLogins struct {
+	MsGraphFilter `yaml:",inline" json:",inline"`
+}
+
+type AzureUsers struct {
+	MsGraphFilter `yaml:",inline" json:",inline"`
+}
+
+type AzureGroups struct {
+	MsGraphFilter `yaml:",inline" json:",inline"`
+}
+
+type AzureAppRegistrations struct {
+	MsGraphFilter `yaml:",inline" json:",inline"`
+}

api/v1/common.go

@@ -6,11 +6,14 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/flanksource/clicky"
+	"github.com/flanksource/clicky/api"
 	"github.com/flanksource/duty"
 	"github.com/flanksource/duty/connection"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/types"
 	"github.com/flanksource/gomplate/v3"
+	"github.com/samber/lo"
 )
 
 // List of types which should not have scraper_id
@@ -63,6 +66,34 @@ func (s Script) String() string {
 	return ""
 }
 
+func (s Script) PrettyShort() api.Text {
+	t := clicky.Text("")
+	if s.GoTemplate != "" {
+		t = t.Append("go: ", "text-green-600").Append(clicky.CodeBlock("go", lo.Ellipsis(s.GoTemplate, 200)))
+	} else if s.JSONPath != "" {
+		t = t.Append("jsonpath: ", "text-blue-600").Append(clicky.CodeBlock("jsonpath", lo.Ellipsis(s.JSONPath, 200)))
+	} else if s.Expression != "" {
+		t = t.Append("expr: ", "text-yellow-600").Append(clicky.CodeBlock("cel", lo.Ellipsis(s.Expression, 200)))
+	} else if s.Javascript != "" {
+		t = t.Append("js: ", "text-purple-600").Append(clicky.CodeBlock("javascript", lo.Ellipsis(s.Javascript, 200)))
+	}
+	return t
+}
+
+func (s Script) Pretty() api.Text {
+	t := clicky.Text("")
+	if s.GoTemplate != "" {
+		t = t.Append("go: ", "text-green-600").Append(clicky.CodeBlock("go", s.GoTemplate))
+	} else if s.JSONPath != "" {
+		t = t.Append("jsonpath: ", "text-blue-600").Append(clicky.CodeBlock("jsonpath", s.JSONPath))
+	} else if s.Expression != "" {
+		t = t.Append("expr: ", "text-yellow-600").Append(clicky.CodeBlock("cel", s.Expression))
+	} else if s.Javascript != "" {
+		t = t.Append("js: ", "text-purple-600").Append(clicky.CodeBlock("javascript", s.Javascript))
+	}
+	return t
+}
+
 type Mask struct {
 	// Selector is a CEL expression that selects on what config items to apply the mask.
 	Selector string `json:"selector,omitempty"`
@@ -361,6 +392,28 @@ func (aws AWSConnection) ToDutyAWSConnection(region string) *connection.AWSConne
 	}
 }
 
+// GCPConnection ...
+type GCPConnection struct {
+	Endpoint    string        `yaml:"endpoint" json:"endpoint,omitempty"`
+	Credentials *types.EnvVar `yaml:"credentials" json:"credentials,omitempty"`
+}
+
+func (gcp GCPConnection) GetModel() *models.Connection {
+	return &models.Connection{
+		URL:         gcp.Endpoint,
+		Certificate: gcp.Credentials.String(),
+	}
+}
+
+type AzureConnection struct {
+	ConnectionName string       `yaml:"connection,omitempty" json:"connection,omitempty"`
+	SubscriptionID string       `yaml:"subscriptionID" json:"subscriptionID"`
+	Organisation   string       `yaml:"organisation" json:"organisation"`
+	ClientID       types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"`
+	ClientSecret   types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"`
+	TenantID       string       `yaml:"tenantID,omitempty" json:"tenantID,omitempty"`
+}
+
 type Connection struct {
 	// Connection is either the name of the connection to lookup
 	// or the connection string itself.
@@ -389,6 +442,15 @@ func (c Connection) GetEndpoint() string {
 	return sanitizeEndpoints(c.Connection)
 }
 
+func (c Connection) Pretty() api.Text {
+	t := clicky.Text("")
+	if c.Connection != "" {
+		clicky.RedactSecretValues()
+		t = t.Append(sanitizeEndpoints(c.Connection))
+	}
+	return t
+}
+
 // Obfuscate passwords of the form ' password=xxxxx ' from connectionString since
 // connectionStrings are used as metric labels and we don't want to leak passwords
 // Returns the Connection string with the password replaced by '###'

api/v1/github_security.go

@@ -0,0 +1,53 @@
+package v1
+
+import (
+	"time"
+
+	"github.com/flanksource/duty/types"
+)
+
+// GitHubSecurity scraper fetches security alerts from GitHub repositories
+// including Dependabot alerts, code scanning alerts, secret scanning alerts,
+// and security advisories.
+type GitHubSecurity struct {
+	BaseScraper `json:",inline" yaml:",inline"`
+
+	// Repositories is the list of repositories to scan
+	Repositories []GitHubSecurityRepository `yaml:"repositories" json:"repositories"`
+
+	// PersonalAccessToken for GitHub API authentication
+	// Required scopes: repo (full) or security_events (read)
+	PersonalAccessToken types.EnvVar `yaml:"personalAccessToken,omitempty" json:"personalAccessToken,omitempty"`
+
+	// ConnectionName, if provided, will be used to populate personalAccessToken
+	ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
+
+	// Filters for security alerts
+	Filters GitHubSecurityFilters `yaml:"filters,omitempty" json:"filters,omitempty"`
+}
+
+// GitHubSecurityRepository specifies a repository to scan
+type GitHubSecurityRepository struct {
+	Owner string `yaml:"owner" json:"owner"`
+	Repo  string `yaml:"repo" json:"repo"`
+}
+
+// GitHubSecurityFilters defines filtering options for security alerts
+type GitHubSecurityFilters struct {
+	// Severity filters: critical, high, medium, low
+	Severity []string `yaml:"severity,omitempty" json:"severity,omitempty"`
+
+	// State filters: open, closed, dismissed, fixed
+	State []string `yaml:"state,omitempty" json:"state,omitempty"`
+
+	// MaxAge filters alerts by age (e.g., "90d", "30d")
+	MaxAge string `yaml:"maxAge,omitempty" json:"maxAge,omitempty"`
+}
+
+// ParseMaxAge converts the MaxAge string to a time.Duration
+func (f GitHubSecurityFilters) ParseMaxAge() (time.Duration, error) {
+	if f.MaxAge == "" {
+		return 0, nil
+	}
+	return time.ParseDuration(f.MaxAge)
+}