Skip to content

Pr/fix azure #1782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
moshloop wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Pr/fix azure #1782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
994f395
fix: item scraping
moshloop Dec 12, 2025
70b77ed
fix: azure devops pipelines with none string template vars
moshloop Dec 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions scrapers/azure/devops/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,17 @@ type Pipeline struct {
Name string `json:"name"`
Folder string `json:"folder"`
Variables map[string]Variable `json:"variables,omitempty"`
TemplateParameters map[string]string `json:"templateParameters,omitempty"`
TemplateParameters map[string]any `json:"templateParameters,omitempty"`
Runs []v1.ChangeResult `json:"-"`
}

func (p Pipeline) GetLabels() map[string]string {
var labels = p.TemplateParameters
var labels = map[string]string{}

for k, v := range p.TemplateParameters {
labels[k] = fmt.Sprintf("%v", v)

}
for k, v := range p.Variables {
labels[k] = v.Value
}
Expand Down Expand Up @@ -80,7 +85,7 @@ type Variable struct {
type Run struct {
Links map[string]Link `json:"_links,omitempty"`
Variables map[string]Variable `json:"variables,omitempty"`
TemplateParameters map[string]string `json:"templateParameters,omitempty"`
TemplateParameters map[string]any `json:"templateParameters,omitempty"`
State string `json:"state"`
Result string `json:"result"`
CreatedDate time.Time `json:"createdDate"`
Expand All @@ -93,7 +98,10 @@ type Run struct {
}

func (r Run) GetTags() map[string]string {
var tags = r.TemplateParameters
var tags = map[string]string{}
for k, v := range r.TemplateParameters {
tags[k] = fmt.Sprintf("%v", v)
}
for k, v := range r.Variables {
tags[k] = v.Value
}
Expand Down
202 changes: 135 additions & 67 deletions scrapers/processors/json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
"strings"
"time"

"github.com/flanksource/clicky"

Check failure on line 12 in scrapers/processors/json.go

View workflow job for this annotation

GitHub Actions / test-prod 

no required module provides package github.com/flanksource/clicky; to add it:

Check failure on line 12 in scrapers/processors/json.go

View workflow job for this annotation

GitHub Actions / test 

no required module provides package github.com/flanksource/clicky; to add it:
Copy link
Contributor

@coderabbitai coderabbitai bot Dec 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Critical: Missing dependency prevents compilation.

The package github.com/flanksource/clicky is not available in the module dependencies, causing build failures.

Run the following to add the dependency:

#!/bin/bash
# Add the missing dependency
go get github.com/flanksource/clicky
🧰 Tools
🪛 GitHub Actions: Build

[error] 12-12: No required module provides package github.com/flanksource/clicky; to add it: go get github.com/flanksource/clicky

🪛 GitHub Actions: Lint

[error] 12-12: could not import github.com/flanksource/clicky (scrapers/processors/json.go:12:2: no required module provides package github.com/flanksource/clicky; to add it:)

🪛 GitHub Actions: Test

[error] 12-12: no required module provides package github.com/flanksource/clicky; to add it:

🪛 GitHub Check: test

[failure] 12-12:
no required module provides package github.com/flanksource/clicky; to add it:

🪛 GitHub Check: test-prod

[failure] 12-12:
no required module provides package github.com/flanksource/clicky; to add it:

🤖 Prompt for AI Agents 
In scrapers/processors/json.go at line 12 the import
"github.com/flanksource/clicky" is referenced but the module is not present in
go.mod, causing compilation failures; add the dependency to the module by
running `go get github.com/flanksource/clicky` (or manually add the appropriate
version to go.mod and run `go mod tidy`) so the package is available for import
and the project builds successfully.

"github.com/flanksource/commons/collections"
"github.com/flanksource/commons/logger"
"github.com/flanksource/duty"
Expand Down Expand Up @@ -331,10 +332,10 @@
}

func (e Extract) Extract(ctx api.ScrapeContext, inputs ...v1.ScrapeResult) ([]v1.ScrapeResult, error) {
var results []v1.ScrapeResult
var err error

logScrapes := ctx.PropertyOn(true, "log.items")
var results = []v1.ScrapeResult{}

for _, input := range inputs {
for k, v := range input.BaseScraper.Labels {
Expand Down Expand Up @@ -388,7 +389,7 @@
} else if input.Format == "yaml" {
contentByte, err := kyaml.YAMLToJSON([]byte(input.Config.(string)))
if err != nil {
return results, errors.Wrapf(err, "Failed parse yaml %s", input)
return nil, errors.Wrapf(err, "Failed parse yaml %s", input)
}
input.Config = string(contentByte)
} else if input.Format != "" {
Expand All @@ -408,87 +409,54 @@
case string:
parsedConfig, err = oj.ParseString(v)
if err != nil {
return results, fmt.Errorf("failed to parse json (format=%s,%s): %v", input.BaseScraper.Format, input.Source, err)
s := "failed to parse json"
if input.Format != "" {
s += fmt.Sprintf(" format=%s", input.Format)
}
if input.Source != "" {
s += fmt.Sprintf(" source=%s", input.Source)
}
return nil, fmt.Errorf("%s: %v\n%s", s, err, v)
}
default:
opts := oj.Options{OmitNil: input.OmitNil(), Sort: true, UseTags: true, FloatFormat: "%g"}
err = json.Unmarshal([]byte(oj.JSON(v, &opts)), &parsedConfig)
if err != nil {
return results, fmt.Errorf("failed to parse json format=%s,%s): %v", input.Format, input.Source, err)
}
}

if e.Items != nil {
items := e.Items.Get(parsedConfig)
ctx.Logger.V(3).Infof("extracted %d items with %s", len(items), *e.Items)
for _, item := range items {
extracted, err := e.WithoutItems().Extract(ctx, input.Clone(item))
if err != nil {
return results, fmt.Errorf("failed to extract items: %v", err)
s := "failed to parse json"
if input.Format != "" {
s += fmt.Sprintf(" format=%s", input.Format)
}
results = append(results, extracted...)
continue
if input.Source != "" {
s += fmt.Sprintf(" source=%s", input.Source)
}
if logger.V(2).Enabled() {
logger.V(2).Infof(clicky.Text("").Append(input.Config).ANSI())
Copy link
Contributor

@coderabbitai coderabbitai bot Dec 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🏁 Script executed:

wc -l scrapers/processors/json.go

Repository: flanksource/config-db

Length of output: 96

🏁 Script executed:

head -50 scrapers/processors/json.go

Repository: flanksource/config-db

Length of output: 1217

🏁 Script executed:

# Search for ScrapeResult type definition
ast-grep --pattern 'type ScrapeResult struct {
  $$$
}'

Repository: flanksource/config-db

Length of output: 5467

🏁 Script executed:

# Search for Script type definition
ast-grep --pattern 'type Script struct {
  $$$
}'

Repository: flanksource/config-db

Length of output: 499

🏁 Script executed:

# Search for all methods on ScrapeResult
rg 'func \(\w+\s+\*?ScrapeResult\)' -A 2

Repository: flanksource/config-db

Length of output: 2474

🏁 Script executed:

# Search for all methods on Script
rg 'func \(\w+\s+\*?Script\)' -A 2

Repository: flanksource/config-db

Length of output: 869

🏁 Script executed:

# Check clicky package import and usage
rg 'import.*clicky|clicky\.' scrapers/processors/json.go | head -20

Repository: flanksource/config-db

Length of output: 157

🏁 Script executed:

# Read lines around 433
sed -n '425,440p' scrapers/processors/json.go

Repository: flanksource/config-db

Length of output: 503

🏁 Script executed:

# Read lines around 451
sed -n '445,460p' scrapers/processors/json.go

Repository: flanksource/config-db

Length of output: 543

🏁 Script executed:

# Read lines around 535
sed -n '530,545p' scrapers/processors/json.go

Repository: flanksource/config-db

Length of output: 697

🏁 Script executed:

# Read lines around 653
sed -n '648,660p' scrapers/processors/json.go

Repository: flanksource/config-db

Length of output: 459

🏁 Script executed:

# Search for Pretty() method definitions
rg 'func.*Pretty\(\)' -A 2

Repository: flanksource/config-db

Length of output: 47

🏁 Script executed:

# Search for PrettyShort() method definitions
rg 'func.*PrettyShort\(\)' -A 2

Repository: flanksource/config-db

Length of output: 47

🏁 Script executed:

# Search for Debug() method definitions
rg 'func.*Debug\(\)' -A 2

Repository: flanksource/config-db

Length of output: 182

🏁 Script executed:

# Search for any method on ScrapeResult that might be similar
rg 'func \(\w+\s+\*?ScrapeResult\).*\(\)'

Repository: flanksource/config-db

Length of output: 722

🏁 Script executed:

# Search entire codebase for Pretty/Debug method definitions
rg '\bPretty\(' --type go

Repository: flanksource/config-db

Length of output: 401

🏁 Script executed:

# Search for any Debug method on any type
rg 'func.*Debug\(\).*{' --type go

Repository: flanksource/config-db

Length of output: 164

🏁 Script executed:

# Check if there are any interface definitions that might define these methods
rg 'interface.*{' -A 20 | grep -A 20 'Pretty\|Debug'

Repository: flanksource/config-db

Length of output: 152

🏁 Script executed:

# Look at clicky package imports and see what's available
rg 'clicky\.' scrapers/processors/json.go

Repository: flanksource/config-db

Length of output: 157

🏁 Script executed:

# Search for Debug() method more carefully
rg 'Debug\(' --type go | grep 'func'

Repository: flanksource/config-db

Length of output: 118

🏁 Script executed:

# Check if input has Debug method - look at ScrapeResult more carefully
sed -n '700,900p' api/v1/interface.go | grep -i 'debug\|pretty'

Repository: flanksource/config-db

Length of output: 47

🏁 Script executed:

# Search in the entire api/v1 directory for these methods
find api/v1 -name "*.go" -exec grep -l "Pretty\|Debug" {} \;

Repository: flanksource/config-db

Length of output: 80

Critical: Undefined methods prevent compilation.

Multiple method calls reference methods that don't exist on the respective types:

  • Line 451: ScrapeResult has no Pretty() method
  • Line 535: Script has no PrettyShort() method
  • Line 541: Script has no Pretty() method
  • Lines 542, 653: ScrapeResult has no Debug() method

These methods need to be implemented or the calls need to be replaced with alternative approaches.

Also applies to: 451-451, 535-535, 541-542, 653-653

}
return nil, fmt.Errorf("%s: %v", s, err)
}
}

input.Config = parsedConfig
var ongoingInput v1.ScrapeResults = []v1.ScrapeResult{input}
if !input.BaseScraper.Transform.Script.IsEmpty() {
ctx.Logger.V(3).Infof("Applying script transformation")
transformed, err := RunScript(ctx, input, input.BaseScraper.Transform.Script)
if err != nil {
return results, fmt.Errorf("failed to run transform script: %v", err)
}

ongoingInput = transformed
currentResults, err := e.transform(ctx, input)
if err != nil {
return nil, err
}
currentResults, err = e.extractItems(ctx, currentResults)

for _, result := range ongoingInput {
for i, configProperty := range result.BaseScraper.Properties {
if configProperty.Filter != "" {
if response, err := gomplate.RunTemplate(result.AsMap(), gomplate.Template{Expression: configProperty.Filter}); err != nil {
result.Errorf("failed to parse filter: %v", err)
continue
} else if boolVal, err := strconv.ParseBool(response); err != nil {
result.Errorf("expected a boolean but property filter returned (%s)", response)
continue
} else if !boolVal {
continue
}
}

// clone the links so as to not mutate the original Links template
configProperty.Links = make([]types.Link, len(result.BaseScraper.Properties[i].Links))
copy(configProperty.Links, result.BaseScraper.Properties[i].Links)

templater := gomplate.StructTemplater{
Values: result.AsMap(),
ValueFunctions: true,
DelimSets: []gomplate.Delims{
{Left: "{{", Right: "}}"},
{Left: "$(", Right: ")"},
},
}

if err := templater.Walk(&configProperty); err != nil {
result.Errorf("failed to template scraper properties: %v", err)
continue
}

result.Properties = append(result.Properties, &configProperty.Property)
}

for _, result := range currentResults {
result.Properties = e.parseProperties(ctx, &result)
extracted, err := e.extractAttributes(result)
if err != nil {
if ctx.IsDebug() || logger.V(2).Enabled() {
ctx.Infof("%s", result.Pretty().ANSI())

Check failure on line 451 in scrapers/processors/json.go

View workflow job for this annotation

GitHub Actions / lint 

result.Pretty undefined (type "github.com/flanksource/config-db/api/v1".ScrapeResult has no field or method Pretty) (typecheck)
}
return results, fmt.Errorf("failed to extract attributes: %v", err)
}

if logScrapes {
ctx.Logger.V(2).Infof("Scraped %s", extracted)
}

extracted = extracted.SetHealthIfEmpty()

// Form new relationships based on the transform configs
if newRelationships, err := getRelationshipsFromRelationshipConfigs(ctx, extracted, e.Transform.Relationship); err != nil {
return results, fmt.Errorf("failed to get relationships from relationship configs: %w", err)
Expand All @@ -499,15 +467,108 @@
results = append(results, extracted)
}

if !input.BaseScraper.Transform.Masks.IsEmpty() {
results, err = e.applyMask(results)
}

return e.postProcess(ctx, results)
}

func (e Extract) parseProperties(ctx api.ScrapeContext, result *v1.ScrapeResult) types.Properties {
properties := types.Properties{}
for i, configProperty := range e.Config.Properties {
if configProperty.Filter != "" {
if response, err := gomplate.RunTemplate(result.AsMap(), gomplate.Template{Expression: configProperty.Filter}); err != nil {
result.Errorf("failed to parse filter: %v", err)
continue
} else if boolVal, err := strconv.ParseBool(response); err != nil {
result.Errorf("expected a boolean but property filter returned (%s)", response)
continue
} else if !boolVal {
continue
}
}

// clone the links so as to not mutate the original Links template
configProperty.Links = make([]types.Link, len(result.BaseScraper.Properties[i].Links))
copy(configProperty.Links, result.BaseScraper.Properties[i].Links)

templater := gomplate.StructTemplater{
Values: result.AsMap(),
ValueFunctions: true,
DelimSets: []gomplate.Delims{
{Left: "{{", Right: "}}"},
{Left: "$(", Right: ")"},
},
}

if err := templater.Walk(&configProperty); err != nil {
result.Errorf("failed to template scraper properties: %v", err)
continue
}

properties = append(properties, &configProperty.Property)
}
return properties
}

func (e Extract) extractItems(ctx api.ScrapeContext, inputs []v1.ScrapeResult) ([]v1.ScrapeResult, error) {
if e.Items == nil {
return inputs, nil
}
var results = []v1.ScrapeResult{}
for _, input := range inputs {
items := e.Items.Get(input.Config)

for _, item := range items {
extracted, err := e.WithoutItems().Extract(ctx, input.Clone(item))
if err != nil {
return results, fmt.Errorf("e.applyMask(); %w", err)
return results, fmt.Errorf("failed to extract items: %v", err)
}
results = append(results, extracted...)
}
}
return results, nil
}

func (e Extract) transform(ctx api.ScrapeContext, input v1.ScrapeResult) ([]v1.ScrapeResult, error) {
if !e.Config.Transform.Script.IsEmpty() {
if logger.V(5).Enabled() {
ctx.Logger.V(5).Infof("Applying script transformation: %s", e.Config.Transform.Script.PrettyShort().ANSI())

Check failure on line 535 in scrapers/processors/json.go

View workflow job for this annotation

GitHub Actions / lint 

e.Config.Transform.Script.PrettyShort undefined (type "github.com/flanksource/config-db/api/v1".Script has no field or method PrettyShort) (typecheck)
}
transformed, err := RunScript(ctx, input, e.Config.Transform.Script)
if err != nil {
if ctx.IsDebug() || ctx.Logger.IsDebugEnabled() {
t := clicky.Text("")
t = t.Append(e.Config.Transform.Script.Pretty()).

Check failure on line 541 in scrapers/processors/json.go

View workflow job for this annotation

GitHub Actions / lint 

e.Config.Transform.Script.Pretty undefined (type "github.com/flanksource/config-db/api/v1".Script has no field or method Pretty) (typecheck)
NewLine().Append(input.Debug())

Check failure on line 542 in scrapers/processors/json.go

View workflow job for this annotation

GitHub Actions / lint 

input.Debug undefined (type "github.com/flanksource/config-db/api/v1".ScrapeResult has no field or method Debug) (typecheck)
if !ctx.Logger.IsDebugEnabled() {
ctx.Logger.Infof(t.ANSI())
} else {
ctx.Logger.Debugf(t.ANSI())
}
return nil, fmt.Errorf("failed to run transform script: %v", err)
} else {
return nil, fmt.Errorf("failed to run transform script: %v", err)
}

}
return transformed, nil
}
return []v1.ScrapeResult{input}, nil
}

func (e Extract) postProcess(ctx api.ScrapeContext, results v1.ScrapeResults) (v1.ScrapeResults, error) {

if !e.Config.Transform.Masks.IsEmpty() {
results, err := e.applyMask(results)
if err != nil {
return results, fmt.Errorf("e.applyMask(); %w", err)
}
}

for i, result := range results {

results[i] = result.SetHealthIfEmpty()

env := result.AsMap()

if val, err := extractLocation(ctx, env, e.Transform.Locations); err != nil {
Expand All @@ -522,7 +583,6 @@
results[i].Aliases = append(results[i].Aliases, val...)
}
}

return results, nil
}

Expand Down Expand Up @@ -589,7 +649,15 @@
}

if input.ID == "" {
return input, fmt.Errorf("no id defined for: %s: %v", input, e.Config)
if len(input.Changes) == 0 {
return input, fmt.Errorf("no id defined for: %s", input.Debug().ANSI())

Check failure on line 653 in scrapers/processors/json.go

View workflow job for this annotation

GitHub Actions / lint 

input.Debug undefined (type "github.com/flanksource/config-db/api/v1".ScrapeResult has no field or method Debug) (typecheck)
}
if len(lo.Filter(input.Changes, func(c v1.ChangeResult, _ int) bool {
return c.ExternalChangeID == ""
})) > 0 {
return input, fmt.Errorf("standalone changes must have both an `external_id` and `external_change_id`: %s: %v", input, e.Config)

}
}

if input.Name == "" {
Expand Down
Loading