Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cache limits for resources and attributes #509

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Add cache limits for resources and attributes #509

Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d9a651d
Add cache limits for resources and attributes
srikanthccv Jan 16, 2025
ee51a19
Add duration checks
srikanthccv Jan 16, 2025
e7da9c0
Add log when limits exceed
srikanthccv Jan 16, 2025
67f9f5d
Revert "Add log when limits exceed"
srikanthccv Jan 16, 2025
9cfd120
Adjust logs
srikanthccv Jan 16, 2025
6cc8932
More pipeline and better count
srikanthccv Jan 16, 2025
a4dd4ee
Remove duplicates
srikanthccv Jan 16, 2025
2889b38
Disable debug by default
srikanthccv Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add duration checks
  • Loading branch information
@srikanthccv
srikanthccv committed Jan 16, 2025
commit ee51a197e969bab1350a08c68b29eecfd430bef1
29 changes: 24 additions & 5 deletions exporter/metadataexporter/exporter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,15 +468,30 @@ func (e *metadataExporter) shouldSkipAttributeUVT(_ context.Context, key, dataso

func (e *metadataExporter) writeToStatementBatch(ctx context.Context, stmt driver.Batch, records []writeToStatementBatchRecord, ds pipeline.Signal) (int, error) {

var existsCheckDuration, addAttrsDuration time.Duration
var existsCheckDuration, addAttrsDuration, resourcesLimitCheckDuration, totalCardinalityLimitCheckDuration time.Duration
resourcesLimitCheckStart := time.Now()
// check max resources limit
if e.keyCache.ResourcesLimitExceeded(ctx, ds) {
return 0, fmt.Errorf("resource limit exceeded")
}
resourcesLimitCheckDuration = time.Since(resourcesLimitCheckStart)

totalCardinalityLimitCheckStart := time.Now()
if e.keyCache.TotalCardinalityLimitExceeded(ctx, ds) {
return 0, fmt.Errorf("total cardinality limit exceeded")
}
totalCardinalityLimitCheckDuration = time.Since(totalCardinalityLimitCheckStart)

e.set.Logger.Info("resourcesLimitCheckDuration",
zap.Int64("duration", resourcesLimitCheckDuration.Milliseconds()),
zap.String("datasource", ds.String()),
zap.Int("records", len(records)),
)
e.set.Logger.Info("totalCardinalityLimitCheckDuration",
zap.Int64("duration", totalCardinalityLimitCheckDuration.Milliseconds()),
zap.String("datasource", ds.String()),
zap.Int("records", len(records)),
)

// Group by resourceFingerprint
// resourceFp -> slice of attributeFp
Expand All @@ -488,7 +503,7 @@ func (e *metadataExporter) writeToStatementBatch(ctx context.Context, stmt drive
indexByFp[resourceFp] = append(indexByFp[resourceFp], i)
}

e.set.Logger.Info("resourceGroupsCount", zap.Int("count", len(recordGroups)))
e.set.Logger.Info("resourceGroupsCount", zap.Int("count", len(recordGroups)), zap.String("datasource", ds.String()), zap.Int("records", len(records)))

totalWrites := 0

Expand Down Expand Up @@ -545,15 +560,19 @@ func (e *metadataExporter) writeToStatementBatch(ctx context.Context, stmt drive
}
}

e.set.Logger.Info("existsCheckDuration", zap.Int64("duration", existsCheckDuration.Milliseconds()))
e.set.Logger.Info("addAttrsDuration", zap.Int64("duration", addAttrsDuration.Milliseconds()))
e.set.Logger.Info("existsCheckDuration", zap.Int64("duration", existsCheckDuration.Milliseconds()), zap.String("datasource", ds.String()), zap.Int("records", len(records)))
e.set.Logger.Info("addAttrsDuration", zap.Int64("duration", addAttrsDuration.Milliseconds()), zap.String("datasource", ds.String()), zap.Int("records", len(records)))

stmtStart := time.Now()
if err := stmt.Send(); err != nil {
return totalWrites, err
}
stmtDuration := time.Since(stmtStart)
e.set.Logger.Info("stmtDuration", zap.Int64("duration", stmtDuration.Milliseconds()))
e.set.Logger.Info("stmtDuration",
zap.Int64("duration", stmtDuration.Milliseconds()),
zap.String("datasource", ds.String()),
zap.Int("records", len(records)),
)

return totalWrites, nil
}
Expand Down
Loading