Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

lambda-promtail: Add ability to ingest logs from S3 #5065

Merged
merged 26 commits into from
Feb 4, 2022
Merged

lambda-promtail: Add ability to ingest logs from S3 #5065

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
fd80dd0
Add ability to ingest logs from S3 on lambda-promtail
AndreZiviani Jan 6, 2022
a2fec89
fix ci
AndreZiviani Jan 6, 2022
774c5e8
fix typo
AndreZiviani Jan 6, 2022
611d604
bump golang and alpine version
AndreZiviani Jan 6, 2022
dc1b849
update changelog
AndreZiviani Jan 7, 2022
b42243e
add s3 permissions on terraform
AndreZiviani Jan 7, 2022
0ab4647
use for_each instead of count
AndreZiviani Jan 7, 2022
394b6c7
fix typo
AndreZiviani Jan 7, 2022
74f88ac
improve function naming
AndreZiviani Jan 13, 2022
ea56137
add documentation and an example of a s3 file path
AndreZiviani Jan 24, 2022
8331cc1
refact logic to identify event type
AndreZiviani Jan 24, 2022
373eb18
add missing iam permission to allow lambda to run inside a vpc
AndreZiviani Jan 24, 2022
e57b6f6
fix typo
AndreZiviani Jan 24, 2022
bdfda42
allow lambda to access only specified s3 buckets
AndreZiviani Jan 26, 2022
5e75cea
configure a default log retention policy on log group
AndreZiviani Jan 26, 2022
df9eceb
add missing depends_on to make sure iam role is created before lambda…
AndreZiviani Jan 26, 2022
5cfd06a
update docs
AndreZiviani Jan 31, 2022
6e5d6a3
fix label naming convention
AndreZiviani Jan 31, 2022
3b025ff
fix merge conflicts
AndreZiviani Jan 31, 2022
227f77c
fix merge conflict
AndreZiviani Jan 31, 2022
b99d800
use new backoff lib and update dependencies
AndreZiviani Jan 31, 2022
2c241b7
add option to limit batch size
AndreZiviani Feb 3, 2022
b443ebc
cache s3 client
AndreZiviani Feb 3, 2022
7d3c4f8
update docs and terraform
AndreZiviani Feb 3, 2022
370f116
address some feedback on PR
AndreZiviani Feb 4, 2022
b31063a
fix typo
AndreZiviani Feb 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
improve function naming
  • Loading branch information
@AndreZiviani
AndreZiviani committed Jan 13, 2022
commit 74f88ac6ed5fdc7a3cd1b2dee61fc4fb2e47a593
8 changes: 4 additions & 4 deletions tools/lambda-promtail/lambda-promtail/cw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"github.com/prometheus/common/model"
)

func checkIfCWEvent(ev map[string]interface{}) (events.CloudwatchLogsEvent, error) {
func convertToCWEvent(ev map[string]interface{}) (events.CloudwatchLogsEvent, error) {
var cw events.CloudwatchLogsEvent

j, _ := json.Marshal(ev)
Expand All @@ -25,7 +25,7 @@ func checkIfCWEvent(ev map[string]interface{}) (events.CloudwatchLogsEvent, erro
return cw, err
}

func createCWStream(ctx context.Context, b *batch, ev events.CloudwatchLogsEvent) error {
func parseCWEvent(ctx context.Context, b *batch, ev events.CloudwatchLogsEvent) error {
data, err := ev.AWSLogs.Parse()
if err != nil {
fmt.Println("error parsing log event: ", err)
Expand All @@ -52,10 +52,10 @@ func createCWStream(ctx context.Context, b *batch, ev events.CloudwatchLogsEvent
return nil
}

func processCW(ctx context.Context, ev events.CloudwatchLogsEvent) error {
func processCWEvent(ctx context.Context, ev events.CloudwatchLogsEvent) error {
batch := newBatch()

err := createCWStream(ctx, batch, ev)
err := parseCWEvent(ctx, batch, ev)
if err != nil {
return err
}
Expand Down
8 changes: 4 additions & 4 deletions tools/lambda-promtail/lambda-promtail/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,14 @@ func init() {
}

func handler(ctx context.Context, ev map[string]interface{}) error {
s3event, err := checkIfS3Event(ev)
s3event, err := convertToS3Event(ev)
if err == nil {
return processS3(ctx, s3event)
return processS3Event(ctx, s3event)
}

cwevent, err := checkIfCWEvent(ev)
cwevent, err := convertToCWEvent(ev)
if err == nil {
return processCW(ctx, cwevent)
return processCWEvent(ctx, cwevent)
}

fmt.Println("invalid event: %s", ev)
Expand Down
4 changes: 2 additions & 2 deletions tools/lambda-promtail/lambda-promtail/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func parseS3Log(b *batch, labels map[string]string, obj io.ReadCloser) error {
return nil
}

func checkIfS3Event(ev map[string]interface{}) (events.S3Event, error) {
func convertToS3Event(ev map[string]interface{}) (events.S3Event, error) {
var s3 events.S3Event

j, _ := json.Marshal(ev)
Expand Down Expand Up @@ -115,7 +115,7 @@ func getLabels(record events.S3EventRecord) (map[string]string, error) {
return labels, nil
}

func processS3(ctx context.Context, ev events.S3Event) error {
func processS3Event(ctx context.Context, ev events.S3Event) error {

batch := newBatch()

Expand Down