diff --git a/CHANGELOG.md b/CHANGELOG.md
index 578463ff..22720c3e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
# Changelog
+## v1.0.78
+### 🚀 New components 🚀
+#### **coralogix-aws-shipper**
+- Add option to use Sqs with out without s3 bucket
+
## v1.0.77
### 🧰 Bug fixes 🧰
#### **coralogix-aws-shipper**
diff --git a/examples/coralogix-aws-shipper/variables.tf b/examples/coralogix-aws-shipper/variables.tf
index 29e4093a..68dc106b 100644
--- a/examples/coralogix-aws-shipper/variables.tf
+++ b/examples/coralogix-aws-shipper/variables.tf
@@ -161,3 +161,9 @@ variable "lambda_name" {
description = "The name of the lambda function"
default = null
}
+
+variable "sqs_name" {
+ description = "The name of the SQS that you want watch"
+ type = string
+ default = null
+}
diff --git a/modules/coralogix-aws-shipper/README.md b/modules/coralogix-aws-shipper/README.md
index 49f2a236..eef8b5d9 100644
--- a/modules/coralogix-aws-shipper/README.md
+++ b/modules/coralogix-aws-shipper/README.md
@@ -53,11 +53,17 @@ Coralogix provides a predefined AWS Lambda function to easily forward your logs
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| [log\_groups](#input\_log\_groups) | The names of the CloudWatch log groups to watch | `list(string)` | n/a | yes |
+
### Integration SNS configuration
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| [sns_topic_name](#input\_sns\_topic\_name) | The SNS topic that will contain the SNS subscription, need only if you use the sns integration | `string` | n/a | no |
+### Integration SQS configuration
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [sqs_topic_name](#input\_sqs\_topic\_name) | The SQS name queue to subscribe to retrieving messages| `string` | n/a | no |
+
### Integration Generic Config (Optional)
| Name | Description | Type | Default | Required |
diff --git a/modules/coralogix-aws-shipper/main.tf b/modules/coralogix-aws-shipper/main.tf
index 67561e5b..7a807429 100644
--- a/modules/coralogix-aws-shipper/main.tf
+++ b/modules/coralogix-aws-shipper/main.tf
@@ -14,7 +14,8 @@ locals {
api_key_is_arn = replace(var.api_key, ":", "") != var.api_key ? true : false
- secret_access_policy = var.store_api_key_in_secrets_manager || local.api_key_is_arn ? {
+ ### these 2 policies repeat in the lambda module so I put them as locals to sort the lambda module code
+ secret_access_policy = var.store_api_key_in_secrets_manager || local.api_key_is_arn ? {
effect = "Allow"
actions = ["secretsmanager:GetSecretValue"]
resources = local.api_key_is_arn ? [var.api_key] : [aws_secretsmanager_secret.coralogix_secret[0].arn]
@@ -28,6 +29,10 @@ locals {
actions = ["sns:publish"]
resources = [aws_sns_topic.this.arn]
}
+
+ is_s3_integration = var.integration_type == "S3" || var.integration_type == "CloudTrail" || var.integration_type == "VpcFlow" ? true : false
+ is_sns_integration = local.sns_enable && (var.integration_type == "S3" || var.integration_type == "Sns" || var.integration_type == "CloudTrail" ) ? true : false
+ is_sqs_integration = var.sqs_name != null && (var.integration_type == "S3" || var.integration_type == "CloudTrail" || var.integration_type == "Sqs") ? true : false
}
module "locals" {
@@ -56,8 +61,13 @@ data "aws_sns_topic" "sns_topic" {
name = var.sns_topic_name
}
+data "aws_sqs_queue" "name" {
+ count = var.sqs_name != null ? 1 : 0
+ name = var.sqs_name
+}
+
data "aws_iam_policy_document" "topic" {
- count = local.sns_enable && var.integration_type != "CloudWatch" && var.integration_type != "Sns" ? 1 : 0
+ count = (local.sns_enable || var.sqs_name != null) && local.is_s3_integration ? 1 : 0
statement {
effect = "Allow"
@@ -66,8 +76,8 @@ data "aws_iam_policy_document" "topic" {
identifiers = ["s3.amazonaws.com"]
}
- actions = ["SNS:Publish"]
- resources = ["arn:aws:sns:*:*:${data.aws_sns_topic.sns_topic[count.index].name}"]
+ actions = local.sns_enable ? ["SNS:Publish"] : ["SQS:SendMessage"]
+ resources = local.sns_enable ? ["arn:aws:sns:*:*:${data.aws_sns_topic.sns_topic[count.index].name}"] : ["arn:aws:sqs:*:*:${data.aws_sqs_queue.name[count.index].name}"]
condition {
test = "ArnLike"
@@ -89,14 +99,6 @@ resource "null_resource" "s3_bucket_copy" {
}
}
-resource "aws_lambda_permission" "cloudwatch_trigger_premission" {
- for_each = local.log_groups
- action = "lambda:InvokeFunction"
- function_name = var.lambda_name == null ? module.locals.function_name : var.lambda_name
- principal = "logs.amazonaws.com"
- source_arn = "${data.aws_cloudwatch_log_group.this[each.key].arn}:*"
-}
-
module "lambda" {
depends_on = [null_resource.s3_bucket_copy]
source = "terraform-aws-modules/lambda/aws"
@@ -134,7 +136,7 @@ module "lambda" {
cloudwatch_logs_retention_in_days = var.lambda_log_retention
create_current_version_allowed_triggers = false
attach_policy_statements = true
- policy_statements = var.integration_type != "CloudWatch" && var.integration_type != "Sns" ? {
+ policy_statements = local.is_s3_integration && var.sqs_name == null ? {
S3 = {
effect = "Allow"
actions = ["s3:GetObject"]
@@ -142,12 +144,41 @@ module "lambda" {
}
secret_permission = local.secret_access_policy
destination_on_failure_policy = local.destination_on_failure_policy
- } : {
+ } : var.sqs_name != null ? {
+ SQS = {
+ effect = "Allow"
+ actions = [
+ "sqs:ReceiveMessage",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes"
+ ]
+ resources = [data.aws_sqs_queue.name[0].arn]
+ }
+ S3_SQS = local.is_s3_integration ? {
+ effect = "Allow"
+ actions = [
+ "s3:GetObject",
+ "s3:ListBucket",
+ "s3:GetBucketLocation",
+ "s3:GetObjectVersion",
+ "s3:GetLifecycleConfiguration"
+ ]
+ resources = ["${data.aws_s3_bucket.this[0].arn}/*", data.aws_s3_bucket.this[0].arn]
+ } : { ### can't leave this as empty so we add a deny statement to s3 as you dont need access to it if you dont use s3 integration
+ effect = "Deny"
+ actions = [
+ "s3:GetObject"
+ ]
+ resources = ["*"]
+ }
+ secret_permission = local.secret_access_policy
+ destination_on_failure_policy = local.destination_on_failure_policy
+ } : {
secret_permission = local.secret_access_policy
destination_on_failure_policy = local.destination_on_failure_policy
}
- allowed_triggers = var.integration_type != "CloudWatch" && local.sns_enable != true ? {
+ allowed_triggers = local.is_s3_integration && local.sns_enable != true ? {
AllowExecutionFromS3 = {
principal = "s3.amazonaws.com"
source_arn = data.aws_s3_bucket.this[0].arn
@@ -174,7 +205,7 @@ resource "aws_lambda_function_event_invoke_config" "invoke_on_failure" {
###################################
resource "aws_s3_bucket_notification" "lambda_notification" {
- count = var.integration_type == "CloudWatch" ? 0 : local.sns_enable == false ? 1 : 0
+ count = local.is_s3_integration && local.sns_enable != true && var.sqs_name == null? 1 : 0
depends_on = [ module.lambda ]
bucket = data.aws_s3_bucket.this[0].bucket
lambda_function {
@@ -189,6 +220,14 @@ resource "aws_s3_bucket_notification" "lambda_notification" {
#### cloudwatch integration resources ####
###########################################
+resource "aws_lambda_permission" "cloudwatch_trigger_premission" {
+ for_each = local.log_groups
+ action = "lambda:InvokeFunction"
+ function_name = var.lambda_name == null ? module.locals.function_name : var.lambda_name
+ principal = "logs.amazonaws.com"
+ source_arn = "${data.aws_cloudwatch_log_group.this[each.key].arn}:*"
+}
+
resource "aws_cloudwatch_log_subscription_filter" "this" {
depends_on = [aws_lambda_permission.cloudwatch_trigger_premission]
@@ -199,31 +238,76 @@ resource "aws_cloudwatch_log_subscription_filter" "this" {
filter_pattern = ""
}
+####################################
+#### SNS integration resources ####
+####################################
+
+resource "aws_s3_bucket_notification" "topic_notification" {
+ count = local.sns_enable == true && (var.integration_type == "S3" || var.integration_type == "CloudTrail" ) ? 1 : 0
+ bucket = data.aws_s3_bucket.this[0].bucket
+ topic {
+ topic_arn = data.aws_sns_topic.sns_topic[0].arn
+ events = ["s3:ObjectCreated:*"]
+ filter_prefix = var.s3_key_prefix != null || var.integration_type != "CloudTrail" ? var.s3_key_prefix : "AWSLogs/"
+ filter_suffix = var.integration_type != "CloudTrail" || var.s3_key_suffix != null ? var.s3_key_suffix : lookup(local.s3_suffix_map, var.integration_type)
+ }
+}
+
resource "aws_sns_topic" "this" {
name_prefix = var.lambda_name == null ? "${module.locals.function_name}-Failure" : "${var.lambda_name}-Failure"
display_name = var.lambda_name == null ? "${module.locals.function_name}-Failure" : "${var.lambda_name}-Failure"
tags = merge(var.tags, module.locals.tags)
}
-resource "aws_sns_topic_subscription" "this" {
- depends_on = [aws_sns_topic.this]
- count = var.notification_email != null ? 1 : 0
- topic_arn = aws_sns_topic.this.arn
- protocol = "email"
- endpoint = var.notification_email
+resource "aws_sns_topic_subscription" "lambda_sns_subscription" {
+ count = local.sns_enable ? 1 : 0
+ depends_on = [module.lambda]
+ topic_arn = data.aws_sns_topic.sns_topic[count.index].arn
+ protocol = "lambda"
+ endpoint = module.lambda.lambda_function_arn
}
-resource "aws_s3_bucket_notification" "topic_notification" {
- count = local.sns_enable == true && (var.integration_type == "S3" || var.integration_type == "CloudTrail" ) ? 1 : 0
+####################################
+#### SQS integration resources ####
+####################################
+
+resource "aws_s3_bucket_notification" "sqs_notification" {
+ count = var.sqs_name != null && (var.integration_type == "S3" || var.integration_type == "CloudTrail" ) ? 1 : 0
bucket = data.aws_s3_bucket.this[0].bucket
- topic {
- topic_arn = data.aws_sns_topic.sns_topic[0].arn
+ queue {
+ queue_arn = data.aws_sqs_queue.name[0].arn
events = ["s3:ObjectCreated:*"]
filter_prefix = var.s3_key_prefix != null || var.integration_type != "CloudTrail" ? var.s3_key_prefix : "AWSLogs/"
filter_suffix = var.integration_type != "CloudTrail" || var.s3_key_suffix != null ? var.s3_key_suffix : lookup(local.s3_suffix_map, var.integration_type)
}
}
+resource "aws_lambda_event_source_mapping" "sqs" {
+ count = local.is_sqs_integration ? 1 : 0
+ event_source_arn = data.aws_sqs_queue.name[0].arn
+ function_name = module.lambda.lambda_function_name
+ enabled = true
+}
+
+resource "aws_sqs_queue_policy" "sqs_policy" {
+ count = local.is_s3_integration && var.sqs_name != null ? 1 : 0
+ queue_url = data.aws_sqs_queue.name[count.index].id
+ policy = data.aws_iam_policy_document.topic[count.index].json
+}
+
+
+####################################
+###lambda integration resources ###
+####################################
+
+resource "aws_sns_topic_subscription" "this" {
+ depends_on = [aws_sns_topic.this]
+ count = var.notification_email != null ? 1 : 0
+ topic_arn = aws_sns_topic.this.arn
+ protocol = "email"
+ endpoint = var.notification_email
+}
+
resource "aws_lambda_permission" "sns_lambda_permission" {
count = local.sns_enable ? 1 : 0
statement_id = "AllowExecutionFromSNS"
@@ -240,13 +324,6 @@ resource "aws_sns_topic_policy" "test" {
policy = data.aws_iam_policy_document.topic[count.index].json
}
-resource "aws_sns_topic_subscription" "lambda_sns_subscription" {
- count = local.sns_enable ? 1 : 0
- depends_on = [module.lambda]
- topic_arn = data.aws_sns_topic.sns_topic[count.index].arn
- protocol = "lambda"
- endpoint = module.lambda.lambda_function_arn
-}
resource "aws_secretsmanager_secret" "coralogix_secret" {
count = var.store_api_key_in_secrets_manager && !local.api_key_is_arn? 1 : 0
diff --git a/modules/coralogix-aws-shipper/variables.tf b/modules/coralogix-aws-shipper/variables.tf
index 29e4093a..808aebdc 100644
--- a/modules/coralogix-aws-shipper/variables.tf
+++ b/modules/coralogix-aws-shipper/variables.tf
@@ -111,8 +111,8 @@ variable "integration_type" {
description = "the aws service that send the data to the s3"
type = string
validation {
- condition = contains(["CloudWatch", "CloudTrail", "VpcFlow", "S3", "S3Csv", "Sns"], var.integration_type)
- error_message = "The integration type must be: [CloudWatch, CloudTrail, VpcFlow, S3, S3Csv, Sns]."
+ condition = contains(["CloudWatch", "CloudTrail", "VpcFlow", "S3", "S3Csv", "Sns", "Sqs"], var.integration_type)
+ error_message = "The integration type must be: [CloudWatch, CloudTrail, VpcFlow, S3, S3Csv, Sns, Sqs]."
}
}
@@ -161,3 +161,9 @@ variable "lambda_name" {
description = "The name of the lambda function"
default = null
}
+
+variable "sqs_name" {
+ description = "The name of the SQS that you want watch"
+ type = string
+ default = null
+}