feat(notifiers): change configuration format to allow multiple notifi…

…ers for each type (breaking changes)

README.md

@@ -16,7 +16,7 @@ Deploying this project will:
 
 - create AWS credentials for their use as GitGuardian Canary Tokens. The users associated with these credentials do not have any permissions, so they cannot perform any action.
 - create the related AWS infrastructure required to store any activities related to these credentials with [AWS CloudTrail](https://aws.amazon.com/cloudtrail/) and [AWS S3](https://aws.amazon.com/s3/).
-- create the related AWS infrastructure required to send alerts when one of the tokens is tampered to different integration such as email, native webhook and Slack. 
+- create the related AWS infrastructure required to send alerts when one of the tokens is tampered to different integration such as email, native webhook and Slack.
 
 # Project setup
 

docs/how_to_add_a_notifier.md

@@ -27,15 +27,9 @@ Finally, you need to modify the `lambda_py/notifiers/__init__.py` file:
 
 You will need to modify the `variables.tf` file:
 
-1. Add a variable specific to your notifier, for example `Custom_notifier`. This variable must have two fields:
+1. Add a variable specific to your notifier, for example `Custom_notifier`. The variable type must be a list of object, where each object holds the configuration for a notifier instance (allowing to configure multiple notifiers of the same kind).
 
-- `enabled` (`bool`), which will specify whether this notifier is enabled
-- `parameters` (`object`), which will hold configuration bits specific to this notifier. The subfields of the `parameters` field will be passed as-is to the lambda function, as environment variables. This means `parameters` should be a mapping from string to string.
-
-2. Extend the `locals.parameters` block with the configuration for you notifier.
-
-- The `name` field should be the same as the one used in your notifier class (comparison will be made case-independant).
-- The `enabled` and `parameters` take the values from the variable declared above.
+2. Extend the `locals.ggcanary_lambda_parameters` block with the configuration for your notifier.
 
 ### Update dependencies
 
@@ -58,10 +52,11 @@ from ..types import ReportEntry
 from .abstract_notifier import INotifier
 
 class MyNotifier(INotifier):
-    NAME = "MY_NOTIFIER"
+    kind = "my_notifier"
 
-    def __init__(self):
-        self.param = os.environ["MY_PARAM"]
+    def __init__(self, param1, param2, **kwargs):
+        self.param1 = param1
+        self.param2 = param2
 
     def send_notification(self, report_entries: List[ReportEntry]):
         # format entries and send message
@@ -72,49 +67,44 @@ class MyNotifier(INotifier):
 Update nofiers `__init__.py` file:
 
 ```python
-from typing import List, Type
-
-from .abstract_notifier import INotifier
 from .ses_notifier import SESNotifier
 ...
 from .my_notifier import MyNotifier
 
-NOTIFIER_CLASSES: List[Type[INotifier]] = [SESNotifier, ..., MyNotifier]
+NOTIFIER_CLASSES = (SESNotifier, ..., MyNotifier)
 
-__all__ = (NOTIFIER_CLASSES)
+__all__ = (NOTIFIER_CLASSES,)
 ```
 
 Update `variables.tf`:
 
 ```terraform
-variable my_notifier {
-  type = object({
-    enabled = bool
-    parameters = object({
-      MY_PARAM = string
-    })
-  })
-  default = {
-    enabled    = false
-    parameters = null
-  }
+variable my_notifiers {
+  type = list(object({
+    my_param = string
+  }))
+  default = []
 }
 
 locals {
-  notifiers = [
-    {
-      name       = "SES"
-      enabled    = var.SES_notifier.enabled
-      parameters = var.SES_notifier.parameters
-    },
-    {
+  ggcanary_lambda_parameters = concat(
+    [
+      for notifier_config in var.SES_notifiers :
+      {
+        kind       = "ses"
+        parameters = notifier_config
+      }
+    ],
+    [
       ...
-    },
-    {
-      name       = "MY_NOTIFIER"
-      enabled    = var.my_notifier.enabled
-      parameters = var.my_notifier.parameters
-    }
+    ],
+    [
+      for notifier_config in var.my_notifiers :
+      {
+        kind       = "my_notifier"  # must correspond to the `kind` attribute of `MyNotifier`
+        parameters = notifier_config
+      }
+    ]
   ]
 }
 ```

docs/variables_reference.md

@@ -12,9 +12,9 @@ This file holds the configuration for the infrastructure and the notifiers.
 | `terraform_backend_s3_bucket` | **Optional**  | Name of the S3 bucket holding the state. Mandatory to configure with `tf_backend`       |
 | `aws_region`                  | **mandatory** | AWS region where the project will be deployed.                                          |
 | `global_prefix`               | **mandatory** | Prefix that will be used to generate unique name for resources, especially for buckets. |
-| `SES_notifier`                | Optional      | Configuration of the SES Notifier.                                                      |
-| `Slack_notifier`              | Optional      | Configuration of the Slack Notifier.                                                    |
-| `SendGrid_notifier`           | Optional      | Configuration of the SendGrid Notifier.                                                 |
+| `SES_notifiers`               | Optional      | Configuration of the SES Notifiers.                                                     |
+| `Slack_notifiers`             | Optional      | Configuration of the Slack Notifiers.                                                   |
+| `SendGrid_notifiers`          | Optional      | Configuration of the SendGrid Notifiers.                                                |
 
 See examples in [`examples/tf_vars`](/examples/tf_vars)
 

examples/tf_vars/multiple_notifiers.tfvars.example

@@ -10,18 +10,18 @@ aws_region  = "us-west-2"
 global_prefix = "CHANGEME"  # prefix used for all ggcanary resources
 
 
-SendGrid_notifier = {
-  enabled = true
-  parameters = {
-    SOURCE_EMAIL_ADDRESS = "canary@my_domain.org"  # email address to send emails from
-    DEST_EMAIL_ADDRESS   = "security_email@my_domain.org"  # email address receiving alerts
-    API_KEY              = "SG.XXXX"  # Account API key
-  }
-}
+SendGrid_notifiers = [{
+  source_email_address = "canary@my_domain.org"  # email address to send emails from
+  dest_email_address   = "security_email@my_domain.org"  # email address receiving alerts
+  api_key              = "SG.XXXX"  # Account API key
+}]
 
-Slack_notifier = {
-  enabled = true
-  parameters = {
-    WEBHOOK = "REDACTED"  # eg: https://hooks.slack.com/services/CHANGE/ME/FOR_REAL
+Slack_notifiers = [
+  {
+    webhook = "REDACTED"  # eg: https://hooks.slack.com/services/CHANGE/ME/FOR_REAL
+  },
+  {
+    webhook = "ANOTHER_REDACTED"
   }
-}
+]
+

examples/tf_vars/sendgrid.tfvars.example

@@ -11,11 +11,8 @@ aws_region  = "us-west-2"
 global_prefix = "CHANGEME"  # prefix used for all ggcanary resources
 
 
-SendGrid_notifier = {
-  enabled = true
-  parameters = {
-    SOURCE_EMAIL_ADDRESS = "canary@my_domain.org"  # email address to send emails from
-    DEST_EMAIL_ADDRESS   = "security_email@my_domain.org"  # email address receiving alerts
-    API_KEY              = "SG.XXXX"  # Account API key
-  }
-}
+SendGrid_notifiers = [{
+  source_email_address = "canary@my_domain.org"  # email address to send emails from
+  dest_email_address   = "security_email@my_domain.org"  # email address receiving alerts
+  api_key              = "sG.XXXX"  # Account API key
+}]

examples/tf_vars/ses.tfvars.example

@@ -12,12 +12,8 @@ aws_region  = "us-west-2"
 global_prefix = "CHANGEME"  # prefix used for all ggcanary resources
 
 
-SES_notifier = {
-  enabled = true
-  parameters = {
-    SOURCE_EMAIL_ADDRESS = "canary@my_domain.org"  # email address to send emails from
-    DEST_EMAIL_ADDRESS   = "security_email@my_domain.org"  # email address receiving alerts
-    zone_id              = "000000000000"  # Value to retrieve from Route53, that corresponds to the SOURCE_EMAIL_ADDRESS domain.
-
-  }
-}
+SES_notifiers = [{
+  zone_id              = "000000000000"  # Value to retrieve from Route53, that corresponds to the SOURCE_EMAIL_ADDRESS domain.
+  source_email_address = "canary@my_domain.org"  # email address to send emails from
+  dest_email_address   = "security_email@my_domain.org"  # email address receiving alerts
+}]

examples/tf_vars/slack.tfvars.example

@@ -11,9 +11,6 @@ aws_region  = "us-west-2"
 global_prefix = "CHANGEME"  # prefix used for all ggcanary resources
 
 
-Slack_notifier = {
-  enabled = true
-  parameters = {
-    WEBHOOK = "REDACTED"  # eg: https://hooks.slack.com/services/CHANGE/ME/FOR_REAL
-  }
-}
+Slack_notifiers = [{
+  webhook = "REDACTED"  # eg: https://hooks.slack.com/services/CHANGE/ME/FOR_REAL
+}]

lambda.tf

@@ -1,28 +1,24 @@
 locals {
-  lambda_function_name = "${var.global_prefix}-lambda"
+  lambda_function_name   = "${var.global_prefix}-lambda"
+  lambda_parameters_file = "builds/ggcanary_lambda_parameters.json"
 }
 
-locals {
-  enabled_notifiers = join(",", [
-    for value in local.notifiers : upper(value.name)
-    if value.enabled
-  ])
-  parameters_map_list = tolist([
-    for value in local.notifiers : {
-      for param_name, param_value in value.parameters :
-      "${upper(value.name)}_${param_name}" => param_value
-    }
-    if value.enabled
-  ])
 
-  parameters_map = merge(local.parameters_map_list...)
+resource "local_sensitive_file" "notifier_parameters" {
+  content         = jsonencode(local.ggcanary_lambda_parameters)
+  filename        = local.lambda_parameters_file
+  file_permission = "0600"
 }
 
 
+
 module "lambda_function" {
 
   source  = "terraform-aws-modules/lambda/aws"
   version = "3.2.0"
+  depends_on = [
+    local_sensitive_file.notifier_parameters
+  ]
 
   source_path = [
     {
@@ -32,20 +28,21 @@ module "lambda_function" {
     {
       path          = "lambda/lambda_py",
       prefix_in_zip = "lambda_py",
+    },
+    {
+      path     = "."
+      patterns = ["!.*", local.lambda_parameters_file]
     }
   ]
   function_name = local.lambda_function_name
   handler       = "entrypoint.lambda_handler"
   runtime       = "python3.8"
   publish       = true
 
-  environment_variables = merge(
-    {
-      GGCANARY_USER_PREFIX = var.global_prefix,
-      ENABLED_NOTIFIERS    = local.enabled_notifiers
-    },
-    local.parameters_map
-  )
+  environment_variables = {
+    GGCANARY_USER_PREFIX = var.global_prefix,
+  }
+
 
   allowed_triggers = {
     AllowExecutionFromS3Bucket = {
@@ -107,3 +104,13 @@ resource "aws_s3_bucket_notification" "bucket_notification" {
   depends_on = [module.lambda_function]
 
 }
+
+resource "null_resource" "remove_temp_file" {
+  provisioner "local-exec" {
+    command = "rm -f ${local.lambda_parameters_file}"
+  }
+  depends_on = [module.lambda_function]
+  triggers = {
+    always_run = timestamp()
+  }
+}

lambda/lambda_py/lambda_function.py

@@ -6,11 +6,12 @@
 
 import boto3
 
-from .notifiers import NOTIFIER_CLASSES, INotifier
+from .notifiers import NOTIFIER_CLASSES
 from .types import LogRecord, ReportEntry
 
 
 USERNAME_TO_TRACK = os.environ["GGCANARY_USER_PREFIX"]
+LAMBDA_PARAMETERS_PATH = "./builds/ggcanary_lambda_parameters.json"
 
 
 def fetch_log_records(bucket: str, key: str) -> List[Dict]:
@@ -44,17 +45,26 @@ def get_report_entries(records: List[Dict]) -> List[ReportEntry]:
     ]
 
 
-def load_notifiers() -> List[INotifier]:
-    active_classes = [
-        notifier_class
-        for notifier_class in NOTIFIER_CLASSES
-        if notifier_class.is_enabled()
+def load_notifiers():
+    with open(LAMBDA_PARAMETERS_PATH) as json_file:
+        notifier_configs = json.load(json_file)
+
+    notifier_classes_by_kind = {
+        notif_class.kind: notif_class for notif_class in NOTIFIER_CLASSES
+    }
+    # fmt: off
+    notifiers = [
+        notifier_classes_by_kind[notifier["kind"]](
+            **notifier["parameters"]
+        )
+        for notifier in notifier_configs
     ]
+    # fmt: on
     print(
         "Enabled notifiers:",
-        [notifier_class.NAME for notifier_class in active_classes],
+        [notifier.kind for notifier in notifiers],
     )
-    return [active_class() for active_class in active_classes]
+    return notifiers
 
 
 def handler(event: Dict):

lambda/lambda_py/notifiers/__init__.py

@@ -1,11 +1,8 @@
-from typing import List, Type
-
-from .abstract_notifier import INotifier
 from .sendgrid_notifier import SendGridNotifier
 from .ses_notifier import SESNotifier
 from .slack_notifier import SlackNotifier
 
 
-NOTIFIER_CLASSES: List[Type[INotifier]] = [SESNotifier, SlackNotifier, SendGridNotifier]
+NOTIFIER_CLASSES = (SESNotifier, SlackNotifier, SendGridNotifier)
 
-__all__ = (NOTIFIER_CLASSES,)
+__all__ = NOTIFIER_CLASSES

lambda/lambda_py/notifiers/abstract_notifier.py

@@ -1,32 +1,12 @@
 import abc
-import os
 from typing import List
 
 from ..types import ReportEntry
 
 
 class INotifier(abc.ABC):
-    NAME = "ABSTRACT"
+    kind: str
 
     @abc.abstractmethod
     def send_notification(self, events: List[ReportEntry]):
         pass
-
-    @classmethod
-    def param(cls, param_name):
-        return cls.params()[param_name]
-
-    @classmethod
-    def params(cls):
-        prefix = cls.NAME + "_"
-        uppercased_env = {key.upper(): value for key, value in os.environ.items()}
-        return {
-            key[len(prefix) :]: value
-            for key, value in uppercased_env.items()
-            if key.startswith(prefix)
-        }
-
-    @classmethod
-    def is_enabled(cls) -> bool:
-        enabled_notifiers = os.environ.get("ENABLED_NOTIFIERS", "").upper().split(",")
-        return cls.NAME in enabled_notifiers