NHSDigital · JackSpagnoliNHS · Sep 30, 2024 · Sep 25, 2024 · Sep 25, 2024 · Sep 25, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -117,6 +117,7 @@ jobs:
       CREATE_INT_RELEASE_NOTES: true
       CREATE_PROD_RELEASE_NOTES: true
       TOGGLE_GET_STATUS_UPDATES: true
+      ENABLE_ALERTS: true
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -173,6 +174,7 @@ jobs:
       LOG_LEVEL: DEBUG
       LOG_RETENTION_DAYS: 30
       TOGGLE_GET_STATUS_UPDATES: true
+      ENABLE_ALERTS: true
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.QA_CLOUD_FORMATION_DEPLOY_ROLE }}

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -73,6 +73,7 @@ jobs:
       LOG_LEVEL: DEBUG
       LOG_RETENTION_DAYS: 30
       TOGGLE_GET_STATUS_UPDATES: true
+      ENABLE_ALERTS: false
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -131,6 +131,7 @@ jobs:
       CREATE_INT_RELEASE_NOTES: true
       CREATE_PROD_RELEASE_NOTES: true
       TOGGLE_GET_STATUS_UPDATES: true
+      ENABLE_ALERTS: true
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -187,6 +188,7 @@ jobs:
       LOG_LEVEL: DEBUG
       LOG_RETENTION_DAYS: 30
       TOGGLE_GET_STATUS_UPDATES: true
+      ENABLE_ALERTS: true
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.REF_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -216,6 +218,7 @@ jobs:
       LOG_LEVEL: DEBUG
       LOG_RETENTION_DAYS: 30
       TOGGLE_GET_STATUS_UPDATES: true
+      ENABLE_ALERTS: true
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.QA_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -240,6 +243,7 @@ jobs:
       CREATE_INT_RELEASE_NOTES: true
       CREATE_INT_RC_RELEASE_NOTES: true
       TOGGLE_GET_STATUS_UPDATES: true
+      ENABLE_ALERTS: true
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.INT_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -294,6 +298,7 @@ jobs:
       CREATE_PROD_RELEASE_NOTES: true
       TOGGLE_GET_STATUS_UPDATES: true
       RUN_REGRESSION_TESTS: false
+      ENABLE_ALERTS: true
     secrets:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.PROD_CLOUD_FORMATION_DEPLOY_ROLE }}

diff --git a/.github/workflows/sam_release_code.yml b/.github/workflows/sam_release_code.yml
@@ -63,6 +63,9 @@ on:
       RUN_REGRESSION_TESTS:
         type: boolean
         default: true
+      ENABLE_ALERTS:
+        type: boolean
+        default: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE:
         required: true
@@ -142,6 +145,7 @@ jobs:
           DOMAIN_NAME_EXPORT: ${{ inputs.DOMAIN_NAME_EXPORT }}
           ZONE_ID_EXPORT: ${{ inputs.ZONE_ID_EXPORT }}
           TOGGLE_GET_STATUS_UPDATES: ${{ inputs.TOGGLE_GET_STATUS_UPDATES }}
+          ENABLE_ALERTS: ${{ inputs.ENABLE_ALERTS }}
         run: ./release_code.sh
 
       - name: create_int_release_notes

diff --git a/Makefile b/Makefile
@@ -34,7 +34,8 @@ sam-sync: guard-AWS_DEFAULT_PROFILE guard-stack_name compile download-get-secret
 		--parameter-overrides \
 			  EnableSplunk=false\
 			  TargetSpineServer=$$TARGET_SPINE_SERVER \
-			  TargetServiceSearchServer=$$TARGET_SERVICE_SEARCH_SERVER
+			  TargetServiceSearchServer=$$TARGET_SERVICE_SEARCH_SERVER \
+				EnableAlerts=false
 
 sam-sync-sandbox: guard-stack_name compile download-get-secrets-layer
 	sam sync \
@@ -50,7 +51,8 @@ sam-deploy: guard-AWS_DEFAULT_PROFILE guard-stack_name
 		--parameter-overrides \
 			  EnableSplunk=false \
 			  TargetSpineServer=$$TARGET_SPINE_SERVER \
-			  TargetServiceSearchServer=$$TARGET_SERVICE_SEARCH_SERVER
+			  TargetServiceSearchServer=$$TARGET_SERVICE_SEARCH_SERVER \
+				EnableAlerts=false
 
 sam-delete: guard-AWS_DEFAULT_PROFILE guard-stack_name
 	sam delete --stack-name $$stack_name
@@ -105,7 +107,8 @@ sam-deploy-package: guard-artifact_bucket guard-artifact_bucket_prefix guard-sta
 			  Env=$$TARGET_ENVIRONMENT \
 				DomainNameExport=$$DOMAIN_NAME_EXPORT \
 				ZoneIDExport=$$ZONE_ID_EXPORT \
-			  ToggleGetStatusUpdates=$$TOGGLE_GET_STATUS_UPDATES
+			  ToggleGetStatusUpdates=$$TOGGLE_GET_STATUS_UPDATES \
+			  EnableAlerts=$$ENABLE_ALERTS
 
 compile-node:
 	npx tsc --build tsconfig.build.json

diff --git a/SAMtemplates/alarms/main.yaml b/SAMtemplates/alarms/main.yaml
@@ -0,0 +1,84 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Transform: AWS::Serverless-2016-10-31
+Description: |
+  PfP Cloudwatch alarms and related resources
+
+Parameters:
+  StackName:
+    Type: String
+    Default: none
+
+  GetMyPrescriptionsFunctionName:
+    Type: String
+    Default: none
+
+  EnableAlerts:
+    Type: String
+
+Resources:
+  ServiceSearchErrorsLogsMetricFilter:
+    Type: AWS::Logs::MetricFilter
+    Properties:
+      FilterName: GetMyPrescriptionsErrors
+      # Match logs with $.message starting with "call to service search unsuccessful"
+      FilterPattern: !Sub '{ ($.level = "WARN") && ($.function_name = "${GetMyPrescriptionsFunctionName}") && $.message = %call to service search unsuccessful% }' # function_name included to allow it to be set as a dimension on the metric
+      LogGroupName:
+        Fn::ImportValue: !Sub ${StackName}:functions:${GetMyPrescriptionsFunctionName}:LambdaLogGroupName
+      MetricTransformations:
+        - MetricNamespace: LambdaLogFilterMetrics
+          MetricName: ErrorCount
+          MetricValue: 1
+          Unit: Count
+          Dimensions: # dimensions for a logs filter metric can only be a field/value from the filter pattern
+            - Key: FunctionName
+              Value: $.function_name
+
+  ServiceSearchErrorsAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmDescription: Count of Service Search errors
+      AlarmName: !Sub ${StackName}_ServiceSearch_Errors
+      Namespace: LambdaLogFilterMetrics
+      MetricName: ErrorCount
+      Dimensions:
+        - Name: FunctionName
+          Value: !Ref GetMyPrescriptionsFunctionName
+      Period: 60 #seconds
+      EvaluationPeriods: 1
+      Statistic: Sum
+      ComparisonOperator: GreaterThanOrEqualToThreshold
+      Threshold: 1
+      Unit: Count
+      TreatMissingData: notBreaching
+      ActionsEnabled: !Ref EnableAlerts
+      AlarmActions:
+        - !ImportValue lambda-resources:SlackAlertsSnsTopicArn
+      InsufficientDataActions:
+        - !ImportValue lambda-resources:SlackAlertsSnsTopicArn
+      OKActions:
+        - !ImportValue lambda-resources:SlackAlertsSnsTopicArn
+
+  ServiceSearchUnhandledErrorsAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmDescription: Count of Service Search unhandled errors
+      AlarmName: !Sub ${StackName}_ServiceSearch_UnhandledErrors
+      Namespace: Lambda
+      MetricName: Errors
+      Dimensions:
+        - Name: FunctionName
+          Value: !Ref GetMyPrescriptionsFunctionName
+      Period: 60 #seconds
+      EvaluationPeriods: 1
+      Statistic: Sum
+      ComparisonOperator: GreaterThanOrEqualToThreshold
+      Threshold: 1
+      Unit: Count
+      TreatMissingData: notBreaching
+      ActionsEnabled: !Ref EnableAlerts
+      AlarmActions:
+        - !ImportValue lambda-resources:SlackAlertsSnsTopicArn
+      InsufficientDataActions:
+        - !ImportValue lambda-resources:SlackAlertsSnsTopicArn
+      OKActions:
+        - !ImportValue lambda-resources:SlackAlertsSnsTopicArn
diff --git a/SAMtemplates/functions/lambda_resources.yaml b/SAMtemplates/functions/lambda_resources.yaml
@@ -134,3 +134,9 @@ Outputs:
     Value: !GetAtt ExecuteLambdaManagedPolicy.PolicyArn
     Export:
       Name: !Sub ${StackName}:functions:${LambdaName}:ExecuteLambdaPolicyArn
+
+  LogGroupName:
+    Description: Lambda log group name
+    Value: !Ref LambdaLogGroup
+    Export:
+      Name: !Sub ${StackName}:functions:${LambdaName}:LambdaLogGroupName
diff --git a/SAMtemplates/main_template.yaml b/SAMtemplates/main_template.yaml
@@ -75,6 +75,13 @@ Parameters:
     Type: String
     Default: false
 
+  EnableAlerts:
+    Type: String
+    Default: true
+    AllowedValues:
+      - true
+      - false
+
 Resources:
   Apis:
     Type: AWS::Serverless::Application
@@ -121,3 +128,12 @@ Resources:
         EnrichPrescriptionsFunctionArn: !GetAtt Functions.Outputs.EnrichPrescriptionsFunctionArn
         LogRetentionInDays: !Ref LogRetentionInDays
         EnableSplunk: !Ref EnableSplunk
+
+  Alarms:
+    Type: AWS::Serverless::Application
+    Properties:
+      Location: alarms/main.yaml
+      Parameters:
+        StackName: !Ref AWS::StackName
+        GetMyPrescriptionsFunctionName: !GetAtt Functions.Outputs.GetMyPrescriptionsFunctionName
+        EnableAlerts: !Ref EnableAlerts