AWS Canary and alarm in Terraform

Question

I am trying to set up an AWS canary that will monitor an API gateway. When the canary fails, I would like to send out an email to email addresses subscribed to the SNS topic.

The canary is set up as follows (works well):

resource "aws_synthetics_canary" "my-api-canary" {
  name                 = "my-canary"
  artifact_s3_location = "s3://${aws_s3_bucket.canary-output-bucket.bucket}/"
  execution_role_arn   = aws_iam_role.some-role.arn
  handler              = "canary.handler"
  zip_file             = data.archive_file.canary.output_path
  runtime_version      = "syn-python-selenium-1.3"
  schedule {
    expression         = "rate(5 minutes)"
  }
  start_canary         = true
}

Next (and here is the part where my knowledge/understanding fails), a tried setting up a metric alarm and an accompanying topic:

resource "aws_cloudwatch_metric_alarm" "my-alarm" {
  alarm_name                = "my-canary-alarm"
  comparison_operator       = "GreaterThanOrEqualToThreshold"
  evaluation_periods        = "2"
  metric_name               = "Errors"
  namespace                 = "AWS/Lambda"
  period                    = "30"
  statistic                 = "Sum"
  threshold                 = "1"
  alarm_description         = "This alarm fires if the canary fails"
  insufficient_data_actions = []
  alarm_actions = [aws_sns_topic.my-topic.arn]

  dimensions = {
    CanaryName = "my-canary"
  }
}

resource "aws_sns_topic" "my-topic" {
  name = "some-topic"
}

resource "aws_sns_topic_subscription" "my-topic-sub" {
  topic_arn = aws_sns_topic.my-topic.arn
  protocol  = "email"
  endpoint  = "[email protected]"

  depends_on = [
    aws_sns_topic.my-topic
  ]
}

Everything executes well and gets created. I have also ``confirmed'' my subscription to the topic, but no emails get delivered when the canary fails. This must be such an easy thing to do, but I can't get it working... any help, please?

Answer 1

This creates a canary, an alarm, SNS topic and sends email when an error occurs:

resource "aws_s3_bucket" "canary-output-bucket" {
  bucket = "some-bucket-to-dump-canary-logs"
  acl    = "private"
  force_destroy = true
  lifecycle {
    prevent_destroy = false
  }
}

resource "aws_iam_role" "my-cloudwatch-syn-role" {
  name = "my-cloudwatch-syn-role"
  description = "Role used to provide permissions for the canary to run."
  assume_role_policy = <<-EOF
    {
      "Version": "2012-10-17",
      "Statement": [
          {
              "Effect": "Allow",
              "Principal": {
                  "Service": "lambda.amazonaws.com"
              },
              "Action": "sts:AssumeRole"
          }
      ]
    }
    EOF
}


resource "aws_synthetics_canary" "my-api-canary" {
  name                 = "my-canary"
  artifact_s3_location = "s3://${aws_s3_bucket.canary-output-bucket.bucket}/"
  execution_role_arn   = aws_iam_role.my-cloudwatch-syn-role.arn
  handler              = "my_api.handler"
  zip_file             = data.archive_file.my-canary.output_path
  runtime_version      = "syn-python-selenium-1.3"
  schedule {
    expression         = "rate(5 minutes)"
  }
  start_canary         = true
}

resource "aws_cloudwatch_metric_alarm" "my-api-canary-alarm" {
  alarm_name                = "my-canary-alarm"
  comparison_operator       = "LessThanThreshold"
  evaluation_periods        = "2"
  metric_name               = "SuccessPercent"
  namespace                 = "CloudWatchSynthetics"
  threshold                 = "100"
  statistic                 = "Average"
  period                    = "300"
  alarm_description         = "This alarm fires if the canary fails"
  insufficient_data_actions = []
  alarm_actions = [aws_sns_topic.my-topic.arn]

  dimensions = {
    CanaryName = "my-canary"
  }
}

resource "aws_sns_topic" "my-topic" {
  name = "my-topic"
}

resource "aws_sns_topic_subscription" "my-topic-sub" {
  topic_arn = aws_sns_topic.my-topic.arn
  protocol  = "email"
  endpoint  = "[email protected]"

  depends_on = [
    aws_sns_topic.my-topic
  ]
}