The Idle EC2 Reaper: Automating Cloud Resource Shutdowns in Non-Production Environments

import boto3
import logging
 
logger = logging.getLogger()
logger.setLevel(logging.INFO)
 
ec2 = boto3.client("ec2")
rds = boto3.client("rds")
ecs = boto3.client("ecs")
 
 
def get_tagged_instances(action: str) -> list[str]:
    """Find EC2 instances tagged for the Reaper."""
    filters = [
        {"Name": "tag:Environment", "Values": ["dev", "staging", "qa"]},
        {"Name": "tag:Reaper", "Values": ["enabled"]},
    ]
 
    if action == "stop":
        filters.append(
            {"Name": "instance-state-name", "Values": ["running"]}
        )
    elif action == "start":
        filters.append(
            {"Name": "instance-state-name", "Values": ["stopped"]}
        )
 
    response = ec2.describe_instances(Filters=filters)
    instance_ids = []
    for reservation in response["Reservations"]:
        for instance in reservation["Instances"]:
            instance_ids.append(instance["InstanceId"])
 
    return instance_ids
 
 
def stop_rds_instances():
    """Stop all non-prod RDS instances."""
    response = rds.describe_db_instances()
    for db in response["DBInstances"]:
        tags = rds.list_tags_for_resource(
            ResourceName=db["DBInstanceArn"]
        )["TagList"]
        tag_map = {t["Key"]: t["Value"] for t in tags}
 
        if (
            tag_map.get("Environment") in ("dev", "staging", "qa")
            and tag_map.get("Reaper") == "enabled"
            and db["DBInstanceStatus"] == "available"
        ):
            rds.stop_db_instance(
                DBInstanceIdentifier=db["DBInstanceIdentifier"]
            )
            logger.info(f"Stopped RDS: {db['DBInstanceIdentifier']}")
 
 
def start_rds_instances():
    """Start all non-prod RDS instances."""
    response = rds.describe_db_instances()
    for db in response["DBInstances"]:
        tags = rds.list_tags_for_resource(
            ResourceName=db["DBInstanceArn"]
        )["TagList"]
        tag_map = {t["Key"]: t["Value"] for t in tags}
 
        if (
            tag_map.get("Environment") in ("dev", "staging", "qa")
            and tag_map.get("Reaper") == "enabled"
            and db["DBInstanceStatus"] == "stopped"
        ):
            rds.start_db_instance(
                DBInstanceIdentifier=db["DBInstanceIdentifier"]
            )
            logger.info(f"Started RDS: {db['DBInstanceIdentifier']}")
 
 
def scale_ecs_services(desired_count: int):
    """Scale non-prod ECS services to the desired count."""
    clusters = ecs.list_clusters()["clusterArns"]
 
    for cluster_arn in clusters:
        services = ecs.list_services(
            cluster=cluster_arn
        )["serviceArns"]
 
        for service_arn in services:
            service_detail = ecs.describe_services(
                cluster=cluster_arn,
                services=[service_arn],
            )["services"][0]
 
            tags = {
                t["key"]: t["value"]
                for t in service_detail.get("tags", [])
            }
 
            if (
                tags.get("Environment") in ("dev", "staging", "qa")
                and tags.get("Reaper") == "enabled"
            ):
                ecs.update_service(
                    cluster=cluster_arn,
                    service=service_arn,
                    desiredCount=desired_count,
                )
                logger.info(
                    f"Scaled ECS {service_detail['serviceName']} "
                    f"to {desired_count}"
                )
 
 
def handler(event, context):
    action = event.get("action", "stop")
 
    if action == "stop":
        # Stop EC2
        instance_ids = get_tagged_instances("stop")
        if instance_ids:
            ec2.stop_instances(InstanceIds=instance_ids)
            logger.info(f"Stopped EC2 instances: {instance_ids}")
 
        # Stop RDS
        stop_rds_instances()
 
        # Scale ECS to 0
        scale_ecs_services(desired_count=0)
 
        return {
            "action": "stop",
            "ec2_stopped": len(instance_ids),
            "message": "Non-prod environments shut down for the night.",
        }
 
    elif action == "start":
        # Start EC2
        instance_ids = get_tagged_instances("start")
        if instance_ids:
            ec2.start_instances(InstanceIds=instance_ids)
            logger.info(f"Started EC2 instances: {instance_ids}")
 
        # Start RDS
        start_rds_instances()
 
        # Scale ECS back up
        scale_ecs_services(desired_count=1)
 
        return {
            "action": "start",
            "ec2_started": len(instance_ids),
            "message": "Non-prod environments are waking up.",
        }

resource "aws_lambda_function" "reaper" {
  function_name = "ec2-reaper"
  runtime       = "python3.12"
  handler       = "reaper.handler"
  timeout       = 300
  memory_size   = 256
 
  filename         = data.archive_file.reaper_zip.output_path
  source_code_hash = data.archive_file.reaper_zip.output_base64sha256
 
  role = aws_iam_role.reaper_role.arn
 
  environment {
    variables = {
      SLACK_WEBHOOK_URL = var.slack_webhook_url
    }
  }
}
 
# Stop everything at 7 PM UTC on weekdays
resource "aws_cloudwatch_event_rule" "stop_weekday" {
  name                = "reaper-stop-weekday"
  schedule_expression = "cron(0 19 ? * MON-FRI *)"
}
 
resource "aws_cloudwatch_event_target" "stop_weekday" {
  rule = aws_cloudwatch_event_rule.stop_weekday.name
  arn  = aws_lambda_function.reaper.arn
  input = jsonencode({ action = "stop" })
}
 
# Start everything at 8 AM UTC on weekdays
resource "aws_cloudwatch_event_rule" "start_weekday" {
  name                = "reaper-start-weekday"
  schedule_expression = "cron(0 8 ? * MON-FRI *)"
}
 
resource "aws_cloudwatch_event_target" "start_weekday" {
  rule = aws_cloudwatch_event_rule.start_weekday.name
  arn  = aws_lambda_function.reaper.arn
  input = jsonencode({ action = "start" })
}
 
# Stop everything Friday night for the entire weekend
resource "aws_cloudwatch_event_rule" "stop_weekend" {
  name                = "reaper-stop-weekend"
  schedule_expression = "cron(0 19 ? * FRI *)"
}
 
resource "aws_cloudwatch_event_target" "stop_weekend" {
  rule = aws_cloudwatch_event_rule.stop_weekend.name
  arn  = aws_lambda_function.reaper.arn
  input = jsonencode({ action = "stop" })
}
 
# IAM Role with least-privilege permissions
resource "aws_iam_role" "reaper_role" {
  name = "ec2-reaper-role"
 
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [{
      Action    = "sts:AssumeRole"
      Effect    = "Allow"
      Principal = { Service = "lambda.amazonaws.com" }
    }]
  })
}
 
resource "aws_iam_role_policy" "reaper_policy" {
  name = "ec2-reaper-policy"
  role = aws_iam_role.reaper_role.id
 
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "ec2:DescribeInstances",
          "ec2:StartInstances",
          "ec2:StopInstances",
        ]
        Resource = "*"
        Condition = {
          StringEquals = {
            "ec2:ResourceTag/Reaper" = "enabled"
          }
        }
      },
      {
        Effect = "Allow"
        Action = [
          "rds:DescribeDBInstances",
          "rds:ListTagsForResource",
          "rds:StartDBInstance",
          "rds:StopDBInstance",
        ]
        Resource = "*"
      },
      {
        Effect = "Allow"
        Action = [
          "ecs:ListClusters",
          "ecs:ListServices",
          "ecs:DescribeServices",
          "ecs:UpdateService",
        ]
        Resource = "*"
      },
      {
        Effect   = "Allow"
        Action   = ["logs:*"]
        Resource = "arn:aws:logs:*:*:*"
      },
    ]
  })
}