How to reuse Elastic IPs for a set of private and public subnets dedicated to Fargate tasks

Question

I got the following setup to create the networking requirements for a Fargate setup:

resource "aws_vpc" "main" {
  cidr_block = var.cidr
  tags = {
    Environment = var.environment
    DO_NOT_DELETE = true
    CreatedBy = "terraform"
  }
}
 
resource "aws_internet_gateway" "main" {
  vpc_id = aws_vpc.main.id
  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

data "aws_availability_zones" "region_azs" {
  state = "available"
}

locals {
  az_count = length(data.aws_availability_zones.region_azs.names)
}

resource "aws_subnet" "private" {
  vpc_id            = aws_vpc.main.id
  cidr_block        = cidrsubnet(aws_vpc.main.cidr_block, 4, count.index)
  availability_zone = data.aws_availability_zones.region_azs.names[count.index]
  count             = local.az_count

  tags = {
    Name = "public-subnet-${data.aws_availability_zones.region_azs.names[count.index]}"
    AvailabilityZone = data.aws_availability_zones.region_azs.names[count.index]
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
    Type = "private"
    DO_NOT_DELETE = true
  }
}
 
resource "aws_subnet" "public" {
  vpc_id                  = aws_vpc.main.id
  cidr_block              = cidrsubnet(aws_vpc.main.cidr_block, 4, count.index + local.az_count )
  availability_zone = data.aws_availability_zones.region_azs.names[count.index]
  count             = local.az_count
  map_public_ip_on_launch = true

  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
    DO_NOT_DELETE = true
    Type = "public"
  }
}

resource "aws_route_table" "public" {
  vpc_id = aws_vpc.main.id
  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
    Type = "public"
  }
}
 
resource "aws_route" "public" {
  route_table_id         = aws_route_table.public.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.main.id
}
 
resource "aws_route_table_association" "public" {
  count          = local.az_count
  subnet_id      = element(aws_subnet.public.*.id, count.index)
  route_table_id = aws_route_table.public.id
}

resource "aws_nat_gateway" "main" {
  count         = local.az_count
  allocation_id = element(aws_eip.nat.*.id, count.index)
  subnet_id     = element(aws_subnet.public.*.id, count.index)
  depends_on    = [aws_internet_gateway.main]

  tags = {
    Environment = var.environment
    CreatedBy = "terraform" 
    Vpc = aws_vpc.main.id
  }
}
 
resource "aws_eip" "nat" {
  count = local.az_count
  vpc = true

  tags = {
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

resource "aws_route_table" "private" {
  count  = local.az_count
  vpc_id = aws_vpc.main.id

  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Type = "private"
    Vpc = aws_vpc.main.id
  }
}
 
resource "aws_route" "private" {
  count                  = local.az_count
  route_table_id         = element(aws_route_table.private.*.id, count.index)
  destination_cidr_block = "0.0.0.0/0"
  nat_gateway_id         = element(aws_nat_gateway.main.*.id, count.index)
}
 
resource "aws_route_table_association" "private" {
  count          = local.az_count
  subnet_id      = element(aws_subnet.private.*.id, count.index)
  route_table_id = element(aws_route_table.private.*.id, count.index)
}

resource "aws_security_group" "alb" {
  name   = "${var.resources_name_prefix}-alb-sg"
  vpc_id = aws_vpc.main.id
 
  ingress {
   protocol         = "tcp"
   from_port        = 80
   to_port          = 80
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }
 
  ingress {
   protocol         = "tcp"
   from_port        = 443
   to_port          = 443
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }
 
  egress {
   protocol         = "-1"
   from_port        = 0
   to_port          = 0
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }

  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

resource "aws_security_group" "ecs_tasks" {
  name   = "${var.resources_name_prefix}-ecs-sg"
  vpc_id = aws_vpc.main.id
 
  ingress {
   protocol         = "tcp"
   from_port        = 3000
   to_port          = 3000
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }

  egress {
   protocol         = "-1"
   from_port        = 0
   to_port          = 0
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }

  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

This is been working great for a couple availability zones, but now that I'm dynamically creating subnets for running tasks in every AZ per region, I'm reaching the limit of Elastic IP's per region.

So I'm getting this erorr while trying to create the stack:

Error creating EIP: AddressLimitExceeded: The maximum number of addresses has been reached.
       status code: 400

I'm wodering if the following part:

resource "aws_nat_gateway" "main" {
  count         = local.az_count
  allocation_id = element(aws_eip.nat.*.id, count.index)
  subnet_id     = element(aws_subnet.public.*.id, count.index)
  depends_on    = [aws_internet_gateway.main]

  tags = {
    Environment = var.environment
    CreatedBy = "terraform" 
    Vpc = aws_vpc.main.id
  }
}
 
resource "aws_eip" "nat" {
  count = local.az_count
  vpc = true

  tags = {
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

Could be structured to use a single EIP and routing internally, if this makes sense.

Answer 1

I modified your code a bit, but it's a mess. For example all private subnets are called "public". It creates two NATs now. Obviously if you have subnets in, lets say, 6 AZs, there will be some cross-AZ traffic to get to those NATs.

Alternatively, simply don't create VPCs spanning so many AZs. Typically only two-three AZs are used for a VPC. Having more than that is not really needed.

Finally, you can request AWS support to give your more EIPs, if you want to preserve your original setup.

resource "aws_vpc" "main" {
  cidr_block = var.cidr
  tags = {
    Environment = var.environment
    DO_NOT_DELETE = true
    CreatedBy = "terraform"
  }
}
 
resource "aws_internet_gateway" "main" {
  vpc_id = aws_vpc.main.id
  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

data "aws_availability_zones" "region_azs" {
  state = "available"
}

locals {
  az_count = length(data.aws_availability_zones.region_azs.names)
}

resource "aws_subnet" "private" {
  vpc_id            = aws_vpc.main.id
  cidr_block        = cidrsubnet(aws_vpc.main.cidr_block, 4, count.index)
  availability_zone = data.aws_availability_zones.region_azs.names[count.index]
  count             = local.az_count

  tags = {
    Name = "private-subnet-${data.aws_availability_zones.region_azs.names[count.index]}"
    AvailabilityZone = data.aws_availability_zones.region_azs.names[count.index]
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
    Type = "private"
    DO_NOT_DELETE = true
  }
}
 
resource "aws_subnet" "public" {
  vpc_id                  = aws_vpc.main.id
  cidr_block              = cidrsubnet(aws_vpc.main.cidr_block, 4, count.index + local.az_count )
  availability_zone = data.aws_availability_zones.region_azs.names[count.index]
  count             = local.az_count
  map_public_ip_on_launch = true

  tags = {
    Name = "public-subnet-${data.aws_availability_zones.region_azs.names[count.index]}"  
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
    DO_NOT_DELETE = true
    Type = "public"
  }
}

resource "aws_route_table" "public" {
  vpc_id = aws_vpc.main.id
  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
    Type = "public"
  }
}
 
resource "aws_route" "public" {
  route_table_id         = aws_route_table.public.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.main.id
}
 
resource "aws_route_table_association" "public" {
  count          = local.az_count
  subnet_id      = element(aws_subnet.public.*.id, count.index)
  route_table_id = aws_route_table.public.id
}

resource "aws_nat_gateway" "main" {
  count         = 2
  allocation_id = element(aws_eip.nat.*.id, count.index)
  subnet_id     = element(aws_subnet.public.*.id, count.index)
  depends_on    = [aws_internet_gateway.main]

  tags = {
    Environment = var.environment
    CreatedBy = "terraform" 
    Vpc = aws_vpc.main.id
  }
}
 
resource "aws_eip" "nat" {
  count = 2
  vpc = true

  tags = {
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

resource "aws_route_table" "private" {
  count  = local.az_count
  vpc_id = aws_vpc.main.id

  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Type = "private"
    Vpc = aws_vpc.main.id
  }
}
 
resource "aws_route" "private" {
  count                  = local.az_count
  route_table_id         = element(aws_route_table.private.*.id, count.index)
  destination_cidr_block = "0.0.0.0/0"
  nat_gateway_id         = element(aws_nat_gateway.main.*.id, count.index)
}
 
resource "aws_route_table_association" "private" {
  count          = local.az_count
  subnet_id      = element(aws_subnet.private.*.id, count.index)
  route_table_id = element(aws_route_table.private.*.id, count.index)
}

resource "aws_security_group" "alb" {
  name   = "${var.resources_name_prefix}-alb-sg"
  vpc_id = aws_vpc.main.id
 
  ingress {
   protocol         = "tcp"
   from_port        = 80
   to_port          = 80
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }
 
  ingress {
   protocol         = "tcp"
   from_port        = 443
   to_port          = 443
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }
 
  egress {
   protocol         = "-1"
   from_port        = 0
   to_port          = 0
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }

  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}

resource "aws_security_group" "ecs_tasks" {
  name   = "${var.resources_name_prefix}-ecs-sg"
  vpc_id = aws_vpc.main.id
 
  ingress {
   protocol         = "tcp"
   from_port        = 3000
   to_port          = 3000
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }

  egress {
   protocol         = "-1"
   from_port        = 0
   to_port          = 0
   cidr_blocks      = ["0.0.0.0/0"]
   ipv6_cidr_blocks = ["::/0"]
  }

  tags = {
    Environment = var.environment
    CreatedBy = "terraform"
    Vpc = aws_vpc.main.id
  }
}