Terraform create and delete route table indefinitely in an aws peering vpc

Question

I'm having strange Terraform behavior when I run it multiple times. The scenario is this: two VPCs in the same region and AWS account and I want to create a peering between them. In the first round Terraform correctly creates the peering between the two VPCs and their respective routes but in the second round TF deletes the routes. This is my terraform code

provider "aws" {
region                = "..."
access_key            = "..."
secret_key            = "..."
}

resource "aws_vpc" "vpc1" {
  cidr_block = "10.0.0.0/16"
}

resource "aws_vpc" "vpc2" {
  cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "public1" {
  vpc_id     = aws_vpc.vpc1.id
  cidr_block = "10.0.1.0/24"
}

resource "aws_subnet" "private1" {
  vpc_id     = aws_vpc.vpc1.id
  cidr_block = "10.0.2.0/24"
}

resource "aws_subnet" "public2" {
  vpc_id     = aws_vpc.vpc2.id
  cidr_block = "10.1.1.0/24"
}

resource "aws_subnet" "private2" {
  vpc_id     = aws_vpc.vpc2.id
  cidr_block = "10.1.2.0/24"
}

resource "aws_internet_gateway" "igw1" {
  vpc_id = aws_vpc.vpc1.id
}

resource "aws_internet_gateway" "igw2" {
  vpc_id = aws_vpc.vpc2.id
}

resource "aws_route_table" "public_route_table1" {
  vpc_id = aws_vpc.vpc1.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw1.id
  }
}

resource "aws_route_table" "public_route_table2" {
  vpc_id = aws_vpc.vpc2.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.igw2.id
  }
}

resource "aws_route_table_association" "public1_rta" {
  subnet_id      = aws_subnet.public1.id
  route_table_id = aws_route_table.public_route_table1.id
}

resource "aws_route_table_association" "public2_rta" {
  subnet_id      = aws_subnet.public2.id
  route_table_id = aws_route_table.public_route_table2.id
}

resource "aws_vpc_peering_connection" "peer" {
  vpc_id      = aws_vpc.vpc1.id
  peer_vpc_id = aws_vpc.vpc2.id
  auto_accept = true
}

resource "aws_route" "peer1_to_2" {
  route_table_id            = aws_route_table.public_route_table1.id
  destination_cidr_block    = aws_vpc.vpc2.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_route" "peer2_to_1" {
  route_table_id            = aws_route_table.public_route_table2.id
  destination_cidr_block    = aws_vpc.vpc1.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}

The first run TF creates peering and the correct route and two ec2 instances ping correctly. But if I run for the second time TF I have

# aws_route_table.public_route_table1 will be updated in-place
  ~ resource "aws_route_table" "public_route_table1" {
        id               = "rtb-0ad43422d1bc73f82"
      ~ route            = [
          - {
              - cidr_block                 = "0.0.0.0/0"
              - gateway_id                 = "igw-06ef35a787e283282"
                # (11 unchanged attributes hidden)
            },
          - {
              - cidr_block                 = "10.1.0.0/16"
              - vpc_peering_connection_id  = "pcx-0afbf2e62bf9a43d2"
                # (11 unchanged attributes hidden)
            },
          + {
              + cidr_block = "0.0.0.0/0"
              + gateway_id = "igw-06ef35a787e283282"
            },
        ]
        tags             = {}
        # (5 unchanged attributes hidden)
    }

  # aws_route_table.public_route_table2 will be updated in-place
  ~ resource "aws_route_table" "public_route_table2" {
        id               = "rtb-0f282179701ca2405"
      ~ route            = [
          - {
              - cidr_block                 = "0.0.0.0/0"
              - gateway_id                 = "igw-015a5ba053949c7cf"
                # (11 unchanged attributes hidden)
            },
          - {
              - cidr_block                 = "10.0.0.0/16"
              - vpc_peering_connection_id  = "pcx-0afbf2e62bf9a43d2"
                # (11 unchanged attributes hidden)
            },
          + {
              + cidr_block = "0.0.0.0/0"
              + gateway_id = "igw-015a5ba053949c7cf"
            },
        ]
        tags             = {}
        # (5 unchanged attributes hidden)
    }

Plan: 0 to add, 2 to change, 0 to destroy.

and after applied this changes running TF for the third time

# aws_route.peer1_to_2 will be created
  + resource "aws_route" "peer1_to_2" {
      + destination_cidr_block    = "10.1.0.0/16"
      + id                        = (known after apply)
      + instance_id               = (known after apply)
      + instance_owner_id         = (known after apply)
      + network_interface_id      = (known after apply)
      + origin                    = (known after apply)
      + route_table_id            = "rtb-0ad43422d1bc73f82"
      + state                     = (known after apply)
      + vpc_peering_connection_id = "pcx-0afbf2e62bf9a43d2"
    }

  # aws_route.peer2_to_1 will be created
  + resource "aws_route" "peer2_to_1" {
      + destination_cidr_block    = "10.0.0.0/16"
      + id                        = (known after apply)
      + instance_id               = (known after apply)
      + instance_owner_id         = (known after apply)
      + network_interface_id      = (known after apply)
      + origin                    = (known after apply)
      + route_table_id            = "rtb-0f282179701ca2405"
      + state                     = (known after apply)
      + vpc_peering_connection_id = "pcx-0afbf2e62bf9a43d2"
    }

Plan: 2 to add, 0 to change, 0 to destroy.

What's wrong?

I expected that after the firs apply terraform will not change any resource but instead it continues to apply and remove the resource aws_route. I'm using all updated (terraform and aws provider).

Answer 1

It's probably because you are using a mix of creating a route inline with route inside of the route table resource and the aws_route resource. As per the documentation(check the two notes at the beginning):

Terraform currently provides both a standalone Route resource and a Route Table resource with routes defined in-line. At this time you cannot use a Route Table with in-line routes in conjunction with any Route resources. Doing so will cause a conflict of rule settings and will overwrite rules.

What I would try to do to fix it is the following:

provider "aws" {
region                = "..."
access_key            = "..."
secret_key            = "..."
}

resource "aws_vpc" "vpc1" {
  cidr_block = "10.0.0.0/16"
}

resource "aws_vpc" "vpc2" {
  cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "public1" {
  vpc_id     = aws_vpc.vpc1.id
  cidr_block = "10.0.1.0/24"
}

resource "aws_subnet" "private1" {
  vpc_id     = aws_vpc.vpc1.id
  cidr_block = "10.0.2.0/24"
}

resource "aws_subnet" "public2" {
  vpc_id     = aws_vpc.vpc2.id
  cidr_block = "10.1.1.0/24"
}

resource "aws_subnet" "private2" {
  vpc_id     = aws_vpc.vpc2.id
  cidr_block = "10.1.2.0/24"
}

resource "aws_internet_gateway" "igw1" {
  vpc_id = aws_vpc.vpc1.id
}

resource "aws_internet_gateway" "igw2" {
  vpc_id = aws_vpc.vpc2.id
}

resource "aws_route_table" "public_route_table1" {
  vpc_id = aws_vpc.vpc1.id
}

resource "aws_route" "public_route_1" {
  route_table_id         = aws_route_table.public_route_table1.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.igw1.id
}

resource "aws_route_table" "public_route_table2" {
  vpc_id = aws_vpc.vpc2.id
}

resource "aws_route" "public_route_2" {
  route_table_id         = aws_route_table.public_route_table2.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.igw2.id
}

resource "aws_route_table_association" "public1_rta" {
  subnet_id      = aws_subnet.public1.id
  route_table_id = aws_route_table.public_route_table1.id
}

resource "aws_route_table_association" "public2_rta" {
  subnet_id      = aws_subnet.public2.id
  route_table_id = aws_route_table.public_route_table2.id
}

resource "aws_vpc_peering_connection" "peer" {
  vpc_id      = aws_vpc.vpc1.id
  peer_vpc_id = aws_vpc.vpc2.id
  auto_accept = true
}

resource "aws_route" "peer1_to_2" {
  route_table_id         = aws_route_table.public_route_table1.id
  destination_cidr_block = aws_vpc.vpc2.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_route" "peer2_to_1" {
  route_table_id         = aws_route_table.public_route_table2.id
  destination_cidr_block = aws_vpc.vpc1.cidr_block
  vpc_peering_connection_id = aws_vpc_peering_connection.peer.id

  depends_on = [aws_vpc_peering_connection.peer]
  lifecycle {
    create_before_destroy = true
  }
}