Search code examples
azureterraformazure-api-managementterraform-provider-azureazure-application-gateway

InternalServerError for Application Gateway and API Management - Azure/Terraform

I'm trying to deploy an infrastructure in Azure via Terraform, the infrastructure is made of an Application Gateway (tier WAF_v2) and an API Management in the backend.

The error that I get after almost 20 minutes of running is the following: "Error: waiting for create of Application Gateway: (Name "myAppGateway" / Resource Group "csj-hub-euw-chb-rg"): Code="InternalServerError" Message="An error occurred." Details=[]" (see image below for more details)

1

The following is the file that contains the Application Gateway:

module "agw_subnet" {
  source = "../modules/resources-blocks/subnet"

  subnet_name             = "agw_subnet"
  resource_group_name     = module.resource_group.name
  vnet_name               = module.vnet.name
  subnet_address_prefixes = ["10.22.1.0/24"]
}

resource "azurerm_web_application_firewall_policy" "exampleWAF" {
  name                = "example_wafpolicy_name"
  resource_group_name = module.resource_group.name
  location            = module.resource_group.location

  custom_rules {
    name      = "Rule1"
    priority  = 1
    rule_type = "MatchRule"

    match_conditions {
      match_variables {
        variable_name = "RemoteAddr"
      }

      operator           = "IPMatch"
      negation_condition = true
      match_values       = ["x.x.x.x"]
    }

    action = "Block"
  }

  policy_settings {
    enabled                     = true
    mode                        = "Prevention"
    request_body_check          = true
    file_upload_limit_in_mb     = 100
    max_request_body_size_in_kb = 128
  }

  managed_rules {
    managed_rule_set {
      type    = "OWASP"
      version = "3.2"
    }
  }
}

resource "azurerm_application_gateway" "app_gw" {
  name                = "myAppGateway"
  resource_group_name = module.resource_group.name
  location            = module.resource_group.location

  sku {
    name     = "WAF_v2"
    tier     = "WAF_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = module.agw_subnet.id
  }

  frontend_port {
    name = var.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = var.frontend_ip_configuration_name
    public_ip_address_id = module.app_gw_pip.id
  }

  backend_address_pool {
    name = "devBackend"
    ip_addresses = ["10.22.40.19"] 
  }

  backend_http_settings {
    name                  = "devHttpSetting"
    cookie_based_affinity = "Disabled"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 20
    host_name             = "xxxx.be"
    probe_name            = "apim-probe"
  }

  probe {
    interval                                  = 30
    name                                      = "apim-probe"
    path                                      = "/status-0123456789abcdef"
    protocol                                  = "Http"
    timeout                                   = 30
    unhealthy_threshold                       = 3
    pick_host_name_from_backend_http_settings = true
    match {
      body = ""
      status_code = [
        "200-399"
      ]
    }
  }

  http_listener {
    name                           = "devListener"
    frontend_ip_configuration_name = var.frontend_ip_configuration_name
    frontend_port_name             = var.frontend_port_name
    protocol                       = "Http"
    host_name                      = "xxxx.be"
    firewall_policy_id             =  azurerm_web_application_firewall_policy.exampleWAF.id
  }

  request_routing_rule {
    name                       = "devRule"
    rule_type                  = "Basic"
    priority                   = 25
    http_listener_name         = "devListener"
    backend_address_pool_name  = "devBackend"
    backend_http_settings_name = "devHttpSetting"
  }


  firewall_policy_id = var.firewall_policy_id ==""?null : var.firewall_policy_id

  dynamic "waf_configuration"  {
    for_each =  var.waf_configuration
      content{
            enabled                  = lookup(waf_configuration.value,"enabled",true)
            file_upload_limit_mb     = lookup(waf_configuration.value,"file_upload_limit_mb",30)
            firewall_mode            = lookup(waf_configuration.value,"firewall_mode","Prevention")
            max_request_body_size_kb = lookup(waf_configuration.value,"max_request_body_size_kb",128)
            request_body_check       = lookup(waf_configuration.value,"request_body_check",true)
            rule_set_type            = lookup(waf_configuration.value,"rule_set_type","OWASP")
            rule_set_version         = lookup(waf_configuration.value,"rule_set_version", "3.1")
      }
  }
}

The Resource Group is already existing since the infrastructure I'm deploying (App-GTW + APIM) will be a complement of an already created infrastructure contained in this resource group:

module "resource_group" {
  source = "../modules/resources/resource_group"

  resource_group_location = var.LOCATION
  resource_group_name     = local.resource_group_name
}

resource "azurerm_resource_group" "rg" {
  name     = var.resource_group_name
  location = var.resource_group_location
}

Also the Virtual Network where the Subnet of the App-GTW will be hosted is already created (note that the VNET_ADDRESS SPACE is 10.22.0.0/21):

module "vnet" {
  source = "../modules/resources/vnet"

  vnet_name               = local.vnet_name
  resource_group_name     = module.resource_group.name
  resource_group_location = module.resource_group.location
  vnet_address_space      = [var.VNET_ADDRESS_SPACE]

  log_analytics_workspace_id = module.log_analytics_workspace.id
  enable_diagnostic_setting  = true
}
resource "azurerm_virtual_network" "vnet" {
  name                = var.vnet_name
  location            = var.resource_group_location
  resource_group_name = var.resource_group_name
  address_space       = var.vnet_address_space
}

resource "azurerm_subnet" "subnet" {
  name                                           = var.subnet_name
  resource_group_name                            = var.resource_group_name
  virtual_network_name                           = var.vnet_name
  address_prefixes                               = var.subnet_address_prefixes
  service_endpoints                              = var.service_endpoints
  enforce_private_link_endpoint_network_policies = var.enforce_private_link_endpoint_network_policies

  dynamic "delegation" {
    for_each = var.service_delegation == null ? [] : [1]
    content {
      name = "${var.subnet_name}-delegation"
      service_delegation {
        name    = var.service_delegation
        actions = var.delegation_actions
      }
    }
  }
}

So, to do a recap, I'm trying to deploy an App-GTW with an APIM as backend. The VNET that host the App-GTW is already existing, also the Resource Group that will contains the App-GTW is already existing and also the APIM is already existing.

Note also that the APIM is in a different VNET with respect to the App-GTW, in other words, the App-GTW is in a VNET-A (example name) and the APIM is in a VNET-B, the two VNETs are connected toghether via a Virtual Network Peering.

I spent a lot of time reading documentation to try to figure out a solution but I still have this error, so I would ask your help.

Thank you!

Apparently seems that if I create a new Resource Group with a new VNET while deploying the App-GTW instead of using the already existing Resource Group and VNET, I don't get this error.

But the problem is that I have to use the already existing Resource Group and VNET.

Solution

  • I tried to reproduce the same in my environment:

    resource "azurerm_web_application_firewall_policy" "example" {
  name                = "example_wafpolicy_name"
  location            = data.azurerm_resource_group.example.location
  resource_group_name = data.azurerm_resource_group.example.name

  custom_rules {
    name      = "Rule1"
    priority  = 1
    rule_type = "MatchRule"

    match_conditions {
      match_variables {
        variable_name = "RemoteAddr"
      }

      operator           = "IPMatch"
      negation_condition = true
      match_values       = ["x.x.x.x"]
    }

    action = "Block"
  }

  policy_settings {
    ....
  }

  managed_rules {
    managed_rule_set {
      .....
    }
  }
}

resource "azurerm_application_gateway" "app_gateway" {
  name                = "myAppGateway"
  location            = data.azurerm_resource_group.example.location
   resource_group_name = data.azurerm_resource_group.example.name

  sku {
    name     = "WAF_v2"
    tier     = "WAF_v2"
    capacity = 2
  }

  gateway_ip_configuration {
    name      = "my-gateway-ip-configuration"
    subnet_id = module.agw_subnet.id
  }

  frontend_port {
    name = var.frontend_port_name
    port = 80
  }

  frontend_ip_configuration {
    name                 = var.frontend_ip_configuration_name
    public_ip_address_id = module.app_gw_pip.id
  }

  backend_address_pool {
    name = "devBackend"
    ip_addresses = ["10.22.40.19"] 
  }

  backend_http_settings {
    name                  = "devHttpSetting"
    cookie_based_affinity = "Disabled"
    port                  = 80
    protocol              = "Http"
    request_timeout       = 20
    host_name             = "xxxx.be"
    probe_name            = "apim-probe"
  }

  probe {
    interval                                  = 30
    name                                      = "apim-probe"
    path                                      = "/status-0123456789abcdef"
    protocol                                  = "Http"
    timeout                                   = 30
    unhealthy_threshold                       = 3
    pick_host_name_from_backend_http_settings = true
    match {
      body = ""
      status_code = [
        "200-399"
      ]
    }
  }

  http_listener {
    name                           = "devListener"
    frontend_ip_configuration_name = var.frontend_ip_configuration_name
    frontend_port_name             = var.frontend_port_name
    protocol                       = "Http"
    host_name                      = "xxxx.be"
    firewall_policy_id             =  azurerm_web_application_firewall_policy.exampleWAF.id
  }

  request_routing_rule {
    name                       = "devRule"
    rule_type                  = "Basic"
    priority                   = 25
    http_listener_name         = "devListener"
    backend_address_pool_name  = "devBackend"
    backend_http_settings_name = "devHttpSetting"
  }

 ............

    I got some parallel errors :

    enter image description here

    Please note that :

    To communicate with private resources in the back end, Application Gateway and API Management must be in the same virtual network as the resources.

    For that set up a virtual network for your resource. From that solution, it creates subnets for Application Gateway and API Management.

    enter image description here

    See Protect APIs with Azure Application Gateway and Azure API Management - Azure Reference Architectures | Microsoft Learn .