Search code examples
terraformdatabricksazure-databricksterraform-provider-databricks

Terraform combine multiple attribute blocks into one


I am trying to automate and provision DataBricks cluster in DataBricks workspace using terraform.

We have predefined libraries which need to be installed on cluster creation, we do have roughly 20 libraries

variable"libraries"{
default = ["pypi","jar"]
}

variable"pythonpackages"{
default = ["conda","pymongo"]
}

variable"jarpackages"{
default = ["etc1","etc1"]
}

variable"repositories"{
default = ["internal1","internal2"]
}

resource "databricks_cluster" "shared_autoscaling" {

  cluster_name            = "Shared Autoscaling"
  spark_version           = data.databricks_spark_version.latest_lts.id
  node_type_id            = data.databricks_node_type.smallest.id
  autotermination_minutes = 20
  autoscale {
    min_workers = 1
    max_workers = 50
  }
  library { 
    pypi  {
        package = "dnspython"
        }
    }
      library { 
    pypi  {
        package = "condo"
        }
    }
         library { 
    pypi  {
        package = "kedro"
        repo = "artifactory-url"
        }
    }
  
}

I am trying to remove the library attribute duplicacy. I tried it with for_each inside the library attribute but it did not work out right.

What I am trying to achieve

variable"libraries"{
default = ["pypi","jar"]
}

variable"pythonpackages"{
default = ["conda","pymongo"]
}

variable"jarpackages"{
default = ["etc1","etc1"]
}

variable"repositories"{
default = ["internal1","internal2"]
}

resource "databricks_cluster" "shared_autoscaling" {

  cluster_name            = "Shared Autoscaling"
  spark_version           = data.databricks_spark_version.latest_lts.id
  node_type_id            = data.databricks_node_type.smallest.id
  autotermination_minutes = 20
  autoscale {
    min_workers = 1
    max_workers = 50
  }

      library { 
    pypi  {
        package = ["condo","pymongo","listofpackages"]
        }
    }
  
}

Solution

  • You can use dynamic blocks to repeat nested blocks.

    To repeat the library block for example, you can write:

    resource "databricks_cluster" "shared_autoscaling" {
      cluster_name            = "Shared Autoscaling"
      spark_version           = data.databricks_spark_version.latest_lts.id
      node_type_id            = data.databricks_node_type.smallest.id
      autotermination_minutes = 20
      autoscale {
        min_workers = 1
        max_workers = 50
      }
      
      dynamic "library" {
        for_each = ["condo","pymongo","listofpackages"]
        content {
          pypi {
            package = library.value
          }
        }
      }
     }