Declarative pipeline dynamic, parallel stages on Kubernetes agents?

Question

Jenkins 3.346.2

My proof of concept.

pipeline {
  agent {
    label 'master'
  }
  stages {
    stage('Test') {
      steps {
        script {
          // The would be dynamically determined.
          projects = [
            [ name: 'project1', dir: 'path/1' ],
            [ name: 'project2', dir: 'path/2' ],
            [ name: 'project3', dir: 'path/3' ]
          ]
          
          projectStages = [:]

          projects.each { project ->          
            projectStages[project.name] = node {
              agent {
                kubernetes {
                  // Load a pod definition from a shared library.
                  yaml libraryResource('my-agent.yaml')
                }
              }
              stages {
                stage("Test $project.name") {
                  steps {
                    container('my-build-container') {
                      echo "Running: $project.name"
                      // Hostnames should be different (one for each project/pod).
                      sh('hostname')
                    }
                  }
                }
              }
            }
          }
          
          parallel projectStages
        }
      }
    }
  }
}

It gets stuck on the projects.each line and hangs indefinitely.

12:52:38  [Pipeline] node
12:52:53  Still waiting to schedule task
12:52:53  ‘Jenkins’ is reserved for jobs with matching label expression
etc...

Answer 1

Here is the solution I found.

I had the right idea but the syntax was a bit off. Here is what I ended up with (and it builds and runs in parallel as expected).

pipeline {
  agent {
    label 'master'
  }
  stages {
    stage('Test') {
      steps {
        script {
          // These would be dynamically determined.
          projects = [
            [ name: 'project1', dir: 'path/1' ],
            [ name: 'project2', dir: 'path/2' ],
            [ name: 'project3', dir: 'path/3' ]
          ]

          projectStages = [:]

          echo "Found projects:\n$projects"

          projects.each { project ->
            projectStages[project.name] = generateStage(project.name, project.dir)
          }

          echo 'running...'

          parallel projectStages

          echo 'done...'
        }
      }
    }
  }
}

def generateStage(name, dir) {
  def podLabel = "my-test-pod-$name"
  return {
    stage("Test $name") {
      script {
        podTemplate(
          label: podLabel,
          yaml: libraryResource('my-agent.yaml')
        ) {
          node(podLabel) {
            container('my-build-container') {
              // Execute some steps in this container first.
              echo "Running: $name at $dir"
            }
            container('my-test-container') {
              // Execute some steps in this container next.
              sh('hostname')
            }
          }
        }
      }
    }
  }
}

And from Jenkins: