Dynamic generation of parallel stages in jenkins

Question

I am trying to generate dynamic parallel stages in jenkins. I have reach a point in which it work but they are not dynamic:

pipeline{
  agent {
    label "node-18"
  }
  parameters{
    ...
  }
  environment{
    ...
  }
  stages{
    stage('whatever previos stage'){
      script{
        .... whatever previos things here (it doesnt matter)
      }
    }
  }

  stage('Run test in parallel'){
    parallel{
      stage('test 0'){
        agent{
          label "node-18"
        }
        steps{
          script{
            runTest('0')
          }
        }
        post{
          always{
            publishrepo('0') //this method use the html publisher
          }
        }
      }
      stage('test 1'){
        agent{
          label "node-18"
        }
        steps{
          script{
            runTest('1')
          }
        }
        post{
          always{
            publishrepo('1') //this method use the html publisher
          }
        }
      }
    }
  }
}

This works perfect and also publish the HTML reports in the jenkins page so, so far so good, the problem is that there are not always two stages, there is a method that calculate the max numbers of paralelization and according to this numbers I have to create 0 to N stages.

I tried this checking some post on the internet and so on:

def maxParallelSteps = [:]
pipeline{
  agent {
    label "node-18"
  }
  parameters{
    ...
  }
  environment{
    ...
  }
  stages{
    stage('whatever previos stage'){
      script{
        .... whatever previos things here (it doesnt matter)
      }
    }
  }

  stage('Check number of stages and create them'){
    steps{
      script {
        def max = checkMaxParallel()
        for(int i = 0 ; i< max ; i++){
           maxParallelSteps.put("test ${i}", generateStage(i)
        }
      }
    }
  }

  stage('Run test in parallel'){
    steps{
      script{
        parallel maxParallelSteps 
      }
    }
  }
}

def generateStage(slot){
  return {
    stage('test ' + slot){
      agent{
        label 'node-18'
      }
      steps{
        script{
          runTest('1')
        }
      }
      post{
        always{
          publishrepo('1') //this method use the html publisher
        }
      }
    } 
  }
}

In this case I am getting the following error:

take into account that I didnt copy and paste but the idea is that test 0 and so on are P0_test and so on.

I also tried to move the for loop just in one stage and a lot of crazy things. I dont know what else to test and where to find a proper example of doc about how to dynamically do that. The point is that the first code I showed you works perfectly the only point will be to be able to according to just a number create X number of stages.

I hope I have explain properly my issue and thanks in advance to all!

Answer 1

Creating real dynamic parallel steps in declarative pipeline is indeed not possible, so you used the only available way which is to use the parallel keyword which is taken from the scripted pipeline syntax (thus requires the script context).
This means that the execution code that is passed to the parallel step should be also written in scripted syntax, in your case the generateStage function is generating decelerative syntax code and therefore the errors.

To solve it you can modify your function to scripted syntax:

def generateStage(slot) {
  return {
    stage("test ${slot}") {
       node('node-18') {
          try {
             runTest('1')
          }
          finally {
              publishrepo('1') //this method use the html publisher
          }
      }
    }
  }
}

Btw you don't have to separate into a function and can run it inside the stage:

pipeline{
  agent {
    label "node-18"
  }
  parameters{
    ...
  }
  environment{
    ...
  }
  stages{
    stage('whatever previos stage'){
      script{
        .... whatever previos things here (it doesnt matter)
      }
    }
  }

  stage('Create and run stages'){
    steps{
      script {
        def parallelSteps = (1..checkMaxParallel()).collectEntries {
          ["test ${i}" : {
              stage("test ${slot}") {
                  node('node-18') {
                     try {
                        runTest('1')
                     }
                     finally {
                        publishrepo('1') //this method use the html publisher
                     }
                 }
              }
           }]
        parallel parallelSteps 
      }
    }
  }
}