How to use the nextToken in the Step function's GetHistoryExecution?

Question

I am trying to get all the execution history using lambda function and store it to DynamoDB. The function returns about 20 executions and a string value called NextToken that is to be used in the next call to get the rest of the executions.

This is my code.

const AWS = require('aws-sdk');
const dynamoDb = new AWS.DynamoDB.DocumentClient({
   region: 'myregion' 
});

exports.handler = (event, context, callback) => {
    const table = 'myDynamoDB';
    const executionARN = "arn:aws:region:accountid:execution:myStateMachine:test";
    var stepfunctions = new AWS.StepFunctions();
    var params = {
        executionArn: executionARN,
        maxResults: 20,
        nextToken: null,
        reverseOrder: false
    };

    stepfunctions.getExecutionHistory(params, function(err, dataExecution) {
        if (err){
          console.log(err, err.stack);
        } 
        else {
            const params2 = {
                TableName: table,
                Item: {
                    id: executionARN,
                    execution_history: dataExecution
                }
            };
                dynamoDb.put(params2).promise();
        }
    });
};

Answer 1

nextToken needs to be used in parms pass to next call to getExecutionHistory. You can recursively call this function till all tokens are exhausted. Came across similar situation while fetching logs via Cloud watch.

Sample for fetching history recursively,

Wrap getExecutionHistory into promise and add to different JS file (let's say writer.js) then your main index.js file can call that function like this,

// writer.js which writes record to Dynamodb
// returns promise
// when history is fetched , dynamodb will be inserted and it will resolve dataexecution which has nextToken

module.exports.get = function(fwdtoken) {    

    if (fwdtoken) parms.nextToken= fwdtoken;

    return new Promise ( (resolve, reject)=>{
    stepfunctions.getExecutionHistory(params, function(err, dataExecution) {
        if (err){
           reject(err.stack)
        } 
        else {
            const params2 = {
                TableName: table,
                Item: {
                    id: executionARN,
                    execution_history: dataExecution
                }
            };
            dynamoDb.put(params2).promise();
        resolve(dataExecution)
        }
      });   
    })    
};  

//This goes in main logic
// Invokes getAllLogs recursilvely

var writer = require('./writer');
var fwdtoken;

function getAllLogs(fwdtoken, fetchCount) {    
    fetchCount = fetchCount || 0;
    if (fetchCount > 40) {
        throw new Error("Fetched too many times.");
    }
    return new Promise( (resolve) => {
            writer.get(fwdtoken).then( function consolidate( dataExecution ) {                  
            resolve( dataExecution );            
        });    
    })
    .then(function ( dataExecution ) {
        if (dataExecution.nextForwardToken) {
            fwdtoken = dataExecution.nextForwardToken;
            getAllLogs(fwdtoken, fetchCount+ 1)
        }
        else
        return fwdtoken        
    });
}
getAllLogs(fwdtoken, 0);