Search code examples
apache-nifitransformationcombineshiftjolt

JOLT: "multiply" each record of an array with each record of another array


Source JSON (example) I try to combine each record of "connections" with each record of "tests". (I am a beginner in terms of JOLT and stack overflow so please be lenient ;-)

{
  "scen_id": 62,
  "parameters": {
    "name": "TWAMP_S1NR_VBEI",
    "duration": 1,
    "upload": 60,
    "endless_duration": true,
    "scen_id": 62
  },
  "connections": [
    {
      "synchro": false,
      "manufacturer": 6,
      "light": true,
      "conn_id": 1,
      "_id": "63051ddf26a5ce557ee2cf39",
      "index": 0
    },
    {
      "synchro": false,
      "manufacturer": 6,
      "light": true,
      "conn_id": 2,
      "_id": "63051ddf26a5ce557ee2cf38",
      "index": 1
    }
  ],
  "tests": [
    {
      "name": "TOS30",
      "test_id": 1,
      "_id": "63051ddf26a5ce557ee2cf3a"
    },
    {
      "name": "TOS31",
      "test_id": 2,
      "_id": "63051ddf26a5ce557ee2cf3a"
    }
  ]
}

JOLT spec I tried

I intended to loop through one of the arrays first and "multiply" it with each record of the second array.

[
  {
    "operation": "shift",
    "spec": {
      "connections": {
        "*": {
          "*": "&2.[&1].&",
          "@(2,tests)": "&2.[&1].tests"
        }
      }
    }
  }
]

The result is this

I add the entire second array to each record of array1

{
  "connections" : [ {
    "tests" : [ {
      "name" : "TOS30",
      "test_id" : 1,
      "_id" : "63051ddf26a5ce557ee2cf3a"
    }, {
      "name" : "TOS31",
      "test_id" : 2,
      "_id" : "63051ddf26a5ce557ee2cf3a"
    } ],
    "synchro" : false,
    "manufacturer" : 6,
    "light" : true,
    "conn_id" : 1,
    "_id" : "63051ddf26a5ce557ee2cf39",
    "index" : 0
  }, {
    "tests" : [ {
      "name" : "TOS30",
      "test_id" : 1,
      "_id" : "63051ddf26a5ce557ee2cf3a"
    }, {
      "name" : "TOS31",
      "test_id" : 2,
      "_id" : "63051ddf26a5ce557ee2cf3a"
    } ],
    "synchro" : false,
    "manufacturer" : 6,
    "light" : true,
    "conn_id" : 2,
    "_id" : "63051ddf26a5ce557ee2cf38",
    "index" : 1
  } ]
}

What I expected

{
  "connections": [
    {
      "synchro": false,
      "manufacturer": 6,
      "light": true,
      "conn_id": 1,
      "_id": "63051ddf26a5ce557ee2cf39",
      "index": 0,
      "name": "TOS30",
      "test_id": 1,
      "_id": "63051ddf26a5ce557ee2cf3a"
    },
    {
      "synchro": false,
      "manufacturer": 6,
      "light": true,
      "conn_id": 1,
      "_id": "63051ddf26a5ce557ee2cf39",
      "index": 0,
      "name": "name": "TOS3",
      "test_id": 2,
      "_id": "63051ddf26a5ce557ee2cf3a"
    },
    {
      "synchro": false,
      "manufacturer": 6,
      "light": true,
      "conn_id": 2,
      "_id": "63051ddf26a5ce557ee2cf38",
      "index": 1,
      "name": "TOS30",
      "test_id": 1,
      "_id": "63051ddf26a5ce557ee2cf3a"
    },
    {
      "synchro": false,
      "manufacturer": 6,
      "light": true,
      "conn_id": 2,
      "_id": "63051ddf26a5ce557ee2cf38",
      "index": 1,
      "name": "name": "TOS3",
      "test_id": 2,
      "_id": "63051ddf26a5ce557ee2cf3a"      
    }   
  ]
}

Every hint / explanation is highly appreciated!


Solution

  • You can use such a spec to flatten the JSON :

    [
      {
        "operation": "shift",
        "spec": {
          "connections": {
            "*": {
              "*": "&1.&",
              "@(2,tests)": "&1.tests"
            }
          }
        }
      },
      {
        "operation": "shift",
        "spec": {
          "*": {
            "tests": {
              "*": {
                "@(2,synchro)": "&3.&1.synchro",
                "@(2,manufacturer)": "&3.&1.manufacturer",
                "@(2,light)": "&3.&1.light",
                "@(2,conn_id)": "&3.&1.conn_id",
                "@": "&3.&1"
              }
            }
          }
        }
      },
      {
        "operation": "shift",
        "spec": {
          "*": {
            "*": {
              "*": {
                "*": "&3[&2].&"
              }
            }
          }
        }
      },
      {
        "operation": "shift",
        "spec": {
          "*": {
            "*": ""
          }
        }
      }
    ]
    

    where I only picked the _id attribute inside test array, since a JSON value, which's expected in this case, cannot have duplicate keys.