Search code examples
jsonapache-nifitransformationjolt

Transform complex data by jolt processor in apache nifi

I have following expected input and expected output and i want to transform data by jolt processor

Input JSON

{
  "subscriptionId": "63",
  "data": [
    {
      "type": "demo",
      "Data": {
        "transactionId": "598958",
        "type": "json",
        "xyz": "pqr",
        "name": "john"
      },
      "nameData": [
        {
          "name": "Ama",
          "nameCount": "215",
          "genderData": [
            {
              "gender": "Male",
              "genderCount": "140"
            },
            {
              "gender": "Female",
              "genderCount": "75"
            }
          ]
        },
        {
          "name": "Aedes",
          "nameCount": "161",
          "genderData": [
            {
              "gender": "Female",
              "genderCount": "134"
            },
            {
              "gender": "Male",
              "genderCount": "27"
            }
          ]
        },
        {
          "name": "Culex",
          "nameCount": "2610",
          "genderData": [
            {
              "gender": "Male",
              "genderCount": "1926"
            },
            {
              "gender": "Female",
              "genderCount": "684"
            }
          ]
        },
        {
          "name": "Kamp",
          "nameCount": "465",
          "genderData": [
            {
              "gender": "Male",
              "genderCount": "465"
            }
          ]
        }
      ]
    }
  ]
}

Expected Output JSON

{
    "transactionId": "598958",
    "type": "json",
    "xyz": "pqr",
    "name": "john",
    "alert_array": [{
        "abc": "123",
        "xyz": "pqrs",
        "properties": [{
                "key": "Ama",
                "value": "215",
                "object": [{
                        "key": "Male",
                        "value": "140"
                    },
                    {
                        "key": "Female",
                        "value": "75"
                    }
                ]
            },
            {
                "key": "Aedes",
                "value": "161",
                "object": [{
                        "key": "Male",
                        "value": "134"
                    },
                    {
                        "key": "Female",
                        "value": "27"
                    }
                ]
            },
            {
                "key": "Culex",
                "value": "2610",
                "object": [{
                        "key": "Male",
                        "value": "1926"
                    },
                    {
                        "key": "Female",
                        "value": "684"
                    }
                ]
            },
            {
                "key": "Kamp",
                "value": "465",
                "object": [{
                    "key": "Male",
                    "value": "465"
                }]
            },
            {
                "key": "type",
                "value": "demo"
            }
        ]
    }]
}

I have the above following input JSON with nameData being an array and having genderData another array within it. Need to convert the input JSON to expected JSON output as shown above. please suggest JOLT spec to transform the above JSON.

Solution

  • You can start with determining the nodes alert_array,properties,object as diving to the innermost attributes within the shift transformation such as

    [
  {
    "operation": "shift",
    "spec": {
      "data": {
        "*": {
          "Da*": "",
          "nameD*": {
            "#123": "alert_array[0].abc", // # wildcard is used to add fixed values 
            "#pqrs": "alert_array[0].xyz",
            "*": {
              "name": "alert_array[0].properties[&1].key",
              "nameC*": "alert_array[0].properties[&1].value",
              "*": "alert_array[0].properties[&1].object"
            }
          }
        }
      }
    }
  },
  {
    // rename the innermost attributes
    "operation": "modify-overwrite-beta",
    "spec": {
      "*": {
        "*": {
          "*": {
            "*": {
              "*": {
                "*": {
                  "key": "@(1,gender)",
                  "value": "@(1,genderCount)"
                }
              }
            }
          }
        }
      }
    }
  },
  {
    // get rid of the former names of the innermost attributes
    "operation": "remove",
    "spec": {
      "*": {
        "*": {
          "*": {
            "*": {
              "*": {
                "*": {
                  "gender": "",
                  "genderCount": ""
                }
              }
            }
          }
        }
      }
    }
  }
]

    the demo on the site http://jolt-demo.appspot.com/ is

    enter image description here

    Edit : You can add two more shift transformation specs in order to put the lately desired object

    {
  "key" : "type",
  "value" : "demo"
}

    such as

    [
  {
    "operation": "shift",
    "spec": {
      "data": {
        "*": {
          "Da*": "",
          "nameD*": {
            "#123": "alert_array[0].abc", // # wildcard is used to add fixed values 
            "#pqrs": "alert_array[0].xyz",
            "*": {
              "name": "alert_array[0].properties[&1].key",
              "nameC*": "alert_array[0].properties[&1].value",
              "*": "alert_array[0].properties[&1].object"
            }
          }
        }
      }
    }
  },
  {
    // rename the innermost attributes
    "operation": "modify-overwrite-beta",
    "spec": {
      "*": {
        "*": {
          "*": {
            "*": {
              "*": {
                "*": {
                  "key": "@(1,gender)",
                  "value": "@(1,genderCount)"
                }
              }
            }
          }
        }
      }
    }
  },
  {
    // get rid of the former names of the innermost attributes
    "operation": "remove",
    "spec": {
      "*": {
        "*": {
          "*": {
            "*": {
              "*": {
                "*": {
                  "gender": "",
                  "genderCount": ""
                }
              }
            }
          }
        }
      }
    }
  },
  {
    "operation": "shift",
    "spec": {
      "*": "&",
      "alert_array": {
        "*": {
          "*": "&2[&1].&",
          "pro*": {
            "#type": "&3[&2].&1.&2.key",
            "#demo": "&3[&2].&1.&2.value",
            "*": "&3[&2].&1.&"
          }
        }
      }
    }
  },
  {
    "operation": "shift",
    "spec": {
      "*": "&",
      "alert_array": {
        "*": {
          "*": "&2[&1].&",
          "pro*": {
            "*": {
              "@": "&4[&3].&2"
            }
          }
        }
      }
    }
  }
]