arraysjsonapache-nifijolt
Filling a JSON array that can have a single element using JOLT V0.1.1
I am working on NiFi and I am wanting to do a JOLT V0.1.1 transformation that generates a JSON array from a JSON input. But I'm having the problem that, when the JSON comes with a single potential element for that array, the output is null. This is the transformation Este es uno de los inputs:
{
"data": {
"type": "applications",
"id": "66g4cfcfc9efaad060007dda346b24",
"attributes": {
"appliedAt": 1716321481,
"completedAt": 1716321566,
"status": "new",
"progress": {
"all": 5,
"completed": 5
},
"matchingScore": 12,
"language": "en-GB",
"matchingProfile": {
"bracketId": "poor",
"label": "Below B2"
}
}
},
"included": [
{
"type": "matching-indicators",
"id": "65bd129c74158f00070d67d4",
"attributes": {
"name": "English Proficiency",
"score": 12,
"bracketId": "poor",
"externalId": "",
"fail": false,
"label": "Below B2"
}
},
{
"type": "matching-indicators",
"id": "65bd1463de98c00007e0fa9d",
"attributes": {
"name": "Situational Judgement Test",
"score": 42,
"bracketId": "good",
"externalId": "",
"fail": false,
"label": "Good Fit",
"brackets": [
{
"bracketId": "good",
"cutoff": false,
"name": "Good Fit",
"treshold": 40
},
{
"bracketId": "great",
"cutoff": false,
"name": "Great Fit",
"treshold": 80
}
]
}
}
]
}
And this is the JOLT transformation that I was using to obtain an array with all the Matching Indicators information:
[
{
"operation": "shift",
"spec": {
"included": {
"*": {
"type": {
"matching-indicators": {
"@2": ""
}
}
}
}
}
},
{
"operation": "shift",
"spec": {
"*": {
"attributes": {
"name": "[&2].name",
"label": "[&2].label",
"score": "[&2].score"
}
}
}
}
]
The output obtained was:
[
{
"name": "English Proficiency",
"label": "Below B2",
"score": 12
},
{
"name": "Situational Judgement Test",
"label": "Good Fit",
"score": 42
}
]
Which is exactly what I need for this case.
Now, when I have just one matching indicator in that input, I get a null output using the exact same JOLT. This is an example of an input with one matching indicator:
{
"data": {
"type": "applications",
"id": "664cfcc9efaa0600rgd0798346b24",
"attributes": {
"appliedAt": 1716321481,
"completedAt": 1716321566,
"status": "new",
"progress": {
"all": 5,
"completed": 5
},
"matchingScore": 12,
"language": "en-GB",
"matchingProfile": {
"bracketId": "poor",
"label": "Below B2"
}
}
},
"included": [
{
"type": "matching-indicators",
"id": "65bd129c74skyt15h8f00070d67d4",
"attributes": {
"name": "English Proficiency",
"score": 12,
"bracketId": "poor",
"externalId": "",
"fail": false,
"label": "Below B2"
}
}
]
}
I tried with this transformation too:
[
{
"operation": "shift",
"spec": {
"included": {
"*": {
"attributes": {
"name": "[&1].name",
"label": "[&1].label",
"score": "[&1].score"
}
}
}
}
}
]
And when I validated it at https://jolt-demo.appspot.com/ it seems to work for both scenarios. But when I use it directly in NiFi, it gives me this kind of outputs for the second input:
[
null,
null,
null,
null,
null,
{
"name": "English Proficiency",
"label": "Below B2",
"score": 12
}
]
How could I obtain the correct output? Is there at least any way to delete those 'null' elements in the same transformation?
This is actually the complete json input:
{
"data": {
"type": "applications",
"id": "664caa6588bf5100078bb060",
"attributes": {
"appliedAt": 1716300389,
"completedAt": 1716300607,
"status": "new",
"progress": {
"all": 12,
"completed": 12
},
"matchingScore": 48,
"language": "en-GB",
"matchingProfile": {
"bracketId": "poor",
"label": "Poor Fit"
}
},
"relationships": {
"candidate": {
"data": {
"type": "candidates",
"id": "664caa6588bf5100078bb060",
"email": "test.test+9@test.net",
"attributes": {
"firstName": "test",
"lastName": "test",
"email": "test.test+9@test.net"
}
}
},
"vacancy": {
"data": {
"type": "vacancies",
"id": "65bd009dde98c00007e0e7b4"
}
},
"report": {
"data": {
"type": "reports",
"id": "17163006095testREPORT.pdf"
}
},
"fact-sheets": {
"data": {
"type": "fact-sheets",
"id": "1716300609489agtest_SHEET.pdf"
}
},
"candidate-detail-page": {
"data": {
"type": "candidate-detail-page",
"id": "664caa6588bf5100078bb060"
}
},
"matching-results": {
"data": [
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bcfffd74158f00070d5a96"
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bd0023de98c00007e0e6a8"
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bd006cde98c00007e0e6fa"
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bd007f74158f00070d5b0f"
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65c6263974b26da67537703d"
}
]
},
"matching-indicators": {
"data": [
{
"type": "matching-indicators",
"id": "65bd128774158f00070d67bd"
},
{
"type": "matching-indicators",
"id": "65bd129c74158f00070d67d4"
},
{
"type": "matching-indicators",
"id": "65bd133fde98c00007e0fa12"
},
{
"type": "matching-indicators",
"id": "65bd1463de98c00007e0fa9d"
}
]
},
"personal-information": {
"data": {
"type": "personal-information",
"id": "65bcfff5de98c00007e0e64b"
}
}
}
},
"included": [
{
"type": "reports",
"id": "171630060testREPORT.pdf",
"attributes": {
"expiresAt": 1716390988,
"url": "https://ZGE4MGRmMmU2MmZhOGQzYmU1NmQ1MGFlMWZiMjhjM2Q3OGQ2NmUyM2M3NjQxZWI5ODY3Nzg1NzdlNDc0ZjM3/ENG/Y29tLmhhcnZlci50ZXN0LmludGVncmF0aW9uLnRyYW5zY29tZW1lYQ=="
}
},
{
"type": "fact-sheets",
"id": "17163006094testT_SHEET.pdf",
"attributes": {
"expiresAt": 1716390988,
"url": "https://e=20240522T150128Z&X-Amz-Expires=900&X-Amz-Signature=c7429579e22e1f72efd57e60c9cd17510c389723ef12fe17659223b408ff3cc4&X-Amz-SignedHeaders=host"
}
},
{
"type": "candidate-detail-page",
"id": "664caa6588bf5100078bb060",
"attributes": {
"url": "https://sults/664caa6588bf5100078bb060"
}
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bcfffd74158f00070d5a96",
"attributes": {
"moduleType": "PersonalityPrintModule",
"calculatedAt": 1716300607,
"moduleScore": 0,
"moduleWeight": 0
},
"relationships": {
"module": {
"data": {
"type": "modules",
"id": "65bcfffd74158f00070d5a96"
}
}
},
"meta": {
"dimensionScores": {
"agreeableness": {
"percentile": 5.7,
"rawScore": 3.5
},
"assertiveness": {
"percentile": 91.14,
"rawScore": 4.333333333333333
},
"empathy": {
"percentile": 23.9,
"rawScore": 4.125
},
"followThrough": {
"percentile": 61.3,
"rawScore": 5
},
"learningFocused": {
"percentile": 81.05,
"rawScore": 5
},
"multitasking": {
"percentile": 18,
"rawScore": 3
},
"organization": {
"percentile": 0.01,
"rawScore": 2.25
},
"preferenceForDirection": {
"percentile": 0.01,
"rawScore": 2.25
},
"resilience": {
"percentile": 77.82,
"rawScore": 5.166666666666667
},
"sociability": {
"percentile": 84.81,
"rawScore": 5
},
"workIntensity": {
"percentile": 4.3,
"rawScore": 3.142857142857143
}
},
"normGroup": "europe",
"variant": "short"
}
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bd0023de98c00007e0e6a8",
"attributes": {
"moduleType": "NOAModule",
"calculatedAt": 1716300607,
"moduleScore": 0,
"moduleWeight": 0
},
"relationships": {
"module": {
"data": {
"type": "modules",
"id": "65bd0023de98c00007e0e6a8"
}
}
},
"meta": {
"components": [
{
"normGroup": "medium",
"score": 0,
"type": "NOA Exclusion"
}
]
}
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bd006cde98c00007e0e6fa",
"attributes": {
"moduleType": "LanguageTestModule",
"calculatedAt": 1716300607,
"moduleScore": 16.333333333333332,
"moduleWeight": 0
},
"relationships": {
"module": {
"data": {
"type": "modules",
"id": "65bd006cde98c00007e0e6fa"
}
}
},
"meta": {
"grading": "<B2",
"sectionScores": [
{
"name": "vocabulary",
"score": 13
},
{
"name": "grammar",
"score": 20
},
{
"name": "comprehension",
"score": 16
}
]
}
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65bd007f74158f00070d5b0f",
"attributes": {
"moduleType": "JobKnowledgeTestModule",
"calculatedAt": 1716300607,
"moduleScore": 0,
"moduleWeight": 0
},
"relationships": {
"module": {
"data": {
"type": "modules",
"id": "65bd007f74158f00070d5b0f"
}
}
},
"meta": {
"scoresByQuestionId": {
"0daeeeb4-eabd-4b64-9f0f-7d01fb09bf08": 0,
"2dba74a2-d527-4c65-a734-2d6017355ac3": 0,
"3bbe3045-6a74-463a-8242-8f8b1ae778c7": 0,
"3db7a0c7-0027-4836-8208-475e41009d56": 0,
"613592b8-6b95-4e55-ac87-2dc3d08f92c7": 0,
"68a710b6-ac2b-4986-90e2-5d14695d3405": 0,
"6c331725-12f0-4ebd-8282-37a1361771d8": 0,
"8156b45b-6c4f-469d-8698-902e6cc12400": 0,
"8a1c0138-3f57-42d0-bbf7-4c6f577ab5a9": 0,
"bee5f06f-7353-4de7-b078-4ce17fa58542": 0
}
}
},
{
"type": "matching-results",
"id": "664caa6588bf5100078bb060-65c6263974b26da67537703d",
"attributes": {
"moduleType": "SituationalJudgmentTestModule",
"calculatedAt": 1716300607,
"moduleScore": 41.66666666666667,
"moduleWeight": 0
},
"relationships": {
"module": {
"data": {
"type": "modules",
"id": "65c6263974b26da67537703d"
}
}
},
"meta": {
"pointsPerQuestion": {
"026c7707-7f0e-43ef-bfc0-c165f6a1e5e2": {
"points": -1,
"score": 25
},
"5a4922e4-b15d-4e5f-9fb7-566ee88b7a9c": {
"points": 1,
"score": 75
},
"a0052eaa-fc4f-4235-b5d4-77209c79e0db": {
"points": -1,
"score": 25
}
}
}
},
{
"type": "ats-parameters",
"attributes": {
"candidateId": "395303",
"jobId": "893",
"createdBy": "api"
}
},
{
"type": "matching-indicators",
"id": "65bd128774158f00070d67bd",
"attributes": {
"name": "Logical Reasoning",
"score": 0,
"bracketId": "poor",
"externalId": "",
"fail": false,
"label": "Poor Fit",
"brackets": [
{
"bracketId": "good",
"cutoff": false,
"name": "Good Fit",
"treshold": 40
},
{
"bracketId": "great",
"cutoff": false,
"name": "Great Fit",
"treshold": 66
}
]
}
},
{
"type": "matching-indicators",
"id": "65bd129c74158f00070d67d4",
"attributes": {
"name": "English Proficiency",
"score": 16,
"bracketId": "poor",
"externalId": "",
"fail": false,
"label": "Below B2",
"brackets": [
{
"bracketId": "good",
"cutoff": false,
"name": "B2 and above",
"treshold": 66
}
]
}
},
{
"type": "matching-indicators",
"id": "65bd133fde98c00007e0fa12",
"attributes": {
"name": "Customer Support Personality V2TEST",
"score": 57,
"bracketId": "poor",
"externalId": "",
"fail": false,
"label": "Poor Fit",
"brackets": [
{
"bracketId": "good",
"cutoff": false,
"name": "Good Fit",
"treshold": 65
},
{
"bracketId": "great",
"cutoff": false,
"name": "Great Fit",
"treshold": 80
}
]
}
},
{
"type": "matching-indicators",
"id": "65bd1463de98c00007e0fa9d",
"attributes": {
"name": "Situational Judgement Test",
"score": 42,
"bracketId": "good",
"externalId": "",
"fail": false,
"label": "Good Fit",
"brackets": [
{
"bracketId": "good",
"cutoff": false,
"name": "Good Fit",
"treshold": 40
},
{
"bracketId": "great",
"cutoff": false,
"name": "Great Fit",
"treshold": 80
}
]
}
}
]
}
And I am obtaining this output:
[ null, null, null, null, null, null, null, null, null, {
"name" : "Logical Reasoning",
"score" : 0,
"label" : "Poor Fit"
}, {
"name" : "English Proficiency",
"score" : 16,
"label" : "Below B2"
}, {
"name" : "Customer Support Personality V2TEST",
"score" : 57,
"label" : "Poor Fit"
}, {
"name" : "Situational Judgement Test",
"score" : 42,
"label" : "Good Fit"
} ]
Solution
The problem is due to missing square brackets for the key-value pair nested within "matching-indicators" object, that should be : "@2": "[]" instead of "@2": "".
eg use :
"matching-indicators": {
"@2": "[]"
}
That technique is mostly used for the sake of single objects nested within an array, and doesn't lead a problem for the multiple objects nested within an array as well.
Indeed, only one shift transformation spec would handle the issue(for both cases) such as
[
{
"operation": "shift",
"spec": {
"included": {
"*": {
"type": {
"matching-indicators": {
"@2,attributes": {
"name|score|label": "[&4].&"
}
}
}
}
}
}
}
]
Demo for Case 1 ( on https://jolt-demo.appspot.com/ ) :
Demo for Case 2 :
