I am trying to migrate data from postgre to Hubspot. I am doing it with integration APIs provided by the Hubspot CRM. The number of API calls can reach up to 1000 at a time. I want to maintain logs showing which records were migrated successfully and which weren't. For this, I need to know which HTTP request returned the result. How can I implement this?
<flow name="migration-to-hubspootFlow" doc:id="0b2433b4-0de3-4f10-92e9-fc1014015043" >
<scheduler doc:name="Scheduler" doc:id="63c6576a-1cca-4a17-b7d7-1f033ba88453" >
<scheduling-strategy >
<fixed-frequency frequency="60" timeUnit="MINUTES"/>
</scheduling-strategy>
</scheduler>
<db:select doc:name="Select" doc:id="b258bb4c-6e34-4b66-a59f-8ae634975373" config-ref="Database_Config">
<db:sql ><![CDATA[select distinct c.email, c.lastmodifieddate, c.id, c.sfid, c.firstname, c.lastname, c.title, c.phone,c.mailingstate, c.mailingstreet, c.mailingcity, c.mailingpostalcode, c.mailingcountry
from salesforceuat.contact c
where c.email is not null
limit 500]]></db:sql>
</db:select>
<batch:job jobName="migration-to-hubspootBatch_Job" doc:id="ba2f58cd-ddd2-4e28-911f-107d6e7333db" >
<batch:process-records >
<batch:step name="Batch_Step" doc:id="f4fa295c-bc1f-4c22-8fb3-6193f415cafb" >
<http:request method="POST" doc:name="Request" doc:id="c7eb69e0-0a43-4526-91e2-022469c203bc" config-ref="HTTP_Request_configuration" path="/contacts/v1/contact/createOrUpdate/email/{userEmail}/" sendBodyMode="ALWAYS" requestStreamingMode="AUTO">
<http:body><![CDATA[#[%dw 2.0
output application/json
---
{
"properties": [
{
"property": "JobTitle",
"value": payload.title
},
{
"property": "firstname",
"value": payload.firstname
},
{
"property": "lastname",
"value": payload.lastname
},
{
"property": "website",
"value": ""
},
{
"property": "company",
"value": "HubSpot"
},
{
"property": "phone",
"value": payload.phone
},
{
"property": "address",
"value": payload.mailingstreet
},
{
"property": "city",
"value": payload.mailingcity
},
{
"property": "state",
"value": payload.mailingstate
},
{
"property": "zip",
"value": payload.mailingpostalcode
}
]
}]]]></http:body>
<http:headers><![CDATA[#[output application/java
---
{
"Host" : "api.hubapi.com",
"Content-Type" : "application/json"
}]]]></http:headers>
<http:uri-params><![CDATA[#[output application/java
---
{
"userEmail" : payload.email
}]]]></http:uri-params>
<http:query-params><![CDATA[#[output application/java
---
{
"hapikey" : "************"
}]]]></http:query-params>
</http:request>
<logger level="INFO" doc:name="Logger" doc:id="531d9bdf-3eca-4159-88b4-6ab86016927b" message="#[message.payload] #[message.attributes]" />
</batch:step>
</batch:process-records>
</batch:job>
</flow>
If you set maxFailedRecords to -1 (docs), then records that result in a mule error (like HTTP:BAD_REQUEST) can be handled in a subsequent batch step that uses acceptPolicy=ONLY_FAILURES (docs). There you can handle them one at a time or all together using a Batch Aggregator.