I need to receive JSON array of objects (with almost any shape) and store them to ES database using function IndexMany
(or some similar bulk indexing function). I found some clumsy solution with one drawback - it doesn't set _id
property in ES correctly (according to the id
property in JSON object).
And also I would like to know if there is any more elegant way how to achieve my goal without casting every JArray item to string and back to ExpandoObject.
Elasticsearch DB 7.5.1
NEST (7.6.1)
Newtonsoft.Json (12.0.3)
Is there any elegant solution of following code:
var settings = new ConnectionSettings(new Uri("http://localhost:9200")).DefaultIndex("people");
var client = new ElasticClient(settings);
// this string represents incoming JSON message
string json = @"[
{
""id"": ""1"",
""name"": ""Andrej"",
""surname"": ""Burak"",
""dob"": ""1921-11-10T00:00:00+00:00""
},
{
""id"": ""2"",
""name"": ""Franta"",
""surname"": ""Dobrota"",
""dob"": ""1933-10-05T00:00:00+00:00""
},
{
""id"": ""3"",
""name"": ""Milos"",
""surname"": ""Ovcacek"",
""dob"": ""1988-05-05T00:00:00+00:00""
}
]";
JArray jArray = JArray.Parse(json);
foreach (var jtoken in jArray)
{
var jobj = (JObject)jtoken;
jobj.Add("DbCreated", JToken.FromObject(DateTime.UtcNow));
jobj.Add("DbCreatedBy", JToken.FromObject("authors name"));
}
//working, but seems to me a bit too clumsy to convert every item to string and then back to dynamic object
var converter = new ExpandoObjectConverter();
dynamic[] dlst = jArray.Select(t => (dynamic)JsonConvert.DeserializeObject<ExpandoObject>(t.ToString(), converter)).ToArray();
//not working cast
dynamic[] dlstNW = jArray.ToObject<dynamic[]>();
var indexManyResponse = client.IndexMany(dlst); //working partially (but not using ID as index)
var indexManyResponseNotWorking = client.IndexMany(jArray); //not working
var indexManyResponseNotWorking2 = client.IndexMany(dlstNW); //not working
// expected behavior
dynamic[] jsondyn = new dynamic[]
{
new { Id = "1", Name = "foo" },
new { Id = "2", Name = "bar" },
new { Id = "3", Name = "baz" },
};
var indexManyResponseWithIndex = client.IndexMany(jsondyn); //working perfectly, but don't know how to acieve this
After @gnud pointed me to low-level client I think I have found my answer. Important thing is to build request manually like it was explained here.
var settings = new ConnectionSettings(new Uri("http://localhost:9200")).DefaultIndex("people").DisableDirectStreaming();
var client = new ElasticClient(settings);
// this string represents incoming JSON message
string json = @"[
{
""id"": ""1"",
""name"": ""Andrej"",
""surname"": ""Burak"",
""dob"": ""1921-11-10T00:00:00+00:00""
},
{
""Id"": ""2"",
""name"": ""Franta"",
""surname"": ""Dobrota"",
""dob"": ""1933-10-05T00:00:00+00:00""
},
{
""Id"": ""3"",
""name"": ""Milos"",
""surname"": ""Ovcacek"",
""dob"": ""1988-05-05T00:00:00+00:00""
}
]";
JArray jArray = JArray.Parse(json);
// I have to build my request manually
List<string> esRequest = new List<string>();
foreach (var jtoken in jArray)
{
var jobj = (JObject)jtoken;
jobj.Add("DbCreated", JToken.FromObject(DateTime.UtcNow));
jobj.Add("DbCreatedBy", JToken.FromObject("authors name"));
string id;
// Ensure that Id is set even if we receive it in lowercase
if( jobj["Id"] == null)
{
if( jobj["id"] == null)
{
throw new Exception("missing Id");
}
id = jobj["id"].Value<string>();
}
else
{
id = jobj["Id"].Value<string>();
}
string indexName = "people";
esRequest.Add($"{{\"index\":{{\"_index\":\"{indexName}\",\"_id\":\"{id}\"}}}}");
esRequest.Add(jobj.ToString(Formatting.None));
}
PostData pd = PostData.MultiJson(esRequest);
var llres = client.LowLevel.Bulk<BulkResponse>(pd);
Hope it helps someone.