I have solved a lot of this issue I am now left wondering what is the issue and what I am possibly doing wrong.
The console query I am aiming to recreate using NEST is:
{
"tdindex" : {
"mappings" : {
"documentModel" : {
"properties" : {
"accessControl" : {
"type" : "boolean",
"copy_to" : [
"copyTo"
]
},
"comments" : {
"properties" : {
"comment" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]},
"createDate" : {
"type" : "date",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]},
"user" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]
}
}
},
"copyTo" : {
"type" : "text"
},
"documentType" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]
},
"filename" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]
},
"folderID" : {
"type" : "long",
"copy_to" : [
"copyTo"
]
},
"metadata" : {
"properties" : {
"key" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]
},
"value" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]
}
}
},
"uploadDate" : {
"type" : "date",
"copy_to" : [
"copyTo"
]
},
"uploadUser" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
},
"copy_to" : [
"copyTo"
]
}
}
}
}
}
}
I currently have the following Console query being created my nest:
{
"mappings": {
"properties": {
"folderid": {
"copy_to": [
"CopyTo"
],
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"filename": {
"copy_to": [
"CopyTo"
],
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"documenttype": {
"copy_to": [
"CopyTo"
],
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"uploaddate": {
"copy_to": [
"CopyTo"
],
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"uploaduser": {
"copy_to": [
"CopyTo"
],
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"comments": {
"properties": {
"createdate": {
"type": "date"
},
"comment": {
"type": "text"
},
"user": {
"type": "text"
}
},
"copy_to": [
"CopyTo"
],
"type": "nested"
},
"metadata": {
"properties": {
"key": {
"type": "text"
},
"value": {
"type": "text"
}
},
"copy_to": [
"CopyTo"
],
"type": "nested"
},
"CopyTo": {
"type": "text"
}
}
}
}
As you can see it is leaving out the DocumentModel…
The NEST code I am using is:
_elasticClient.CreateIndex(indexParameters.IndexName, c =>
{
c.Map<DocumentModel>(m => m
.Properties(ps => ps
.Text(t => t.Name(n => n.FolderID).CopyTo(n => n.Field(f => f.CopyTo)).Fields(fd => fd.Keyword(k => k.Name("keyword").IgnoreAbove(256))))
.Text(t => t.Name(n => n.Filename).CopyTo(n => n.Field(f => f.CopyTo)).Fields(fd => fd.Keyword(k => k.Name("keyword").IgnoreAbove(256))))
.Text(t => t.Name(n => n.DocumentType).CopyTo(n => n.Field(f => f.CopyTo)).Fields(fd => fd.Keyword(k => k.Name("keyword").IgnoreAbove(256))))
.Text(t => t.Name(n => n.UploadDate).CopyTo(n => n.Field(f => f.CopyTo)).Fields(fd => fd.Keyword(k => k.Name("keyword").IgnoreAbove(256))))
.Text(t => t.Name(n => n.uploadUser).CopyTo(n => n.Field(f => f.CopyTo)).Fields(fd => fd.Keyword(k => k.Name("keyword").IgnoreAbove(256))))
.Nested<Comments>(cm => cm.Name(n => n.Comments).AutoMap().CopyTo(n => n.Field(f => f.CopyTo)))
.Nested<Metadata>(md => md.Name(n => n.Metadata).AutoMap().CopyTo(n => n.Field(f => f.CopyTo)))
.Text(t => t.Name(n => n.CopyTo)))
My DocumentModel looks like this (I cannot use [ElasticsearchType(Name = "documentModel")] for some reason in the model not sure if this is part of the cause of the problem?):
public class DocumentModel
{
[Text(Name = "filename")]
public string Filename { get; set; }
[Text(Name = "folderid")]
public int FolderID { get; set; }
[Text(Name = "uploaduser")]
public string uploadUser { get; set; }
[Date(Format = "MMddyyyy")]
public DateTime UploadDate { get; set; }
[Text(Name = "documenttype")]
public string DocumentType { get; set; }
[Boolean(NullValue = false)]
public bool AccessControl { get; set; }
[Nested]
public List<Comments> Comments { get; set; }
[Nested]
public List<Metadata> Metadata { get; set; }
[Text(Name = "CopyTo")]
public string CopyTo { get; set; }
}
public class Comments
{
[Date(Format = "MMddyyyy")]
public DateTime CreateDate { get; set; }
[Text(Name = "comment")]
public string Comment { get; set; }
[Text(Name = "user")]
public string User { get; set; }
}
public class Metadata
{
[Text(Name = "key")]
public string Key { get; set; }
[Text(Name = "value")]
public string Value { get; set; }
}
The error I get from the Console when running the query is:
{
"error": {
"root_cause": [
{
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [metadata : {copy_to=[CopyTo], type=nested, properties={value={type=text}, key={type=text}}}] [filename : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [comments : {copy_to=[CopyTo], type=nested, properties={createdate={type=date}, comment={type=text}, user={type=text}}}] [uploaddate : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [CopyTo : {type=text}] [documenttype : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [folderid : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [uploaduser : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}]"
}
],
"type": "mapper_parsing_exception",
"reason": "Failed to parse mapping [properties]: Root mapping definition has unsupported parameters: [metadata : {copy_to=[CopyTo], type=nested, properties={value={type=text}, key={type=text}}}] [filename : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [comments : {copy_to=[CopyTo], type=nested, properties={createdate={type=date}, comment={type=text}, user={type=text}}}] [uploaddate : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [CopyTo : {type=text}] [documenttype : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [folderid : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [uploaduser : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}]",
"caused_by": {
"type": "mapper_parsing_exception",
"reason": "Root mapping definition has unsupported parameters: [metadata : {copy_to=[CopyTo], type=nested, properties={value={type=text}, key={type=text}}}] [filename : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [comments : {copy_to=[CopyTo], type=nested, properties={createdate={type=date}, comment={type=text}, user={type=text}}}] [uploaddate : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [CopyTo : {type=text}] [documenttype : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [folderid : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}] [uploaduser : {copy_to=[CopyTo], fields={keyword={ignore_above=256, type=keyword}}, type=text}]"
}
},
"status": 400
Any Ideas?
I Assume the issue is due to the Map?
P.S. I am running Kibana 6.7.1 and Elastic 6.7.1 over localhost. I am running the query (from the JSON object created by the request using NEST)
For Elasticsearch 6.7.1, please use the latest 6.x NEST client, which is 6.7.0 at this time. Major versions of the client are compatible with major versions of Elasticsearch.
With NEST 6.7.0, the mapping would be something like
private static void Main()
{
var defaultIndex = "tdindex";
var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
var settings = new ConnectionSettings(pool)
.DefaultIndex(defaultIndex);
var client = new ElasticClient(settings);
var visitor = new MyVisitor();
client.CreateIndex(defaultIndex, c => c
.Mappings(m => m
.Map<DocumentModel>(mm => mm
.AutoMap(visitor)
.Properties(ps => ps
.Nested<Comments>(cm => cm
.Name(n => n.Comments)
.AutoMap(visitor)
)
.Nested<Metadata>(md => md
.Name(n => n.Metadata)
.AutoMap(visitor)
)
.Text(t => t.Name(n => n.CopyTo))
)
)
)
);
}
public class MyVisitor : NoopPropertyVisitor
{
public override void Visit(ITextProperty property, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute)
{
base.Visit(property, propertyInfo, attribute);
property.CopyTo = Infer.Fields<DocumentModel>(f => f.CopyTo);
property.Fields = new Properties
{
{ "keyword", new KeywordProperty { IgnoreAbove = 256 } }
};
}
}
[ElasticsearchType(Name = "documentModel")]
public class DocumentModel
{
[Text(Name = "filename")]
public string Filename { get; set; }
[Text(Name = "folderid")]
public int FolderID { get; set; }
[Text(Name = "uploaduser")]
public string uploadUser { get; set; }
[Date(Format = "MMddyyyy")]
public DateTime UploadDate { get; set; }
[Text(Name = "documenttype")]
public string DocumentType { get; set; }
[Boolean(NullValue = false)]
public bool AccessControl { get; set; }
[Nested]
public List<Comments> Comments { get; set; }
[Nested]
public List<Metadata> Metadata { get; set; }
[Text(Name = "copyTo")]
public string CopyTo { get; set; }
}
public class Comments
{
[Date(Format = "MMddyyyy")]
public DateTime CreateDate { get; set; }
[Text(Name = "comment")]
public string Comment { get; set; }
[Text(Name = "user")]
public string User { get; set; }
}
public class Metadata
{
[Text(Name = "key")]
public string Key { get; set; }
[Text(Name = "value")]
public string Value { get; set; }
}
Since all of the text mappings except CopyTo property have a keyword multi_fields and copy_to to copy to the CopyTo field, the easiest way to define this is with a visitor. First, Automap() is called, passing the visitor. Automapping will pick up the attribute mappings on the models, and the Visit methods on the visitor will allow us to override any of these. Next, Properties() will override any mappings from the automapping process.
The final output mapping is
PUT http://localhost:9200/tdindex?pretty=true
{
"mappings": {
"documentModel": {
"properties": {
"filename": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"folderid": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uploaduser": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uploadDate": {
"type": "date",
"format": "MMddyyyy"
},
"documenttype": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"accessControl": {
"type": "boolean",
"null_value": false
},
"comments": {
"type": "nested",
"properties": {
"createDate": {
"type": "date",
"format": "MMddyyyy"
},
"comment": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"user": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"metadata": {
"type": "nested",
"properties": {
"key": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"value": {
"type": "text",
"copy_to": [
"copyTo"
],
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"copyTo": {
"type": "text"
}
}
}
}
}
The syntax for NEST 7.x, which is compatible with Elasticsearch 7.x, is the same; there are a couple of things that are obsolete that can be changed to remove the warnings, but the 6.x syntax works as is
client.CreateIndex(defaultIndex, c => c
// remove .Mappings()
.Map<DocumentModel>(mm => mm
.AutoMap(visitor)
.Properties(ps => ps
.Nested<Comments>(cm => cm
.Name(n => n.Comments)
.AutoMap(visitor)
)
.Nested<Metadata>(md => md
.Name(n => n.Metadata)
.AutoMap(visitor)
)
.Text(t => t.Name(n => n.CopyTo))
)
)
);
// Use RelationName instead of Name
[ElasticsearchType(RelationName = "documentModel")]
public class DocumentModel
{
[Text(Name = "filename")]
public string Filename { get; set; }
[Text(Name = "folderid")]
public int FolderID { get; set; }
[Text(Name = "uploaduser")]
public string uploadUser { get; set; }
[Date(Format = "MMddyyyy")]
public DateTime UploadDate { get; set; }
[Text(Name = "documenttype")]
public string DocumentType { get; set; }
[Boolean(NullValue = false)]
public bool AccessControl { get; set; }
[Nested]
public List<Comments> Comments { get; set; }
[Nested]
public List<Metadata> Metadata { get; set; }
[Text(Name = "copyTo")]
public string CopyTo { get; set; }
}