How to get termvectors when using ingest plugin in ElasticSearch 5.5?

Question

All,

I have the following code to index a file using ingest plugin in elasticsearch

public class Document
{
  public string Id { get; set; }
  public string Content { get; set; }
  public Attachment Attachment { get; set; }
}

var indexResponse = client.CreateIndex("documents", c => c
  .Settings(s => s
    .Analysis(a => a
     .TokenFilters(f=>f.Stemmer("english_stem",st=>st.Language("english")).Stop("english_stop",sp=>sp.StopWords("_english_")))
     .CharFilters(cf => cf.PatternReplace("num_filter", nf => nf.Pattern("(\\d+)").Replacement(" ")))                
     .Analyzers(an => an.Custom("tm_analyzer", ta => ta.CharFilters("num_filter").Tokenizer("standard").Filters("english_stem","english_stop","lowercase")))))
  .Mappings(m => m
             .Map<Document>(mm => mm
              .AllField(al=>al.Enabled(false))
              .Properties(p => p                 
               .Object<Attachment>(o=>o
                .Name(n=>n.Attachment)
                .Properties(ps=>ps
                 .Text(s => s
                  .Name(nm => nm.Content)
                  .TermVector(TermVectorOption.Yes)
                  .Store(true)
                  .Analyzer("tm_analyzer")))))));

client.PutPipeline("attachments", p => p
  .Description("Document attachment pipeline")
  .Processors(pr => pr
    .Attachment<Document>(a => a
      .Field(f => f.Content)
      .TargetField(f => f.Attachment)
    )
    .Remove<Document>(r => r
      .Field(f => f.Content)
    )
  )
);

var base64File = Convert.ToBase64String(File.ReadAllBytes("file1.xml"));
client.Index(new Document
{
  Id = "file1.xml", 
  Content = base64File
}, i => i.Pipeline("attachments"));

As you can see i have set the termvector otpion to yes on the Content field. But when i query like below using postman or in C# Nest i get nothing

POST /documents/document/_mtermvectors
{
    "ids" : ["1.xml"],
    "parameters": {
        "fields": [
                "content"
        ],
        "term_statistics": true
    }
}

Any ideas what I am doing wrong? Thanks for the help!

Answer 1

You're removing the content field in the ingest processor here

.Remove<Document>(r => r
  .Field(f => f.Content)
)

This is probably what you want because it'll contain the base64 encoded attachment. I think your API call should be looking at the attachment.content field, which will contain the extracted content from the attachment

POST /documents/document/_mtermvectors
{
    "ids" : ["1.xml"],
    "parameters": {
        "fields": [
            "attachment.content"
        ],
        "term_statistics": true
    }
}