C# Nest & Elasticsearch 6.x : How to filter/count nested fields (in inner List<object>)

Question

Here is my mapping

[ElasticsearchType(Name = "Topic")]
public class Topic
{
    [Number(NumberType.Integer, Coerce = true)]
    public EnumStatus Status { get; set; }

    [Nested]
    public List<KeywordValue> KeywordValues { get; set; }

}

[ElasticsearchType(Name = "KeywordValue")]
public class KeywordValue
{
    [Keyword]
    public string KeywordId { get; set; }

}

I have 10 documents of type Topic in the index, each KeywordValues property/field of the type List<KeywordValue> contains 5 KeywordValue (5 elements in the list).

9 documents have status "Enabled";

I'm trying to count the total number of elements in each nested KeywordValues field. The result returned is 9 but I would like to get 45 (9*5)

I'm doing this:

var response = Topic.CurrentConnection.Search<Topic>(s => s
                .Size(0)
                .Aggregations(fa => fa
                    .Filter("filtered_aggs", f => f
                        .Filter(fd => fd.Term(t => t.Status, Topic.EnumStatus.Enabled))
                        .Aggregations(ta => ta
                                .Nested("kv", n=>n.Path(p => p.KeywordValues)
                                    .Aggregations(aa => aa
                                        .ValueCount("vc", v => v.Field(vf => vf.KeywordValues.First().KeywordId))))
                            )
                        )
                    )
            );


        if (response.IsValid)
        {
            var agg = response.Aggregations.Nested("filtered_aggs");
            var n = agg.Nested("kv");
            var z = n.ValueCount("vc");
            result.Object = z.Value;
        }

Raw query equivalent:

# Request:
{
  "size": 0,
  "aggs": {
    "filtered_aggs": {
      "filter": {
        "term": {
          "Status": {
            "value": 0
          }
        }
      },
      "aggs": {
        "kv": {
          "nested": {
            "path": "KeywordValues"
          },
          "aggs": {
            "vc": {
              "value_count": {
                "field": "KeywordValues.KeywordId"
              }
            }
          }
        }
      }
    }
  }
}
# Response:
{
  "took" : 80,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 10,
    "max_score" : 0.0,
    "hits" : [ ]
  },
  "aggregations" : {
    "filter#filtered_aggs" : {
      "doc_count" : 9,
      "nested#kv" : {
        "doc_count" : 9,
        "value_count#vc" : {
          "value" : 9
        }
      }
    }
  }
}

Any idea? Thanks a lot.

Answer 1

Here's a working example

private static void Main()
{
    var defaultIndex = "topics";
    var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));

    var settings = new ConnectionSettings(pool)
        .DefaultIndex(defaultIndex);

    var client = new ElasticClient(settings);

    if (client.IndexExists(defaultIndex).Exists)
        client.DeleteIndex(defaultIndex);

    client.CreateIndex(defaultIndex, c => c
        .Mappings(m => m
            .Map<Topic>(mm => mm
                .AutoMap()
            )
        )
    );

    var documents = Enumerable.Range(1, 10)
        .Select(i => new Topic
        {
            Status = i == 1 ? EnumStatus.Disabled : EnumStatus.Enabled,
            KeywordValues = Enumerable.Range(1, 5)
                .Select(j => new KeywordValue
                {
                    KeywordId = $"keyword {i} {j}"
                }).ToList()
        });

    client.Bulk(b => b
        .IndexMany(documents, (d, document) => d
            .Document(document)
        )
        .Refresh(Refresh.WaitFor)
    );

    client.Search<Topic>(s => s
        .Size(0)
        .Query(q => +q
            .Term(t => t.Status, (int)EnumStatus.Enabled)
        )
        .Aggregations(ta => ta
            .Nested("kv", n => n.Path(p => p.KeywordValues)
                .Aggregations(aa => aa
                    .ValueCount("vc", v => v.Field(vf => vf.KeywordValues.First().KeywordId))))
        )
    );
}

[ElasticsearchType(Name = "Topic")]
public class Topic
{
    [Number(NumberType.Integer, Coerce = true)]
    public EnumStatus Status { get; set; }

    [Nested]
    public List<KeywordValue> KeywordValues { get; set; }

}

[ElasticsearchType(Name = "KeywordValue")]
public class KeywordValue
{
    [Keyword]
    public string KeywordId { get; set; }
}

public enum EnumStatus
{
    Enabled,

    Disabled
}

The response to the search request is

{
  "took" : 9,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : 9,
    "max_score" : 0.0,
    "hits" : [ ]
  },
  "aggregations" : {
    "nested#kv" : {
      "doc_count" : 45,
      "value_count#vc" : {
        "value" : 45
      }
    }
  }
}