How to get values of my system parameters of my metricbeat file?

Question

I am newbie with ElasticStack and want to get the value of system parameters like Cpu/Memory/diskio etc from Metricbeat in my .NET core project.

I want to create a page similar to kibana dashboard. Therefore, I need to fetch them through query/API from metricbeat file.

I have write the following code

            ConnectionSettings connectionSettings;
            ElasticClient elasticClient;
            StaticConnectionPool connectionPool;
            var nodes = new Uri[]
            {
                new Uri(_options.Value.ElasticSearchUrl),
            };
            connectionPool = new StaticConnectionPool(nodes);
            string indexName = "metricbeat*";
            connectionSettings = new ConnectionSettings(connectionPool).DefaultIndex(indexName);
            elasticClient = new ElasticClient(connectionSettings);


            var elasticResponse = await elasticClient.SearchAsync<object>(s => s.Size(1).
Query(q => q.Bool(b => b.Must(m => m.Range(r => r.Field("system.memory.actual.used.pct").GreaterThan(0).LessThan(1))))));

But the elasticResponse give me full hit details as below

{
  "took" : 35,
  "timed_out" : false,
  "_shards" : {
    "total" : 15,
    "successful" : 15,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [
      {
        "_index" : "metricbeat-7.4.2-2019.12.18",
        "_type" : "_doc",
        "_id" : "hIK-GG8BkfAhjYcrdBP2",
        "_score" : null,
        "_source" : {
          "@timestamp" : "2019-12-18T11:22:11.595Z",
          "host" : {
            "name" : "a8a441269011"
          },
          "agent" : {
            "id" : "7e4f3202-461e-4f8d-8144-d2db5556ca1b",
            "version" : "7.4.2",
            "type" : "metricbeat",
            "ephemeral_id" : "348fb071-e058-411c-b781-8dd2ec445286",
            "hostname" : "a8a441269011"
          },
          "event" : {
            "dataset" : "system.memory",
            "module" : "system",
            "duration" : 389771
          },
          "metricset" : {
            "name" : "memory",
            "period" : 5000
          },
          "service" : {
            "type" : "system"
          },
          "system" : {
            "memory" : {
              "total" : 33731682304,
              "used" : {
                "pct" : 0.762,
                "bytes" : 25702785024
              },
              "free" : 8028897280,
              "actual" : {
                "free" : 23516848128,
                "used" : {
                  "pct" : 0.3028,
                  "bytes" : 10214834176
                }
              },
              "swap" : {
                "total" : 1023406080,
                "used" : {
                  "pct" : 0.0064,
                  "bytes" : 6553600
                },
                "free" : 1016852480
              },
              "hugepages" : {
                "free" : 0,
                "reserved" : 0,
                "surplus" : 0,
                "default_size" : 2097152,
                "total" : 0,
                "used" : {
                  "bytes" : 0,
                  "pct" : 0
                }
              }
            }
          },
          "ecs" : {
            "version" : "1.1.0"
          }
        },
        "sort" : [
          1576668131595
        ]
      }
    ]
  }
}

But I want just

              "actual" : {
                "free" : 23516848128,
                "used" : {
                  "pct" : 0.3028,
                  "bytes" : 10214834176
                }
              }

Please help.

Solution:

var elasticResponse = elasticClient.Search<object>(s => s .DocValueFields(dvf => dvf.Fields("system.memory.*", "system.cpu.*")) .Size(2) );

Answer 1

You can use source filtering to tell elasticsearch in which fields you are interested in. With NEST your search request will look like

var elasticResponse = await client.SearchAsync<object>(s => s
    .Source(sf => sf.Includes(i => i.Fields("system.memory.actual.*")))
    .Size(1)
    .Query(q => q.Bool(b => b.Must(m =>
        m.Range(r => r.Field("system.memory.actual.used.pct").GreaterThan(0).LessThan(1))))));

UPDATE:

In your case, you can also use docvalue_fields to return a list of fields in a flat structure fashion.

var elasticResponse = await client.SearchAsync<object>(s => s
    .DocValueFields("system.memory.actual.*") 
    .Size(1)
    .Query(q => q.Bool(b => b.Must(m =>
        m.Range(r => r.Field("system.memory.actual.used.pct").GreaterThan(0).LessThan(1))))));

foreach (var fields in elasticResponse.Fields)
{
    foreach (var value in fields)
    {
        Console.WriteLine($"{value.Key}: {fields.Value<object>(value.Key)}");
    }
}

output

system.memory.actual.free: 23516848128
system.memory.actual.used.bytes: 10214834176
system.memory.actual.used.pct: 0.302799999713898

Hope that helps.