Search code examples
jsonlogstashlogstash-grok

Logstash send only json

iam sendding this log via logstash 

2017-02-27T13:00:07+01:00    test    {"createdAt":"2017-02-27T13:00:07+0100","cluster":"undefined","nodeName":"undefined","nodeIP":"10.11.11.50","clientIP":"10.11.11.72","customerId":1,"identityId":332,"appType":"admin","eventGroup":"education","eventName":"insert","eventData":{"education_insert":{"type":"course","data":{"education_id":2055,"education":{"id":2055,"customer_id":1,"creator_id":332,"type":"course","status":"new","is_featured":false,"enroll_deadline":null,"complete_deadline":null,"count_view":0,"count_like":0,"meta_title":"test Course - progress","meta_description":"test Course - progress","discoverable":"everyone","progress_max":0,"instructor_ids":[332],"tag_ids":[135],"discoverable_group_ids":[],"category_ids":[14],"audits":null,"instructors":null,"creator":null,"lessonGroups":null,"categories":null},"duration":"quick"}}},"scopeType":"education","scopeId":"2055"}

how can i remove 2017-02-27T13:00:07+01:00 and test.app.event

Solution

  • i used this and worked for me :) thx for help

    input { kafka { bootstrap_servers=>"localhost:9092" topics=>"test"}}
filter{
grok {
    match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\t%{GREEDYDATA:topic}\t%{GREEDYDATA:json}" }
  }
  json {
     source => "json"
     remove_field => ["timestamp","topic","json","message","@version","@timestamp","tags"]
  }
 }
output{ elasticsearch {hosts=>["127.0.0.1:9200"] document_type=>"app_stats" index=>"test"}}