Search code examples
log4jlogstash-grok

What's wrong with my logstash filter grok syntax?


I'm trying to use OpenSearch over Logstash logs. I have two logs, but the second log does not apply to OpenSearch available fields.

Here is my two logs and each has a different log4j pattern. It is a third-party solution, so these logs pattern can't change.

Log1
%d %5p [%c] [%X{txIp}] [%X{mbrNo}] %m%n

Log2
[%-5p][%d{yyyyMMdd HH:mm:ss.SSS}][%t]%c{4}.%M(%L) - %X{reqCommand}|%X{svcTrId}|%m%n

and this is my logstash.conf filter:

filter {
  if [fields][index] == "log1"{
     grok {
       match => {
         "message" => "%{TIMESTAMP_ISO8601:date} %{LOGLEVEL:logLevel} \[%{DATA:class}\] \[%{DATA:txIp}\] \[%{DATA:mbrNo}\] %{GREEDYDATA:message}"
       }
     }
  } else if [fields][index] == "log2" {
     grok {
       match => {
         "message" => "\[%{LOGLEVEL:logLevel}\]\[%{TIMESTAMP_ISO8601:date}\]\[%{DATA:thread}\]%{DATA:class}.%{DATA:method}\(%{DATA:line}\) - %{DATA:reqCommand}\|%{DATA:svcTrId}\|%{GREEDYDATA:message}"
       }
     }
  }


  date {
    match => ["date", "ISO8601"]
  }
}

Solution

  • Finally It works.

    This is my Filter.

    filter {
      if [fields][index] == "log1" {
         grok {
           match => {
             "message" => "%{TIMESTAMP_ISO8601:logTimestamp} %{DATA:logLevel} \[%{DATA:class}\] \[%{DATA:txIp}\] \[%{DATA:mbrNo}\] %{GREEDYDATA:message}"
           }
         }
         date {
           match => ["logTimestamp", "ISO8601"]
           target => "@timestamp"
           timezone => "Asia/Seoul"
         }
      } else if [fields][index] == "log2" {
         grok {
           match => {
             "message" => "\[%{DATA:logLevel}]\[%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day} %{TIME:time}\]\[%{DATA:thread}\]%{DATA:class}.%{DATA:method}\(%{DATA:line}\) - %{DATA:reqCommand}\|%{DATA:svcTrId}\|%{GREEDYDATA:message}"
           }
         }
         mutate {
           add_field => {
             "logtimestamp" => "%{year}-%{month}-%{day}T%{time}"
           }
         }
         date {
           match => ["logtimestamp", "ISO8601"]
           target =>  "@timestamp"
           timezone => "Asia/Seoul"
         }
    
       }
    }