Search code examples
nginxlogstashelastic-stacklogstash-grok

Logstash 'Cannot load an invalid configuration'


I am trying to configure logstash for the following inputs - nginx access, nginx error & logs following a custom pattern & below is the logstash.conf

input {
  beats {
    port => 5044

codec => multiline {
      # Grok pattern names are valid! :)
      pattern => "^%{TIMESTAMP_ISO8601} "
      negate => true
      what => previous
    }
    }
}

filter {

if [type] == "nginx-access" {
        grok {
        match => [ "message" , "%{COMBINEDAPACHELOG}+%{GREEDYDATA:extra_fields}"]
        overwrite => [ "message" ]
        }

        mutate {
                convert => ["response", "integer"]
                convert => ["bytes", "integer"]
                convert => ["responsetime", "float"]
        }

        geoip {
                source => "clientip"
                target => "geoip"
                add_tag => [ "nginx-geoip" ]
        }

        date {
                match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
                remove_field => [ "timestamp" ]
        }

        useragent { 
                source => "agent"
        }

}  else if [type] == "nginx-error" { 
        grok {
        match => [ "message" , "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<client>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server})(?:, request: %{QS:request})?(?:, upstream: \"%{URI:upstream}\")?(?:, host: %{QS:host})?(?:, referrer: \"%{URI:referrer}\")"] 
        overwrite => [ "message" ]
        }

geoip {
                source => "client"
                target => "geoip"
                add_tag => [ "nginx-geoip" ]
        }

        date {
                match => [ "timestamp" , "YYYY/MM/dd HH:mm:ss" ]
                remove_field => [ "timestamp" ]
        }

} else {

  mutate {
    gsub => ["message", "\n", " "]
  }
  grok {
    match => [ "message", "%{TIMESTAMP_ISO8601:timestamp} \[%{NOTSPACE:uid}\] \[%{NOTSPACE:thread}\] %{LOGLEVEL:loglevel} %{DATA:class}\-%{GREEDYDATA:message}" ]
    overwrite => [ "message" ]
  }
  date {
    match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss" ]
    target => "@timestamp"
  }
  if "_grokparsefailure" in [tags] {
            drop { }
  }
}
}

output {

stdout { codec => rubydebug }
if [type] == "nginx-access" {
  elasticsearch { hosts => localhost }
  index => "nginx-access-%{+YYYY.MM.dd}"
} else if [type] == "nginx-error" {
  elasticsearch { hosts => localhost }
  index => "nginx-error-%{+YYYY.MM.dd}"
} else {
  elasticsearch { hosts => localhost }
}

}

But on running it, I get the following error

ERROR logstash.agent - Cannot load an invalid configuration {:reason=>"Expected one of #, { 
at line 84, column 9 (byte 1883) after output {\n \nstdout { codec => rubydebug }\nif [type] == 
\"nginx-access\" {\n elasticsearch { hosts => localhost }\n index "}

I am not sure what the error is. Can someone help me figure it out ?

Also I tried running hexdump & it seems fine that there are no garbage characters


Solution

  • The correct answer is having 'index' inside elasticsearch block like below

    output {
    
    stdout { codec => rubydebug }
    if [type] == "nginx-access" {
      elasticsearch { hosts => localhost 
        index => "nginx-access-%{+YYYY.MM.dd}"
      }
    } else if [type] == "nginx-error" {
      elasticsearch { 
          hosts => localhost 
          index => "nginx-error-%{+YYYY.MM.dd}"
      }
    
    } else {
      elasticsearch { 
          hosts => localhost 
      }
    }
    }