I have been using ElasticSearch 6 for a long time and managed to apply a mapping for my most used fields. Recently, I've created another cluster (v7.7), but cannot apply the same mappings as the default has been deprecated. I have some nested objects in my mappings. When I replace default with _doc, I can apply the mapping, however all the incoming requests to ES will be rejected and will go to Logstash dead letter queue. (I'm using logstash and sending my application logs to ElasticSearch. Here is my mapping json:
"index_patterns": [
"logstash-*"
],
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "50000"
}
},
"refresh_interval": "5s"
}
},
"mappings": {
"_default_": {
"_all": {
"norms": false
},
"properties": {
"@timestamp": {
"type": "date"
},
"name": {
"ignore_above": 64,
"type": "keyword"
},
"logger": {
"ignore_above": 64,
"type": "keyword"
},
"hostname": {
"ignore_above": 256,
"type": "keyword"
},
"pid": {
"type": "integer"
},
"level": {
"type": "short"
},
"v": {
"type": "short"
},
"env": {
"ignore_above": 64,
"type": "keyword"
},
"service": {
"ignore_above": 256,
"type": "keyword"
},
"message": {
"type": "text"
},
"module": {
"type": "keyword"
},
"subModule": {
"type": "keyword"
},
"docker": {
"properties": {
"container_id": {
"ignore_above": 512,
"type": "keyword"
},
"container_name": {
"ignore_above": 512,
"type": "keyword"
},
"image_id": {
"ignore_above": 512,
"type": "keyword"
},
"image_name": {
"ignore_above": 512,
"type": "keyword"
},
"command": {
"ignore_above": 512,
"type": "keyword"
},
"tag": {
"ignore_above": 512,
"type": "keyword"
},
"created": {
"type": "date"
}
}
},
"data": {
"type": "text"
},
"context": {
"properties": {
"trade": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"trader": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"email": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"payment": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
}
}
},
"err": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
},
"cause": {
"properties": {
"message": {
"type": "text"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"stack": {
"type": "text"
},
"code": {
"ignore_above": 256,
"type": "keyword"
},
"file": {
"ignore_above": 256,
"type": "keyword"
},
"line": {
"type": "integer"
}
}
}
}
}
}
},
"req": {
"properties": {
"id": {
"ignore_above": 64,
"type": "keyword"
},
"remoteAddress": {
"type": "ip"
},
"remotePort": {
"type": "integer"
},
"method": {
"ignore_above": 32,
"type": "keyword"
},
"path": {
"type": "text"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
}
}
},
"res": {
"properties": {
"statusCode": {
"type": "short"
},
"body": {
"type": "text"
},
"bodyLength": {
"type": "integer"
},
"headers": {
"type": "text"
},
"latency": {
"type": "integer"
}
}
},
"event": {
"properties": {
"eventName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
}
}
},
"task": {
"properties": {
"taskName": {
"ignore_above": 256,
"type": "keyword"
},
"context": {
"type": "text"
},
"uuid": {
"ignore_above": 64,
"type": "keyword"
},
"time": {
"type": "long"
},
"attempts": {
"type": "short"
},
"origin": {
"ignore_above": 128,
"type": "keyword"
}
}
}
}
},
"dlq": {
"properties": {
"eventFields": {
"type": "text"
},
"reason": {
"type": "text"
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
I'm wondering how can I apply this to elasticSearch7?
Here is the ElasticSearch error:
[o.e.a.a.i.m.p.TransportPutMappingAction] [0c85334cb2d41154383d0174502e13f2] failed to put mappings on indices [[[__PATH__]]], type [logs]
java.lang.IllegalArgumentException: Rejecting mapping update to [logstash-2020.10.02] as the final mapping would have more than 1 type: [_doc, logs]
I found the answer!
Logstash sends the logs to ElasticSearch with type logs. Replacing the default with logs fixed the issue. However, for applying the mappings in version7, you need to use PUT /_template/template_1?include_type_name=true