问题描述
我已经使用ElasticSearch 6很长时间了，并且设法为我最常用的字段应用了映射。最近，我创建了另一个群集（v7.7），但是不能使用与已弃用 default 相同的映射。我的映射中有一些嵌套对象。当我用_doc替换 default 时，我可以应用映射，但是对ES的所有传入请求都将被拒绝，并将进入Logstash死信队列。 （我正在使用logstash并将应用程序日志发送到ElasticSearch。这是我的映射json：
"index_patterns": [
"logstash-*"
],"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "50000"
}
},"refresh_interval": "5s"
}
},"mappings": {
"_default_": {
"_all": {
"norms": false
},"properties": {
"@timestamp": {
"type": "date"
},"name": {
"ignore_above": 64,"type": "keyword"
},"logger": {
"ignore_above": 64,"hostname": {
"ignore_above": 256,"pid": {
"type": "integer"
},"level": {
"type": "short"
},"v": {
"type": "short"
},"env": {
"ignore_above": 64,"service": {
"ignore_above": 256,"message": {
"type": "text"
},"module": {
"type": "keyword"
},"subModule": {
"type": "keyword"
},"docker": {
"properties": {
"container_id": {
"ignore_above": 512,"type": "keyword"
},"container_name": {
"ignore_above": 512,"image_id": {
"ignore_above": 512,"image_name": {
"ignore_above": 512,"command": {
"ignore_above": 512,"tag": {
"ignore_above": 512,"created": {
"type": "date"
}
}
},"data": {
"type": "text"
},"context": {
"properties": {
"Trade": {
"properties": {
"id": {
"ignore_above": 64,"type": "keyword"
}
}
},"Trader": {
"properties": {
"id": {
"ignore_above": 64,"type": "keyword"
},"email": {
"ignore_above": 1024,"payment": {
"properties": {
"id": {
"ignore_above": 64,"err": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
},"cause": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"type": "keyword"
},"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
},"cause": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"type": "keyword"
},"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
}
}
}
}
}
}
},"req": {
"properties": {
"id": {
"ignore_above": 64,"remoteAddress": {
"type": "ip"
},"remotePort": {
"type": "integer"
},"method": {
"ignore_above": 32,"path": {
"type": "text"
},"body": {
"type": "text"
},"bodyLength": {
"type": "integer"
},"headers": {
"type": "text"
}
}
},"res": {
"properties": {
"statusCode": {
"type": "short"
},"headers": {
"type": "text"
},"latency": {
"type": "integer"
}
}
},"event": {
"properties": {
"eventName": {
"ignore_above": 256,"context": {
"type": "text"
},"uuid": {
"ignore_above": 64,"time": {
"type": "long"
},"attempts": {
"type": "short"
}
}
},"task": {
"properties": {
"taskName": {
"ignore_above": 256,"attempts": {
"type": "short"
},"origin": {
"ignore_above": 128,"type": "keyword"
}
}
}
}
},"dlq": {
"properties": {
"eventFields": {
"type": "text"
},"reason": {
"type": "text"
}
}
},"tags": {
"ignore_above": 1024,"type": {
"ignore_above": 1024,"type": "keyword"
}
}
}
}
}
我想知道如何将其应用于elasticSearch7？
这是ElasticSearch错误：
[o.e.a.a.i.m.p.TransportPutMappingAction] [0c85334cb2d41154383d0174502e13f2] Failed to put mappings on indices [[[__PATH__]]],type [logs]
java.lang.IllegalArgumentException: Rejecting mapping update to [logstash-2020.10.02] as the final mapping would have more than 1 type: [_doc,logs]
解决方法
@YLR给出的答案几乎是正确的，但是在他提供的映射中存在一些解析错误。
_all field在6.0中已弃用
_all可能不再为在6.0+版本中创建的索引启用，请使用自定义字段和映射copy_to参数
请参阅此Elasticsearch documentation，以了解更多有关此的信息。
在Elasticsearch 7.0.0或更高版本中创建的索引不再接受 默认映射。在6.x中创建的索引将继续像在Elasticsearch 6.x中一样运行。在7.0的API中不推荐使用类型， 破坏索引创建的更改，放置映射，获取映射，放置 模板，获取模板并获取字段映射API。
修改后的映射为：
{
"index_patterns": [
"logstash-*"
],"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "50000"
}
},"refresh_interval": "5s"
}
},"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},"name": {
"ignore_above": 64,"type": "keyword"
},"logger": {
"ignore_above": 64,"hostname": {
"ignore_above": 256,"pid": {
"type": "integer"
},"level": {
"type": "short"
},"v": {
"type": "short"
},"env": {
"ignore_above": 64,"service": {
"ignore_above": 256,"message": {
"type": "text"
},"module": {
"type": "keyword"
},"subModule": {
"type": "keyword"
},"docker": {
"properties": {
"container_id": {
"ignore_above": 512,"type": "keyword"
},"container_name": {
"ignore_above": 512,"image_id": {
"ignore_above": 512,"image_name": {
"ignore_above": 512,"command": {
"ignore_above": 512,"tag": {
"ignore_above": 512,"created": {
"type": "date"
}
}
},"data": {
"type": "text"
},"context": {
"properties": {
"trade": {
"properties": {
"id": {
"ignore_above": 64,"type": "keyword"
}
}
},"trader": {
"properties": {
"id": {
"ignore_above": 64,"type": "keyword"
},"email": {
"ignore_above": 1024,"payment": {
"properties": {
"id": {
"ignore_above": 64,"err": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
},"cause": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"type": "keyword"
},"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
},"cause": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"type": "keyword"
},"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
}
}
}
}
}
}
},"req": {
"properties": {
"id": {
"ignore_above": 64,"remoteAddress": {
"type": "ip"
},"remotePort": {
"type": "integer"
},"method": {
"ignore_above": 32,"path": {
"type": "text"
},"body": {
"type": "text"
},"bodyLength": {
"type": "integer"
},"headers": {
"type": "text"
}
}
},"res": {
"properties": {
"statusCode": {
"type": "short"
},"headers": {
"type": "text"
},"latency": {
"type": "integer"
}
}
},"event": {
"properties": {
"eventName": {
"ignore_above": 256,"context": {
"type": "text"
},"uuid": {
"ignore_above": 64,"time": {
"type": "long"
},"attempts": {
"type": "short"
}
}
},"task": {
"properties": {
"taskName": {
"ignore_above": 256,"attempts": {
"type": "short"
},"origin": {
"ignore_above": 128,"type": "keyword"
}
}
}
}
},"dlq": {
"properties": {
"eventFields": {
"type": "text"
},"reason": {
"type": "text"
}
}
},"tags": {
"ignore_above": 1024,"type": {
"ignore_above": 1024,"type": "keyword"
}
}
}
}
我找到了答案！
Logstash使用 logs 类型将日志发送到ElasticSearch。将 default 替换为日志可解决此问题。但是，要在版本7中应用映射，您需要使用PUT /_template/template_1?include_type_name=true
您必须更新v7中不受支持的某些字段。
此请求是一个示例，其中包含“不支持删除/更新”字段：
PUT _index_template/template_1
{
"index_patterns": [
"logstash-*"
],"template": {
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "50000"
}
},"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},"name": {
"ignore_above": 64,"type": "keyword"
},"logger": {
"ignore_above": 64,"hostname": {
"ignore_above": 256,"pid": {
"type": "integer"
},"level": {
"type": "short"
},"v": {
"type": "short"
},"env": {
"ignore_above": 64,"service": {
"ignore_above": 256,"message": {
"type": "text"
},"module": {
"type": "keyword"
},"subModule": {
"type": "keyword"
},"docker": {
"properties": {
"container_id": {
"ignore_above": 512,"type": "keyword"
},"container_name": {
"ignore_above": 512,"image_id": {
"ignore_above": 512,"image_name": {
"ignore_above": 512,"command": {
"ignore_above": 512,"tag": {
"ignore_above": 512,"created": {
"type": "date"
}
}
},"data": {
"type": "text"
},"context": {
"properties": {
"trade": {
"properties": {
"id": {
"ignore_above": 64,"type": "keyword"
}
}
},"trader": {
"properties": {
"id": {
"ignore_above": 64,"type": "keyword"
},"email": {
"ignore_above": 1024,"payment": {
"properties": {
"id": {
"ignore_above": 64,"err": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
},"cause": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"type": "keyword"
},"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
},"cause": {
"properties": {
"message": {
"type": "text"
},"name": {
"ignore_above": 256,"type": "keyword"
},"stack": {
"type": "text"
},"code": {
"ignore_above": 256,"file": {
"ignore_above": 256,"line": {
"type": "integer"
}
}
}
}
}
}
},"req": {
"properties": {
"id": {
"ignore_above": 64,"remoteAddress": {
"type": "ip"
},"remotePort": {
"type": "integer"
},"method": {
"ignore_above": 32,"path": {
"type": "text"
},"body": {
"type": "text"
},"bodyLength": {
"type": "integer"
},"headers": {
"type": "text"
}
}
},"res": {
"properties": {
"statusCode": {
"type": "short"
},"headers": {
"type": "text"
},"latency": {
"type": "integer"
}
}
},"event": {
"properties": {
"eventName": {
"ignore_above": 256,"context": {
"type": "text"
},"uuid": {
"ignore_above": 64,"time": {
"type": "long"
},"attempts": {
"type": "short"
}
}
},"task": {
"properties": {
"taskName": {
"ignore_above": 256,"attempts": {
"type": "short"
},"origin": {
"ignore_above": 128,"type": "keyword"
}
}
}
}
},"dlq": {
"properties": {
"eventFields": {
"type": "text"
},"reason": {
"type": "text"
}
}
},"tags": {
"ignore_above": 1024,"type": {
"ignore_above": 1024,"type": "keyword"
}
}
}
}
}
}