init => "@kname = ['host-name','front','http_x_up_calling_line_id','request','http_user_agent','status','remote_addr_1','id','http_referer','request_time','body_bytes_sent','http_deviceid','http_x_forwarded_for','domain','cookie']"
code => "event.append(Hash[@kname.zip(event['message'].split('`'))]) "
remove_field => ["@version","_score","id","tags","key","message","http_deviceid","http_x_up_calling_line_id","","cookie"]
}
if [front] {
grok {
match => ["front","%{HTTPDATE:logdate}"]
}
date {
match => ["logdate", "dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
remove_field => ["front","logdate"]
}
}
if [request] {
ruby {
init => "@kname = ['method','uri','verb']"
code => "event.append(Hash[@kname.zip(event['request'].split(' '))])"
remove_field => [ "request","method","verb"]
}
}
if [remote_addr_1] {
grok {
match => [ "remote_addr_1", "%{IPV4:remote_addr}" ]
remove_field => ["remote_addr_1"]
}
}
mutate {
convert => [
"body_bytes_sent" , "integer",
"status" , "integer",
"request_time" , "float" ]
}
}
output {
elasticsearch {
hosts => ["10.39.40.94:9200","10.39.40.95:9200","10.39.40.96:9200","10.39.40.97:9200"]
workers => 1
index => "logstash-cms-nginx-%{+YYYY.MM.dd.hh}"
}
#stdout { codec => dots
#workers => 5
#} #测试性能时使用
#stdout { codec => rubydebug } #调试时使用
}
启动命令：./bin/logstash -w 4 -b 1000 -f /etc/logstash/conf.d/kafka_logstash_cms_nginx.conf &
-w 后面的worker数是根据cpu的核心数大概算一下，我这里一台服务器开三个logstash，每个起4个worker