ELK采集之nginx 之高德地图出城市IP分布图

2、 为了方便咱们定义一个 nginx 的的 patten 文件，放在默认目录下： [root@controller logstash-5.0.0]# cd vendor/bundle/jruby/1.9/gems/logstash-patt erns-core-4.0.2/patterns
NGUSERNAME [a-zA-Z\.\@\-\+_%]+ NGUSER %{NGUSERNAME} NGINXACCESS %{IPORHOST:clientip} - %{NGUSER:remote_user} \[%{HTTPDATE:timestam p}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DA TA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS: agent} %{NOTSPACE:http_x_forwarded_for}
"bytes" => "571", "clientip" => "123.67.69.249", "@version" => "1",
"host" => "controller", "http_x_forwarded_for" => "\"-\"",
"httpversion" => "1.1",
"timestamp" => "04/Jan/2017:14:21:54 +0800" }
"timezone" => "Asia/Shanghai", "ip" => "123.67.69.249",
"latitude" => 39.9289, "coordinates" => [ [0] 116.3883, [1] 39.9289 ], "continent_code" => "AS",
Indexer 配置： [root@mail etc]# cat logstash_nginx.conf input {
redis { host => "192.168.10.100" port => 6379 data_type => "list" key => "nginx"
} }
filter { grok { match => {"message" => "%{IPORHOST:clientip} \[%{HTTPDATE:timestam
} } output { if [type] == "nginx_access"{
redis { host => ["192.168.10.100:6379"] data_type =>"list" key => "nginx" }
} } ##说明:这里的 agent 只是做日志发送，对性能影响不大，读取 access.log 日志文件，并且 发送到远端 redis。
Agent 配置home/luohui/logstash-5.0.0/etc/logstashnginx.conf input {
file { path => ["/home/data/logs/access.log"] type => "nginx_access"
[0] 116.3883, [1] 39.9289 ], "postal_code" => nil,
"longitude" => 116.3883, "region_code" => "11" },
"verb" => "GET", "message" => "123.67.69.249 - - [04/Jan/2017:14:21:54 +0800] \ "GET /favicon.ico HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW6 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\" \"-\"", "remote_user" => "-", "referrer" => "\"-\"", "@timestamp" => 2017-01-04T06:21:54.000Z, "response" => "404",
说明：由于可能我线上环境跟大家的日志生成格式不一样，这里主要列举大概配置，后续添 加统一配置：
log_format backend '$http_x_forwarded_for [$time_local] ' '"$host" "$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent"'
####测试返回的结果： {
"request" => "/favicon.ico", "agent" => "\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKi
t/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\"", "geoip" => {
4、 假如说我 nginx 日志在加上一个 nginx 响应时间呢？修改格式加上”request_time”： 4.1：修改日志结构生成数据：
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for" $request_time
} } date { match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ] } geoip { source => "clientip" target => "geoip" database => "/test/logstash-5.0.0/GeoLite2-City.mmdb" add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] } mutate {
3、 先用 input 和 output 插件测试： [root@controller etc]# cat nginx.conf input{
stdin{} }
filter { grok { match => { "message" => "%{NGINXACCESS}" } } date { match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ] } geoip { source => "clientip" target => "geoip" database => "/test/logstash-5.0.0/GeoLite2-City.mmdb" add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] } mutate { convert => [ "[geoip][coordinates]", "float"] }
convert => [ "[geoip][coordinates]", "float"] }
} output {
elasticsearch { action => "index" hosts =>"192.168.10.100:9200" index => "logstash-nginx-%{+yyyy.MM.dd}" } }