Logstash nginx 로그 분석 기록

3913 단어
nginx 로그 설정
  • conf 디 렉 터 리 에 들 어가 ngix. conf 파일 편집
  • cd /usr/conf/
    vim nginx.conf
    
  • nginx 로그 형식 설정
  • log_format  main  "$remote_addr | $remote_user | $time_local | $request | $status | $body_bytes_sent | $http_referer | $http_user_agent | $http_x_forwarded_for ";
    access_log  logs/access.log  main;
    

    logstash 설정
  • logstash 폴 더 에 들 어가 서 새 프로필 test. conf
  • cd /opt/logstash
    vim test.conf
    
    input{
            file{
                    path => "/usr/logs/access.log"
                    start_position => "beginning"
                    codec => json
            }
    }
    filter {
        ruby {
            init => "@kname = ['remote_addr','remote_user','time_local','request','status','body_bytes_sent','http_referer','http_user_agent','http_x_forwarded_for']"
            code => "event.append(Hash[@kname.zip(event['message'].split(' | '))])"
        }
        if [request] {
            ruby {
                init => "@kname = ['method','uri','verb']"
                code => "event.append(Hash[@kname.zip(event['request'].split(' '))])"
            }
            if [uri] {
                ruby {
                    init => "@kname = ['url_path','url_args']"
                    code => "event.append(Hash[@kname.zip(event['request'].split('?'))])"
                }
                kv {
                    prefix => "url_"
                    source => "url_args"
                    field_split => "& "
                    remove_field => [ "url_args","uri","request" ]
                }
            }
        }
        mutate {
            convert => [
                "body_bytes_sent" , "integer"
            ]
        }
        date {
            match => [ "time_local", "dd/MMM/yyyy:hh:mm:ss Z" ]
            locale => "en"
        }
    }
    
    output{
            stdout{
                    codec => rubydebug
            }
            file{
                    path => "/usr/logs/datas.log"
                    codec => json
            }
            //      file , mongodb...
    }
            
    
  • logstash 시작
  • //  conf    
    bin/logstash -f test.conf --configtest
    //  
    bin/logstash -f test.conf
    //    
    {
                     "message" => "127.0.0.1 | - | 11/Nov/2015:13:14:53 +0800 | GET
    /file/test.img?width=800&height=600 HTTP/1.1 | 404 | 570 | - | Mozilla/5.0 (Wind
    ows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Sa
    fari/537.36 | - \r",
                    "@version" => "1",
                  "@timestamp" => "2015-11-11T05:14:53.000Z",
                        "host" => "Jevirs-PC",
                        "path" => "D:\
    ginx\
    ginx\\logs\\access_test.log", "remote_addr" => "127.0.0.1", "remote_user" => "-", "time_local" => "11/Nov/2015:13:14:53 +0800", "status" => "404", "body_bytes_sent" => 570, "http_referer" => "-", "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/5 37.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36", "http_x_forwarded_for" => "- \r", "method" => "GET", "verb" => "HTTP/1.1", "url_path" => "GET /file/test.img", "url_width" => "800", "url_height" => "600" }

    좋은 웹페이지 즐겨찾기