filebeat+logstash收集nginx日志

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

    log_format json   '{ "@timestamp": "$time_iso8601", '
                         '"remote_addr": "$remote_addr", '
                         '"remote_user": "$remote_user", '
                         '"body_bytes_sent": "$body_bytes_sent", '
                         '"request_time": "$request_time", '
                         '"status": "$status", '
                         '"request_host": "$host", '
                         '"request_method": "$request_method", '
                         '"request_uri": "$request_uri", '
                         '"uri": "$uri", '
                         '"http_referrer": "$http_referer", '
                         '"body_bytes_sent":"$body_bytes_sent", '
                         '"http_x_forwarded_for": "$http_x_forwarded_for", '
                         '"http_user_agent": "$http_user_agent" '
                    '}';

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

filebeat.inputs:
- type: log
  paths:
  - /usr/local/nginx/logs/*_access.log
  fields_under_root: true
  fields:
    log_type: nginx_access
    env: test
  json:
    keys_under_root: true
    overwrite_keys: true
- type: log
  paths:
  - /usr/local/nginx/logs/*_error.log
  fields_under_root: true
  fields:
    log_type: nginx_error
    env: test

processors:
- add_host_metadata:
    netinfo.enabled: true
- drop_fields:
    fields:
    - input
    - agent
    - ecs
    - beat
    - prospector
    - name
    - host.architecture
    - host.os
    - host.id
    - host.containerized
    - host.mac
    - host.name

output.redis:
  hosts: ["1.1.1.1"]
  datatype: "list"
  db: 0
  key: "nginx_test"

1
2
3
4
5
6
7
8

input {
  redis {
    data_type => "list"
    db => 0
    host => "1.1.1.1"
    key => "nginx_test"
  }
}

1
2
3
4
5
6
7

filter {
  mutate {
    add_field => {
      "handler" => "${HOSTNAME:logstash-01}"
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

output{
    if [log_type] == "blade_java" or [log_type] == "springboot_java" {
        elasticsearch {
          hosts => ["2.2.2.1:9200", "2.2.2.2:9200", "2.2.2.3:9200"]
          index => "%{project}-%{env}-%{app}-%{+yyyy.MM.dd}"
          user => "log_agent"
          password => "loados-log"
        }
    }

    if [log_type] in ["nginx_access", "nginx_error"] {
        elasticsearch {
          hosts => ["2.2.2.1:9200", "2.2.2.2:9200", "2.2.2.3:9200"]
          index => "%{log_type}-%{env}-%{+yyyy.MM.dd}"
          user => "log_agent"
          password => "loados-log"
        }
    }

}