day 22 (ELK 셋째 날)
1.
-
-
-
-
2. filebeat
- nginx json
- filebeate
setup.template.name: "nginx"
setup.template.pattern: "nginx_*"
setup.template.enabled: false
setup.template.overwrite: true
- filebeat json
json.keys_under_root: true
json.overwrite_keys: true
- filebeat input tags
tags["www"]
- filwbeat output tags
- index: "nginx_www_access-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
tags: "www"
3. tomcat
- tomcat ，
- tomcat json ，139
- tomcat json
- filebeat
-- * tomcat
- type: log
4. java
- filebeat java
- 3
- type: log
enabled: true
paths:
- /var/log/elasticsearch/linux58.log
tags: ["java"]
multiline.pattern: '^\['
multiline.negate: true
multiline.match: after
5.kibana
- ， ， ， ，Data Table,markdown
- Dashboard
- ，
-
- es .kibana
2、ELk filebeat modules
filebeat go
- ， java
：
1. modules
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: true
reload.period: 10s
2.
filebeat modules list
3.
[root@db01 ~]# filebeat modules enable nginx
Enabled nginx
4. nginx
5. filebeat
：
1.
2.
2.1 filebeat modules 사용자 정의 색인 과 보기
1. ，
systemctl stop elasticsearch
systemctl stop kibana
rm -rf /data/elasticsearch/*
rm -rf /var/lib/kibana/*
systemctl start elasticsearch
systemctl start kibana
2. nginx
sed -i 's#json#main#g' /etc/nginx/conf.d/bbs.conf
3. nginx
> /var/log/nginx/bbs_access.log
4. nginx
systemctl restart nginx
5. filebeat :
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: true
setup.kibana:
host: "10.0.0.51:5601"
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "nginx_bbs_access-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
source: "/var/log/nginx/bbs_access.log"
- index: "nginx_error-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
fileset.name: "error"
setup.template.name: "nginx"
setup.template.pattern: "nginx_*"
setup.template.enabled: false
setup.template.overwrite: true
6. nginx
filebeat modules enable nginx
7. nginx modules
/usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-geoip-6.6.0.zip /usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-user-agent-6.6.0.zip
8. es
systemctl restart elasticsearch
9.
[root@db01 ~]# egrep -v "#|^$" /etc/filebeat/modules.d/nginx.yml
- module: nginx
access:
enabled: true
var.paths: ["/var/log/nginx/bbs_access.log"]
error:
enabled: true
var.paths: ["/var/log/nginx/error.log"]
10. kibana
cp -a /usr/share/filebeat/kibana /root
cd /usr/share/filebeat/kibana/6/dashboard
find . -type f ! -name "*nginx*"|xargs rm -rf
rm -rf ml-nginx-*
sed -i 's#filebeat\-\*#nginx\_\*#g' Filebeat-nginx-logs.json
sed -i 's#filebeat\-\*#nginx\_\*#g' Filebeat-nginx-overview.json
cd index-pattern/
sed -i 's#filebeat\-\*#nginx\_\*#g' filebeat.json
filebeat setup --dashboards -E setup.dashboards.directory=/root/kibana/
rm -rf /var/lib/kibana/*
systemctl restart kibana
3. docker 설치 절차
docker
rm -fr /etc/yum.repos.d/local.repo
curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.ustc.edu.cn/docker-ce/linux/centos/docker-ce.repo
sed -i 's#download.docker.com#mirrors.tuna.tsinghua.edu.cn/docker-ce#g' /etc/yum.repos.d/docker-ce.repo
yum install docker-ce -y
systemctl start docker
cat > /etc/docker/daemon.json <3.1 docker 용기 로그 수집
1.
systemctl stop nginx
docker stop $(docker ps -q)
docker rm $(docker ps -aq)
docker commit nginx nginx:v2
docker run --name nginx -p 80:80 -d nginx
docker run --name mysql -p 8080:80 -d nginx:v2
docker images
docker ps
docker logs -f nginx
docker logs -f mysql
2. filebeat
filebeat.inputs:
- type: docker
containers.ids:
- '*'
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.kibana:
host: "10.0.0.51:5601"
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
index: "docker-nginx-%{[beat.version]}-%{+yyyy.MM}"
setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true
3. filebeat
systemctl restart filebeat
3.2 서비스 유형 에 따라 docker 용기 로 그 를 분리 합 니 다.
1. docker-compose
yum install -y python2-pip
2. pip ， ， ，
https://mirrors.tuna.tsinghua.edu.cn/help/pypi/
pip
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple pip -U
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
3. docker-compose
pip install docker-compose
4.
docker-compose version
5. docker-compose
[root@db03 ~]# cat docker-compose.yml
version: '3'
services:
nginx:
image: nginx:latest
# labels
labels:
service: nginx
#logging labels.service
logging:
options:
labels: "service"
ports:
- "80:80"
mysql:
image: nginx:v2
# labels
labels:
service: mysql
#logging labels.service
logging:
options:
labels: "service"
ports:
- "8080:80"
6. docker-compose docker
docker stop $(docker ps -q)
docker rm $(docker ps -aq)
docker-compose up -d
docker ps
7. filebeat
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/lib/docker/containers/*/*-json.log
json.keys_under_root: true
json.overwrite_keys: true
setup.kibana:
host: "10.0.0.51:5601"
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "docker-nginx-access-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
attrs.service: "nginx"
stream: "stdout"
- index: "docker-nginx-error-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
attrs.service: "nginx"
stream: "stderr"
- index: "docker-mysql-access-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
attrs.service: "mysql"
stream: "stdout"
- index: "docker-mysql-error-%{[beat.version]}-%{+yyyy.MM}"
when.contains:
attrs.service: "mysql"
stream: "stderr"
setup.template.name: "docker"
setup.template.pattern: "docker-*"
setup.template.enabled: false
setup.template.overwrite: true
4. filebeat 와 logstash 설정
filebeat redis，logstash redis
#filebeat
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/bbs_access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["bbs"]
- type: log
enabled: true
paths:
- /var/log/nginx/www_access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["www"]
setup.kibana:
host: "10.0.0.51:5601"
output.redis:
hosts: ["localhost"]
keys:
- key: "bbs"
when.contains:
tags: "bbs"
- key: "www"
when.contains:
tags: "www"
db: 0
timeout: 5
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
#redis
redis-cli
keys *
llen bbs
llen www
#logstash
[root@db01 /data/soft]# cat /etc/logstash/conf.d/redis.conf
input {
redis {
host => "127.0.0.1"
port => "6379"
db => "0"
key => "bbs"
data_type => "list"
}
redis {
host => "127.0.0.1"
port => "6379"
db => "0"
key => "www"
data_type => "list"
}
}
#filter {
# mutate {
# convert => ["upstream_time", "float"]
# convert => ["request_time", "float"]
# }
#}
output {
if "bbs" in [tags] {
stdout {}
elasticsearch {
hosts => "http://10.0.0.51:9200"
manage_template => false
index => "nginx-bbs-%{+yyyy.MM}"
}
}
if "www" in [tags] {
stdout {}
elasticsearch {
hosts => "http://10.0.0.51:9200"
manage_template => false
index => "nginx-www-%{+yyyy.MM}"
}
}
}
#logstash
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis.conf
4.1, redis 와 logstash 설정 최적화
#####filebeat#######
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/bbs_access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["bbs"]
- type: log
enabled: true
paths:
- /var/log/nginx/www_access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["www"]
setup.kibana:
host: "10.0.0.51:5601"
output.redis:
hosts: ["localhost"]
key: "all_keys"
db: 0
timeout: 5
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
#######logstash########
input {
redis {
host => "127.0.0.1"
port => "6379"
db => "0"
key => "all_keys"
data_type => "list"
}
#filter {
# mutate {
# convert => ["upstream_time", "float"]
# convert => ["request_time", "float"]
# }
#}
output {
if "bbs" in [tags] {
stdout {}
elasticsearch {
hosts => "http://10.0.0.51:9200"
manage_template => false
index => "nginx-bbs-%{+yyyy.MM}"
}
}
if "www" in [tags] {
stdout {}
elasticsearch {
hosts => "http://10.0.0.51:9200"
manage_template => false
index => "nginx-www-%{+yyyy.MM}"
}
}
}
이 내용에 흥미가 있습니까?
현재 기사가 여러분의 문제를 해결하지 못하는 경우 AI 엔진은 머신러닝 분석(스마트 모델이 방금 만들어져 부정확한 경우가 있을 수 있음)을 통해 가장 유사한 기사를 추천합니다:
다양한 언어의 JSONJSON은 Javascript 표기법을 사용하여 데이터 구조를 레이아웃하는 데이터 형식입니다. 그러나 Javascript가 코드에서 이러한 구조를 나타낼 수 있는 유일한 언어는 아닙니다. 저는 일반적으로 '객체'{}...
텍스트를 자유롭게 공유하거나 복사할 수 있습니다.하지만 이 문서의 URL은 참조 URL로 남겨 두십시오.
CC BY-SA 2.5, CC BY-SA 3.0 및 CC BY-SA 4.0에 따라 라이센스가 부여됩니다.
좋은 웹페이지 즐겨찾기
