Docker 部署 ELK

下载

# 拉取 ELK + Filebeat
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.6.1
docker pull docker.elastic.co/kibana/kibana:7.6.1
docker pull docker.elastic.co/logstash/logstash:7.6.1
docker pull docker.elastic.co/beats/filebeat:7.6.1

# 查看镜像
docker images
# 返回
REPOSITORY                                      TAG                 IMAGE ID            CREATED             SIZE
docker.elastic.co/logstash/logstash             7.6.1               d6d66afe6805        10 days ago         813MB
docker.elastic.co/kibana/kibana                 7.6.1               f9ca33465ce3        10 days ago         1.01GB
docker.elastic.co/elasticsearch/elasticsearch   7.6.1               41072cdeebc5        10 days ago         790MB
docker.elastic.co/beats/filebeat                7.6.1               cd244d9a74c9        10 days ago         364MB

开启 ES

# 参考： https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html
docker run -d -p 9201:9200 -p 9301:9300 -e "discovery.type=single-node" \
docker.elastic.co/elasticsearch/elasticsearch:7.6.1

# 修改配置文件（可选项）
docker exec -it [CONTAINER ID] /bin/bash

开启 Kibana

# 参考：https://www.elastic.co/guide/en/kibana/current/docker.html
docker run --link [ES_CONTAINER ID]:elasticsearch -p 5602:5601 -d \
docker.elastic.co/kibana/kibana:7.6.1

开启 Logstash

• 配置 logstash.yml

# 创建宿主机文件存放文件夹
mkdir -p /usr/share/logstash/config/
cd /usr/share/logstash/config/
# 创建文件
vim logstash.yml
# 写入
http.host: "0.0.0.0"
xpack.management.pipeline.id: ["main"]
# 结束

• 配置 pipeline.yml

# 创建宿主机文件
vim pipeline.yml
# 写入（注意空格）
 - pipeline.id: main
  path.config: "/usr/share/logstash/pipeline/logstash.conf"
# 结束

• 配置 logstash.conf

# 创建宿主机文件存放文件夹
mkdir -p /usr/share/logstash/pipeline/
cd /usr/share/logstash/pipeline/
# 创建文件
vim logstash.conf
# 写入
input {
    beats { port => 5044 }
}

filter {
	 if [project] == "Nginx" {
                grok {
                        match => {
                                "message" =>[
                                        "%{IPORHOST:client_ip}\s{1,}\-\s\-\s\[%{HTTPDATE:time}\]\s{1,}\"(?:%{WORD:verb}\s{1,}%{NOTSPACE:request}(?:\s{1,}HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response}\s{1,}(?:%{NUMBER:bytes}|-)\s{1,}%{QS:referrer}\s{1,}%{QS:agent}"                
                                ]
                        }
                }
                date {
                         match => ["time","dd/MMM/yyyy:HH:mm:ss Z"]
                         target => "logdate"
                }

                mutate {
                        lowercase => ["company", "project", "server", "application"]
                        remove_field => ['cloud', 'tags', 'host', 'agent', 'log', 'ecs', '@version', 'message']
                }

                ruby {
                    code => "event.set('logdate', event.get('logdate').time.localtime)"
                }
                geoip {
                        source => "client_ip"
                }

        }
}

output {
    elasticsearch {
        hosts => ["192.168.60.221:9201"]
        index => "%{[company]}_%{[project]}_%{[server]}_%{[application]}_%{+YYYY}"
    }
}
# 结束

• 开启 Logstash

# 参考：https://www.elastic.co/guide/en/logstash/current/docker.html
docker run -d -p 5046:5044 -p 9601:9600 --rm -it -v /usr/share/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml \
-v /usr/share/logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml \
-v /usr/share/logstash/pipeline/logstash.conf:/usr/share/logstash/pipeline/logstash.conf \
docker.elastic.co/logstash/logstash:7.6.1

开启 Filebeat

# 参考：https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html
# 创建宿主机文件存放文件夹
mkdir -p /usr/share/filebeat/
cd /usr/share/filebeat/
# 创建文件 filebeat.yml 
# 修改权限
chmod go-w /usr/share/filebeat/filebeat.yml
# 开启 Filebeat（失败，没搞定）
docker run -v /usr/share/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml \
-v /usr/local/nginx/logs/access.log:/var/lib/docker/containers/access.log \
docker.elastic.co/beats/filebeat:7.6.1