heka 配置一个go语言实现轻量级logstash 干掉ELK

萧展鹏

2023-12-01

[logstreamer_nginx_access]
type = "LogstreamerInput"
log_directory = "/access_pathlogs"
file_match = '(?P<Dir>[^/]*)/?access_(?P<FileName>.*)\.log'
differentiator = ["access_", "Dir", "_", "FileName"]
decoder = "Sandbox_nginx_access"

[logstreamer_nginx_error]
type = "LogstreamerInput"
log_directory = "/error_pathlogs"
file_match = '(?P<Dir>[^/]*)/?error_(?P<FileName>.*)\.log'
differentiator = ["error_", "Dir", "_", "FileName"]
decoder = "Sandbox_nginx_error"

## Sandboxes
[Sandbox_nginx_access]
type = "SandboxDecoder"
filename = "lua_decoders/nginx_access.lua"

[Sandbox_nginx_error]
type = "SandboxDecoder"
filename = "lua_decoders/nginx_error.lua"

[Sandbox_nginx_access.config]
type = "access"
user_agent_transform = true
##根据自己的日志格式调整
log_format = '$remote_addr - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" $request_time'

[Sandbox_nginx_error.config]
tz = "Asia/Shanghai"
type = "error"

[ESLogstashV0Encoder]
index = "logstash-dev-%{Type}-%{%Y.%m.%d}"
es_index_from_timestamp = true
fields = ["Timestamp", "Severity", "Pid", "Payload", "Hostname", "Logger", "Fields"]
type_name = "%{Type}"

##ES入库
[ElasticSearchOutput]
message_matcher = "Type == 'access' || Type == 'error'"
server = "http://xxxx:9200"
encoder = "ESLogstashV0Encoder"
flush_interval = 50
flush_count = 5000


[alert_smtp_encoder]
type = "SandboxEncoder"
filename = "lua_encoders/alert.lua"


##邮件告警
[SmtpOutput]
message_matcher = "(Severity == 3 && Payload =~ /runtime/) || (Severity == 3 && Payload =~ /matcher sting/ && Payload !~ /not matcher string/)"
encoder = ""
send_from = ""
send_to = ["x x x@qq.com","x x x2@qq.com"]
auth = "Plain"
user =
password = 
host =

在网上看了很多关于heka的配置，没有找到生产环境使用的，所以贴一个我们生产环境的配置。支持通配符文件路径，字符串匹配，ES入库和邮件告警。

相关连接：http://bigbo.github.io/pages/2015/05/23/mozilla_heka/

参考资料：

Heka logstreamer 说明文档
http://hekad.readthedocs.org/en/latest/pluginconfig/logstreamer.html#logstreamerplugin

Heka inputs 说明文档
http://hekad.readthedocs.org/en/latest/config/inputs/index.html

Heka getting started
https://hekad.readthedocs.org/en/latest/getting_started.html

Heka：Go编写，来自Mozilla，高效、灵活的插件式数据挖掘工具

http://www.csdn.net/article/2013-05-02/2815116-introduce-from-mozilla-heka-go

http://blog.mozilla.org/services/2013/04/30/introducing-heka/

PPT介绍

https://cdn.rawgit.com/gophercon/2014-talks/master/rob_miller_heka/index.html#/

Heka, 一个高可扩展的实时数据收集和处理工具