场景示例 - postfix日志

优质
小牛编辑
131浏览
2023-12-01

postfix 是 Linux 平台上最常用的邮件服务器软件。邮件服务的运维复杂度一向较高,在此提供一个针对 postfix 日志的解析处理方案。方案出自:https://github.com/whyscream/postfix-grok-patterns

因为 postfix 默认通过 syslog 方式输出日志,所以可以选择通过 rsyslog 直接转发给 logstash,也可以由 logstash 读取 rsyslog 记录的文件。下列配置中省略了对 syslog 协议解析的部分。

  1. filter {
  2. # grok log lines by program name (listed alpabetically)
  3. if [program] =~ /^postfix.*/anvil$/ {
  4. grok {
  5. patterns_dir => ["/etc/logstash/patterns.d"]
  6. match => [ "message", "%{POSTFIX_ANVIL}" ]
  7. tag_on_failure => [ "_grok_postfix_anvil_nomatch" ]
  8. add_tag => [ "_grok_postfix_success" ]
  9. }
  10. } else if [program] =~ /^postfix.*/bounce$/ {
  11. grok {
  12. patterns_dir => ["/etc/logstash/patterns.d"]
  13. match => [ "message", "%{POSTFIX_BOUNCE}" ]
  14. tag_on_failure => [ "_grok_postfix_bounce_nomatch" ]
  15. add_tag => [ "_grok_postfix_success" ]
  16. }
  17. } else if [program] =~ /^postfix.*/cleanup$/ {
  18. grok {
  19. patterns_dir => ["/etc/logstash/patterns.d"]
  20. match => [ "message", "%{POSTFIX_CLEANUP}" ]
  21. tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ]
  22. add_tag => [ "_grok_postfix_success" ]
  23. }
  24. } else if [program] =~ /^postfix.*/dnsblog$/ {
  25. grok {
  26. patterns_dir => ["/etc/logstash/patterns.d"]
  27. match => [ "message", "%{POSTFIX_DNSBLOG}" ]
  28. tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ]
  29. add_tag => [ "_grok_postfix_success" ]
  30. }
  31. } else if [program] =~ /^postfix.*/local$/ {
  32. grok {
  33. patterns_dir => ["/etc/logstash/patterns.d"]
  34. match => [ "message", "%{POSTFIX_LOCAL}" ]
  35. tag_on_failure => [ "_grok_postfix_local_nomatch" ]
  36. add_tag => [ "_grok_postfix_success" ]
  37. }
  38. } else if [program] =~ /^postfix.*/master$/ {
  39. grok {
  40. patterns_dir => ["/etc/logstash/patterns.d"]
  41. match => [ "message", "%{POSTFIX_MASTER}" ]
  42. tag_on_failure => [ "_grok_postfix_master_nomatch" ]
  43. add_tag => [ "_grok_postfix_success" ]
  44. }
  45. } else if [program] =~ /^postfix.*/pickup$/ {
  46. grok {
  47. patterns_dir => ["/etc/logstash/patterns.d"]
  48. match => [ "message", "%{POSTFIX_PICKUP}" ]
  49. tag_on_failure => [ "_grok_postfix_pickup_nomatch" ]
  50. add_tag => [ "_grok_postfix_success" ]
  51. }
  52. } else if [program] =~ /^postfix.*/pipe$/ {
  53. grok {
  54. patterns_dir => ["/etc/logstash/patterns.d"]
  55. match => [ "message", "%{POSTFIX_PIPE}" ]
  56. tag_on_failure => [ "_grok_postfix_pipe_nomatch" ]
  57. add_tag => [ "_grok_postfix_success" ]
  58. }
  59. } else if [program] =~ /^postfix.*/postdrop$/ {
  60. grok {
  61. patterns_dir => ["/etc/logstash/patterns.d"]
  62. match => [ "message", "%{POSTFIX_POSTDROP}" ]
  63. tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ]
  64. add_tag => [ "_grok_postfix_success" ]
  65. }
  66. } else if [program] =~ /^postfix.*/postscreen$/ {
  67. grok {
  68. patterns_dir => ["/etc/logstash/patterns.d"]
  69. match => [ "message", "%{POSTFIX_POSTSCREEN}" ]
  70. tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ]
  71. add_tag => [ "_grok_postfix_success" ]
  72. }
  73. } else if [program] =~ /^postfix.*/qmgr$/ {
  74. grok {
  75. patterns_dir => ["/etc/logstash/patterns.d"]
  76. match => [ "message", "%{POSTFIX_QMGR}" ]
  77. tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ]
  78. add_tag => [ "_grok_postfix_success" ]
  79. }
  80. } else if [program] =~ /^postfix.*/scache$/ {
  81. grok {
  82. patterns_dir => ["/etc/logstash/patterns.d"]
  83. match => [ "message", "%{POSTFIX_SCACHE}" ]
  84. tag_on_failure => [ "_grok_postfix_scache_nomatch" ]
  85. add_tag => [ "_grok_postfix_success" ]
  86. }
  87. } else if [program] =~ /^postfix.*/sendmail$/ {
  88. grok {
  89. patterns_dir => ["/etc/logstash/patterns.d"]
  90. match => [ "message", "%{POSTFIX_SENDMAIL}" ]
  91. tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ]
  92. add_tag => [ "_grok_postfix_success" ]
  93. }
  94. } else if [program] =~ /^postfix.*/smtp$/ {
  95. grok {
  96. patterns_dir => ["/etc/logstash/patterns.d"]
  97. match => [ "message", "%{POSTFIX_SMTP}" ]
  98. tag_on_failure => [ "_grok_postfix_smtp_nomatch" ]
  99. add_tag => [ "_grok_postfix_success" ]
  100. }
  101. } else if [program] =~ /^postfix.*/lmtp$/ {
  102. grok {
  103. patterns_dir => ["/etc/logstash/patterns.d"]
  104. match => [ "message", "%{POSTFIX_LMTP}" ]
  105. tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ]
  106. add_tag => [ "_grok_postfix_success" ]
  107. }
  108. } else if [program] =~ /^postfix.*/smtpd$/ {
  109. grok {
  110. patterns_dir => ["/etc/logstash/patterns.d"]
  111. match => [ "message", "%{POSTFIX_SMTPD}" ]
  112. tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ]
  113. add_tag => [ "_grok_postfix_success" ]
  114. }
  115. } else if [program] =~ /^postfix.*/tlsmgr$/ {
  116. grok {
  117. patterns_dir => ["/etc/logstash/patterns.d"]
  118. match => [ "message", "%{POSTFIX_TLSMGR}" ]
  119. tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ]
  120. add_tag => [ "_grok_postfix_success" ]
  121. }
  122. } else if [program] =~ /^postfix.*/tlsproxy$/ {
  123. grok {
  124. patterns_dir => ["/etc/logstash/patterns.d"]
  125. match => [ "message", "%{POSTFIX_TLSPROXY}" ]
  126. tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ]
  127. add_tag => [ "_grok_postfix_success" ]
  128. }
  129. } else if [program] =~ /^postfix.*/trivial-rewrite$/ {
  130. grok {
  131. patterns_dir => ["/etc/logstash/patterns.d"]
  132. match => [ "message", "%{POSTFIX_TRIVIAL_REWRITE}" ]
  133. tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ]
  134. add_tag => [ "_grok_postfix_success" ]
  135. }
  136. } else if [program] =~ /^postfix.*/discard$/ {
  137. grok {
  138. patterns_dir => ["/etc/logstash/patterns.d"]
  139. match => [ "message", "%{POSTFIX_DISCARD}" ]
  140. tag_on_failure => [ "_grok_postfix_discard_nomatch" ]
  141. add_tag => [ "_grok_postfix_success" ]
  142. }
  143. }
  144. # process key-value data is it exists
  145. if [postfix_keyvalue_data] {
  146. kv {
  147. source => "postfix_keyvalue_data"
  148. trim => "<>,"
  149. prefix => "postfix_"
  150. remove_field => [ "postfix_keyvalue_data" ]
  151. }
  152. # some post processing of key-value data
  153. if [postfix_client] {
  154. grok {
  155. patterns_dir => ["/etc/logstash/patterns.d"]
  156. match => ["postfix_client", "%{POSTFIX_CLIENT_INFO}"]
  157. tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ]
  158. remove_field => [ "postfix_client" ]
  159. }
  160. }
  161. if [postfix_relay] {
  162. grok {
  163. patterns_dir => ["/etc/logstash/patterns.d"]
  164. match => ["postfix_relay", "%{POSTFIX_RELAY_INFO}"]
  165. tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ]
  166. remove_field => [ "postfix_relay" ]
  167. }
  168. }
  169. if [postfix_delays] {
  170. grok {
  171. patterns_dir => ["/etc/logstash/patterns.d"]
  172. match => ["postfix_delays", "%{POSTFIX_DELAYS}"]
  173. tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ]
  174. remove_field => [ "postfix_delays" ]
  175. }
  176. }
  177. }
  178. # Do some data type conversions
  179. mutate {
  180. convert => [
  181. # list of integer fields
  182. "postfix_anvil_cache_size", "integer",
  183. "postfix_anvil_conn_count", "integer",
  184. "postfix_anvil_conn_rate", "integer",
  185. "postfix_client_port", "integer",
  186. "postfix_nrcpt", "integer",
  187. "postfix_postscreen_cache_dropped", "integer",
  188. "postfix_postscreen_cache_retained", "integer",
  189. "postfix_postscreen_dnsbl_rank", "integer",
  190. "postfix_relay_port", "integer",
  191. "postfix_server_port", "integer",
  192. "postfix_size", "integer",
  193. "postfix_status_code", "integer",
  194. "postfix_termination_signal", "integer",
  195. "postfix_uid", "integer",
  196. # list of float fields
  197. "postfix_delay", "float",
  198. "postfix_delay_before_qmgr", "float",
  199. "postfix_delay_conn_setup", "float",
  200. "postfix_delay_in_qmgr", "float",
  201. "postfix_delay_transmission", "float",
  202. "postfix_postscreen_violation_time", "float"
  203. ]
  204. }
  205. }

配置中使用了一系列自定义 grok 正则,全部内容如下所示。保存成 /etc/logstash/patterns.d/ 下一个文本文件即可。

  1. # common postfix patterns
  2. POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{15,}|NOQUEUE)
  3. POSTFIX_CLIENT_INFO %{HOST:postfix_client_hostname}?[%{IP:postfix_client_ip}](:%{INT:postfix_client_port})?
  4. POSTFIX_RELAY_INFO %{HOST:postfix_relay_hostname}?[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service}
  5. POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL|RCPT|DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|.)
  6. POSTFIX_ACTION (reject|defer|accept|header-redirect)
  7. POSTFIX_STATUS_CODE d{3}
  8. POSTFIX_STATUS_CODE_ENHANCED d.d.d
  9. POSTFIX_DNSBL_MESSAGE Service unavailable; .* [%{GREEDYDATA:postfix_status_data}] %{GREEDYDATA:postfix_status_message};
  10. POSTFIX_PS_ACCESS_ACTION (DISCONNECT|BLACKLISTED|WHITELISTED|WHITELIST VETO|PASS NEW|PASS OLD)
  11. POSTFIX_PS_VIOLATION (BARE NEWLINE|COMMAND (TIME|COUNT|LENGTH) LIMIT|COMMAND PIPELINING|DNSBL|HANGUP|NON-SMTP COMMAND|PREGREET)
  12. POSTFIX_TIME_UNIT %{NUMBER}[smhd]
  13. POSTFIX_KEYVALUE %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data}
  14. POSTFIX_WARNING (warning|fatal): %{GREEDYDATA:postfix_warning}
  15. POSTFIX_TLSCONN (Anonymous|Trusted|Untrusted|Verified) TLS connection established (to %{POSTFIX_RELAY_INFO}|from %{POSTFIX_CLIENT_INFO}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} (%{DATA:postfix_tls_cipher_size} bits)
  16. POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission}
  17. POSTFIX_LOSTCONN (lost connection|timeout|Connection timed out)
  18. POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.*
  19. # helper patterns
  20. GREEDYDATA_NO_COLON [^:]*
  21. # smtpd patterns
  22. POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT_INFO}
  23. POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT_INFO}
  24. POSTFIX_SMTPD_LOSTCONN (%{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data} after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( (%{INT} bytes))? from %{POSTFIX_CLIENT_INFO}|%{GREEDYDATA:postfix_action} from %{POSTFIX_CLIENT_INFO}: %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data})
  25. POSTFIX_SMTPD_NOQUEUE NOQUEUE: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced}( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{GREEDYDATA:postfix_keyvalue_data}
  26. POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:
  27. POSTFIX_SMTPD_PROXY proxy-%{POSTFIX_ACTION:postfix_proxy_result}: (%{POSTFIX_SMTP_STAGE:postfix_proxy_smtp_stage}): %{POSTFIX_PROXY_MESSAGE:postfix_proxy_message}; %{GREEDYDATA:postfix_keyvalue_data}
  28. # cleanup patterns
  29. POSTFIX_CLEANUP_MILTER %{POSTFIX_QUEUEID:postfix_queueid}: milter-%{POSTFIX_ACTION:postfix_milter_result}: %{GREEDYDATA:postfix_milter_message}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}(: %{GREEDYDATA:postfix_milter_data})?
  30. # qmgr patterns
  31. POSTFIX_QMGR_REMOVED %{POSTFIX_QUEUEID:postfix_queueid}: removed
  32. POSTFIX_QMGR_ACTIVE %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data} (queue active)
  33. # pipe patterns
  34. POSTFIX_PIPE_DELIVERED %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data} (delivered via %{WORD:postfix_pipe_service} service)
  35. POSTFIX_PIPE_FORWARD %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data} (mail forwarding loop for %{GREEDYDATA:postfix_to})
  36. # postscreen patterns
  37. POSTFIX_PS_CONNECT CONNECT from %{POSTFIX_CLIENT_INFO} to [%{IP:postfix_server_ip}]:%{INT:postfix_server_port}
  38. POSTFIX_PS_ACCESS %{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access} %{POSTFIX_CLIENT_INFO}
  39. POSTFIX_PS_NOQUEUE %{POSTFIX_SMTPD_NOQUEUE}
  40. POSTFIX_PS_TOOBUSY NOQUEUE: reject: CONNECT from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_postscreen_toobusy_data}
  41. POSTFIX_PS_DNSBL %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation} rank %{INT:postfix_postscreen_dnsbl_rank} for %{POSTFIX_CLIENT_INFO}
  42. POSTFIX_PS_CACHE cache %{DATA} full cleanup: retained=%{NUMBER:postfix_postscreen_cache_retained} dropped=%{NUMBER:postfix_postscreen_cache_dropped} entries
  43. POSTFIX_PS_VIOLATIONS %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}( %{INT})?( after %{NUMBER:postfix_postscreen_violation_time})? from %{POSTFIX_CLIENT_INFO}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})?
  44. # dnsblog patterns
  45. POSTFIX_DNSBLOG_LISTING addr %{IP:postfix_client_ip} listed by domain %{HOST:postfix_dnsbl_domain} as %{IP:postfix_dnsbl_result}
  46. # tlsproxy patterns
  47. POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT_INFO}
  48. # anvil patterns
  49. POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for (%{DATA:postfix_service}:%{IP:postfix_client_ip}) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
  50. POSTFIX_ANVIL_CONN_CACHE statistics: max cache size %{NUMBER:postfix_anvil_cache_size} at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
  51. POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for (%{DATA:postfix_service}:%{IP:postfix_client_ip}) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp}
  52. # smtp patterns
  53. POSTFIX_SMTP_DELIVERY %{POSTFIX_KEYVALUE} status=%{WORD:postfix_status}( (%{GREEDYDATA:postfix_smtp_response}))?
  54. POSTFIX_SMTP_CONNERR connect to %{POSTFIX_RELAY_INFO}: (Connection timed out|No route to host|Connection refused)
  55. POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN} with %{POSTFIX_RELAY_INFO}
  56. # master patterns
  57. POSTFIX_MASTER_START (daemon started|reload) -- version %{DATA:postfix_version}, configuration %{PATH:postfix_config_path}
  58. POSTFIX_MASTER_EXIT terminating on signal %{INT:postfix_termination_signal}
  59. # bounce patterns
  60. POSTFIX_BOUNCE_NOTIFICATION %{POSTFIX_QUEUEID:postfix_queueid}: sender (non-delivery|delivery status|delay) notification: %{POSTFIX_QUEUEID:postfix_bounce_queueid}
  61. # scache patterns
  62. POSTFIX_SCACHE_LOOKUPS statistics: (address|domain) lookup hits=%{INT:postfix_scache_hits} miss=%{INT:postfix_scache_miss} success=%{INT:postfix_scache_success}
  63. POSTFIX_SCACHE_SIMULTANEOUS statistics: max simultaneous domains=%{INT:postfix_scache_domains} addresses=%{INT:postfix_scache_addresses} connection=%{INT:postfix_scache_connection}
  64. POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_scache_timestamp}
  65. # aggregate all patterns
  66. POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE}
  67. POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}
  68. POSTFIX_QMGR %{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_WARNING}
  69. POSTFIX_PIPE %{POSTFIX_PIPE_DELIVERED}|%{POSTFIX_PIPE_FORWARD}
  70. POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING}
  71. POSTFIX_DNSBLOG %{POSTFIX_DNSBLOG_LISTING}
  72. POSTFIX_ANVIL %{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT}
  73. POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}
  74. POSTFIX_DISCARD %{POSTFIX_KEYVALUE} status=%{WORD:postfix_status}
  75. POSTFIX_LMTP %{POSTFIX_SMTP}
  76. POSTFIX_PICKUP %{POSTFIX_KEYVALUE}
  77. POSTFIX_TLSPROXY %{POSTFIX_TLSPROXY_CONN}
  78. POSTFIX_MASTER %{POSTFIX_MASTER_START}|%{POSTFIX_MASTER_EXIT}
  79. POSTFIX_BOUNCE %{POSTFIX_BOUNCE_NOTIFICATION}
  80. POSTFIX_SENDMAIL %{POSTFIX_WARNING}
  81. POSTFIX_POSTDROP %{POSTFIX_WARNING}
  82. POSTFIX_SCACHE %{POSTFIX_SCACHE_LOOKUPS}|%{POSTFIX_SCACHE_SIMULTANEOUS}|%{POSTFIX_SCACHE_TIMESTAMP}
  83. POSTFIX_TRIVIAL_REWRITE %{POSTFIX_WARNING}
  84. POSTFIX_TLSMGR %{POSTFIX_WARNING}
  85. POSTFIX_LOCAL %{POSTFIX_KEYVALUE}