ELK - log4j2自定义Layout以json格式输出日志到logstash，支持收集多个项目的日志

logstash 新增配置文件 log4j2.conf

input {
    #开启远程输入日志服务
    tcp {
        port => "4560"
        mode => "server"
        type => "log4j2"
    }
}

filter {
    #将日志转成json对象
    json {
        source => "message"
    }
    #将远程客户端的日志时间设置为插入时间，不设置默认为当前系统时间，可能会存在时间误差
    date {
        match => ["time", "yyyy-MM-dd HH:mm:ss.SSS"]
        remove_field => ["time"]
    }
}

output {
    elasticsearch {
        hosts => ["192.168.1.138:9200"]
        index  => "application-%{+YYYY.MM.dd}"
    }
}

linux 系统下启动 logstash

bin/logstash -f config/log4j.conf

在项目代码里添加自定义Layout类

import com.fasterxml.jackson.core.JsonProcessingException;
import com.gzkit.core.utils.JacksonUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.Node;
import org.apache.logging.log4j.core.config.plugins.*;
import org.apache.logging.log4j.core.layout.AbstractStringLayout;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.apache.logging.log4j.core.layout.PatternSelector;
import org.apache.logging.log4j.core.pattern.RegexReplacement;

import java.io.File;
import java.nio.charset.Charset;

/**
 * ELK日志格式
 * @Author lnk
 * @Date 2018/3/13
 */
@Plugin(name = "ElkJsonPatternLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true)
public class ElkJsonPatternLayout extends AbstractStringLayout {
    /** 项目路径 */
    private static String PROJECT_PATH;

    private PatternLayout patternLayout;

    private String projectName;
    private String logType;

    static {
        PROJECT_PATH = new File("").getAbsolutePath();
    }

    private ElkJsonPatternLayout(Configuration config, RegexReplacement replace, String eventPattern,
                                 PatternSelector patternSelector, Charset charset, boolean alwaysWriteExceptions,
                                 boolean noConsoleNoAnsi, String headerPattern, String footerPattern, String projectName, String logType) {
        super(config, charset,
                PatternLayout.createSerializer(config, replace, headerPattern, null, patternSelector, alwaysWriteExceptions,
                        noConsoleNoAnsi),
                PatternLayout.createSerializer(config, replace, footerPattern, null, patternSelector, alwaysWriteExceptions,
                        noConsoleNoAnsi));

        this.projectName = projectName;
        this.logType = logType;
        this.patternLayout = PatternLayout.newBuilder()
                .withPattern(eventPattern)
                .withPatternSelector(patternSelector)
                .withConfiguration(config)
                .withRegexReplacement(replace)
                .withCharset(charset)
                .withAlwaysWriteExceptions(alwaysWriteExceptions)
                .withNoConsoleNoAnsi(noConsoleNoAnsi)
                .withHeader(headerPattern)
                .withFooter(footerPattern)
                .build();
    }

    @Override
    public String toSerializable(LogEvent event) {
        //在这里处理日志内容
        String message = patternLayout.toSerializable(event);
        String jsonStr = new JsonLoggerInfo(projectName, message, event.getLevel().name(), logType, event.getTimeMillis()).toString();
        return jsonStr + "\n";
    }

    @PluginFactory
    public static ElkJsonPatternLayout createLayout(
            @PluginAttribute(value = "pattern", defaultString = PatternLayout.DEFAULT_CONVERSION_PATTERN) final String pattern,
            @PluginElement("PatternSelector") final PatternSelector patternSelector,
            @PluginConfiguration final Configuration config,
            @PluginElement("Replace") final RegexReplacement replace,
            // LOG4J2-783 use platform default by default, so do not specify defaultString for charset
            @PluginAttribute(value = "charset") final Charset charset,
            @PluginAttribute(value = "alwaysWriteExceptions", defaultBoolean = true) final boolean alwaysWriteExceptions,
            @PluginAttribute(value = "noConsoleNoAnsi", defaultBoolean = false) final boolean noConsoleNoAnsi,
            @PluginAttribute("header") final String headerPattern,
            @PluginAttribute("footer") final String footerPattern,
            @PluginAttribute("projectName") final String projectName,
            @PluginAttribute("logType") final String logType) {


        return new ElkJsonPatternLayout(config, replace, pattern, patternSelector, charset,
                alwaysWriteExceptions, noConsoleNoAnsi, headerPattern, footerPattern, projectName, logType);
    }

    /**
     * 输出的日志内容
     */
    public static class JsonLoggerInfo{
        /** 项目名 */
        private String projectName;
        /** 项目目录路径 */
        private String projectPath;
        /** 日志信息 */
        private String message;
        /** 日志级别 */
        private String level;
        /** 日志分类 */
        private String logType;
        /** 日志时间 */
        private String time;

        public JsonLoggerInfo(String projectName, String message, String level, String logType, long timeMillis) {
            this.projectName = projectName;
            this.projectPath = PROJECT_PATH;
            this.message = message;
            this.level = level;
            this.logType = logType;
            this.time = DateFormatUtils.format(timeMillis, "yyyy-MM-dd HH:mm:ss.SSS");
        }

        public String getProjectName() {
            return projectName;
        }

        public String getProjectPath() {
            return projectPath;
        }

        public String getMessage() {
            return message;
        }

        public String getLevel() {
            return level;
        }

        public String getLogType() {
            return logType;
        }

        public String getTime() {
            return time;
        }

        @Override
        public String toString() {
            try {
                return new ObjectMapper().writeValueAsString(this);
            } catch (JsonProcessingException e) {
                e.printStackTrace();
            }
            return null;
        }
    }
}

修改 log4j2.xml 添加下面几个节点

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" monitorInterval="30" packages="com.xxx.logging.log4j">
    <Properties>
        <Property name="PROJECT_NAME">my-project</Property>
        <Property name="ELK_LOG_PATTERN">${sys:PID} %c{3.}.%t %m%n</Property>
    </Properties>
    <Appenders>
        <!-- 输出业务日志到ELK日志系统 -->
        <Socket name="elk" host="192.168.1.138" port="4560">
            <ElkJsonPatternLayout pattern="${ELK_LOG_PATTERN}" projectName="${PROJECT_NAME}" logType="business" />
        </Socket>
    </Appenders>
    <Loggers>
        <Logger name="com.xxx" level="info">
            <AppenderRef ref="elk"/>
        </Logger>
    </Loggers>
</Configuration>

1. 根节点 Configuration 添加属性 packages 值为自定义Layout类的包路径，packages属性是扫描指定包下log4j的自定义扩展类。
2. 节点 Appenders 添加节点 Socket 使用 Socket 方式输出去远程 logstath。
3. 节点 Logger 指定日志使用Socket输出方式。
4. 上面配置文件中只贴出了关键配置内容，并不适合直接应用到实现项目中，将上面关键配置添加到自己项目的配置中即可。

在 Kibana 查看日志
1. 先使用log4j生成几条日志。
2. 进入 Kibana 添加新的 index索引 application-*，并刷一下字段，以便json日志的字段生效。
3. 然后去看application-*，可以根据 projectName 字段过滤查看不同项目的日志。