如何使用log4j2版本覆盖log4j，以解决apache-core_2.12版本的“易反序列化的SocketServer类”

Question

如何覆盖log4j版本1.2。17带有log4j核心2.16。0版本以解决spark-core_2.12二进制文件的“易反序列化的SocketServer类”。

我试图排除log4j-1.2。17来自spark-core_2.12，但构建失败，出现以下错误

java.lang.NoClassDefFoundError: org/apache/log4j/spi/Filter
    at com.optum.iqs.runtime.counters.CountersFactoryTest.setup(CountersFactoryTest.java:20)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
    at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:89)
    at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:41)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:541)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:763)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:463)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:209)
Caused by: java.lang.ClassNotFoundException: org.apache.log4j.spi.Filter
    at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:355)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
    ... 19 more

java.lang.NullPointerException
    at com.optum.iqs.runtime.counters.CountersFactoryTest.tearDown(CountersFactoryTest.java:44)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
    at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:89)
    at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:41)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:541)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:763)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:463)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:209)

我的CountersFactoryTest.java类内容是

    package com.optum.iqs.runtime.counters;
    
    import java.io.IOException;
    import java.util.Map;
    import org.apache.spark.sql.Dataset;
    import org.apache.spark.sql.Encoders;
    import org.apache.spark.sql.SparkSession;
    import org.junit.AfterClass;
    import org.junit.Assert;
    import org.junit.BeforeClass;
    import org.junit.Test;
    
    public class CountersFactoryTest {
        public static SparkSession sparkSession;
    
        @BeforeClass
        public static void setup() throws IOException {
            sparkSession = SparkSession.builder().appName("JunitSession").master("local").getOrCreate();
            CountersFactory.initialize(sparkSession.sparkContext());
        }
    
        @Test
        public void testIqsLongCounter() {
            Dataset<String> input = sparkSession.read().textFile("src/test/resources/input.txt").as(Encoders.STRING());
            Dataset<String> output = input.map(new SampleMapFunction(), Encoders.STRING());
            output.show();
            Long counter = CountersFactory.getCounters().get("input records").value();
            Assert.assertEquals(3L, counter.longValue());
        }
        
        @Test
        public void testIqsOutputRecordCounter() {
            Dataset<String> input = sparkSession.read().textFile("src/test/resources/input.txt").as(Encoders.STRING());
            Dataset<String> output = input.map(new SampleMapFunctionOutputCounter(), Encoders.STRING());
            output.show();
            Map<String, Long> counter = CountersFactory.getOutputCounters().get("output records").value();
            Assert.assertEquals(3L, counter.get("key").longValue());
        }
    
        @AfterClass
        public static void tearDown() {
            sparkSession.stop();
        }
    }

从火花核心依赖中排除log4j的参考

<dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.0.1</version>
            <exclusions>
                <exclusion>
                    <groupId>log4j</groupId>
                    <artifactId>log4j</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

spark core是否将log4j依赖项升级到最新版本？

CVE-2019-17571

Answer 1

[备注：CVE-2019-1751几乎不可能触发，除非您明确运行SocketServer（参见此问题）。另一方面，Log4j 1.2.17也会受到CVE-2021-4104，这可能更容易利用。]

从log4j2开始。x与Log4j 1.2不向后兼容升级并不像用一个库替换另一个库那么简单。您需要用log4j1.2和log4j2之间的桥替换log4j依赖关系。x API（log4j-1.2-API）：

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.12</artifactId>
    <version>3.0.1</version>
    <exclusions>
        <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
        </exclusion>
    </exclusions>
</dependency>
<dependency>
    <groupId>org.apache.logging.log4j</groupId>
    <artifactId>log4j-1.2-api</artifactId>
    <version>2.16.0</version>
</dependency>

这应该可以解决ClassNotFoundException，但是如果您想实际将日志保存到某个地方，则需要为Log4j 2安装后端。x APIlog4j core是标准配置。