Spark在Hadooprdd上花了很长时间：输入分裂

Question

18/02/07 04:44:51 INFO TaskSetManager：在1025毫秒内完成localhost（executor driver）上阶段1.0（TID 7）中的任务6.0（7/307）

为什么Spark要做这么多'hadooprdd:Input拆分‘？这样做的目的是什么？我如何加快或摆脱这个过程？

代码如下：

import org.apache.spark.SparkContext
import org.apache.spark.mllib.evaluation.MulticlassMetrics 

import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.mllib.classification.LogisticRegressionWithSGD
import org.apache.spark.mllib.evaluation.BinaryClassificationMetrics
import org.apache.spark.mllib.optimization.L1Updater
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.util.MLUtils
import scala.compat.Platform._ 


object test {

    def main(args: Array[String]) {

        val nnodes = 1
        val epochs = 3

        val conf = new SparkConf().setAppName("Test Name")
        val sc = new SparkContext(conf)

        val t0=currentTime
        val train = MLUtils.loadLibSVMFile(sc, "/ebs2/preprocess/xaa",  262165, 4)
        val test = MLUtils.loadLibSVMFile(sc,  "/ebs2/preprocess/xab",   262165, 4)
        val t1=currentTime;
    
        println("START")
        val lrAlg = new LogisticRegressionWithSGD()
        lrAlg.optimizer.setMiniBatchFraction(10.0/40000000.0)
        lrAlg.optimizer.setNumIterations(12000000)
        lrAlg.optimizer.setStepSize(0.01)

        val model = lrAlg.run(train)        


        model.clearThreshold()
        val scoreAndLabels = test.map { point =>
              val score = model.predict(point.features)
              (score, point.label)
        }

        val metrics = new BinaryClassificationMetrics(scoreAndLabels)
        val auROC = metrics.areaUnderROC()
        println("Area under ROC = " + auROC)
     }
}

Answer 1

我通过跑步解决了速度问题

train=train.coalesce(1)train.cache()

通过将内存增加到总共64场演出。以前，由于内存不足，Spark可能无法正确缓存。