Spark/Spark-JobServer 执行SQL时任务超时
曾晨
1 问题，SQL执行时超时
2022/03/18 17:36:47 HttpRequest GET http://10.1.170.115:8080/v1/statement/20220318_093646_03065_awf2q/2
map[errorClass:java.util.concurrent.TimeoutException message:Futures timed out after [600 seconds] stack:java.util.concurrent.TimeoutException: Futures timed out after [600 seconds]
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:223)
at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:227)
at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:190)
at akka.dispatch.MonitorableThreadFactory$AkkaForkJoinWorkerThread$$anon$3.block(ThreadPoolBuilder.scala:167)
at akka.dispatch.forkjoin.ForkJoinPool.managedBlock(ForkJoinPool.java:3641)
at akka.dispatch.MonitorableThreadFactory$AkkaForkJoinWorkerThread.blockOn(ThreadPoolBuilder.scala:165)
at akka.dispatch.BatchingExecutor$BlockableBatch.blockOn(BatchingExecutor.scala:106)
at scala.concurrent.Await$.result(package.scala:190)
at spark.jobserver.WebApi.spark$jobserver$WebApi$$getJobManagerForContext(WebApi.scala:812)
at spark.jobserver.WebApi$$anonfun$jobRoutes$1$$anonfun$apply$94$$anonfun$apply$95.apply(WebApi.scala:723)
at spark.jobserver.WebApi$$anonfun$jobRoutes$1$$anonfun$apply$94$$anonfun$apply$95.apply(WebApi.scala:713)
at spray.routing.ApplyConverterInstances$$anon$17$$anonfun$apply$6.apply(ApplyConverterInstances.scala:60)
at spray.routing.ApplyConverterInstances$$anon$17$$anonfun$apply$6.apply(ApplyConverterInstances.scala:59)
at spray.routing.ConjunctionMagnet$$anon$3$$anon$4$$anonfun$happly$1$$anonfun$apply$1.apply(Directive.scala:38)
at spray.routing.ConjunctionMagnet$$anon$3$$anon$4$$anonfun$happly$1$$anonfun$apply$1.apply(Directive.scala:37)
at spray.routing.directives.BasicDirectives$$anon$2.happly(BasicDirectives.scala:79)
at spray.routing.Directive$$anon$7$$anonfun$happly$4.apply(Directive.scala:86)
at spray.routing.Directive$$anon$7$$anonfun$happly$4.apply(Directive.scala:86)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30)
at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30)
at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31)
at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext.reject(RequestContext.scala:202)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31)
at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext.reject(RequestContext.scala:202)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31)
at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext.reject(RequestContext.scala:202)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31)
at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext.reject(RequestContext.scala:202)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31)
at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext.reject(RequestContext.scala:202)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31)
at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext.reject(RequestContext.scala:202)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31)
at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130)
at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36)
at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84)
at akka.spray.UnregisteredActorRefBase.$bang(UnregisteredActorRefBase.scala:72)
at spray.routing.RequestContext.reject(RequestContext.scala:202)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35)
at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30)
at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29)
at spray.routing.directives.OnSuccessFutureMagnet$$anon$2$$anonfun$happly$2$$anonfun$apply$2.apply(FutureDirectives.scala:81)
at spray.routing.directives.OnSuccessFutureMagnet$$anon$2$$anonfun$happly$2$$anonfun$apply$2.apply(FutureDirectives.scala:79)
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36)
at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)
at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:91)
at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)
at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)
at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:90)
at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40)
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:43)
at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
]
2022/03/18 17:37:33 [-] Plugin:ParseInputTablePlugin @ 0.0.1 -------------------------
2022/03/18 17:37:33 [-] Plugin:ContextSavePlugin @ 0.0.2 -------------------------
2022/03/18 17:37:33 [-] Plugin:AuditLogPlugin @ 0.0.1 -------------------------
定位
- 可能性1，初步判断Spark在获取结果数据时，数据量过大导致超时
- 可能性2，当Spark 进行broadcast join时，其中一个表数据过大，导致数据拉取时超过broadcast join的广播时间
解决
- 调整参数
spark.sql.broadcastTimeout=3600
类似资料：