问题描述
我正在使用 EMR 6.2.0 和 livy 0.7.0
我有一个工作人员，它不断启动 spark 会话并轮询它的完成情况。这运行了一段时间但突然Livy开始投掷
java.lang.IllegalStateException: Failed to create a child event loop
at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:88)
at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:58)
at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:47)
at io.netty.channel.MultithreadEventLoopGroup.<init>(MultithreadEventLoopGroup.java:59)
at io.netty.channel.nio.NioEventLoopGroup.<init>(NioEventLoopGroup.java:77)
at io.netty.channel.nio.NioEventLoopGroup.<init>(NioEventLoopGroup.java:72)
at io.netty.channel.nio.NioEventLoopGroup.<init>(NioEventLoopGroup.java:59)
at org.apache.livy.rsc.RScclient.<init>(RScclient.java:81)
at org.apache.livy.rsc.RScclientFactory.createClient(RScclientFactory.java:76)
at org.apache.livy.LivyClientBuilder.build(LivyClientBuilder.java:130)
at org.apache.livy.server.interactive.InteractiveSession$.$anonfun$create$1(InteractiveSession.scala:111)
at scala.Option.orElse(Option.scala:447)
at org.apache.livy.server.interactive.InteractiveSession$.create(InteractiveSession.scala:81)
at org.apache.livy.server.interactive.InteractiveSessionServlet.createSession(InteractiveSessionServlet.scala:55)
at org.apache.livy.server.interactive.InteractiveSessionServlet.createSession(InteractiveSessionServlet.scala:40)
at org.apache.livy.server.SessionServlet.$anonfun$new$16(SessionServlet.scala:139)
at org.scalatra.ScalatraBase.liftAction(ScalatraBase.scala:280)
at org.scalatra.ScalatraBase.$anonfun$invoke$1(ScalatraBase.scala:274)
at org.scalatra.ApiFormats.withRouteMultiParams(ApiFormats.scala:183)
at org.scalatra.ApiFormats.withRouteMultiParams$(ApiFormats.scala:169)
at org.apache.livy.server.JsonServlet.withRouteMultiParams(JsonServlet.scala:39)
at org.scalatra.ScalatraBase.invoke(ScalatraBase.scala:273)
at org.scalatra.ScalatraBase.invoke$(ScalatraBase.scala:272)
at org.apache.livy.server.JsonServlet.invoke(JsonServlet.scala:39)
at org.scalatra.ScalatraBase.$anonfun$runRoutes$3(ScalatraBase.scala:248)
at scala.Option.flatMap(Option.scala:271)
at org.scalatra.ScalatraBase.$anonfun$runRoutes$1(ScalatraBase.scala:246)
at scala.collection.immutable.Stream.flatMap(Stream.scala:493)
at org.scalatra.ScalatraBase.runRoutes(ScalatraBase.scala:245)
at org.scalatra.ScalatraBase.runRoutes$(ScalatraBase.scala:243)
at org.apache.livy.server.JsonServlet.runRoutes(JsonServlet.scala:39)
at org.scalatra.ScalatraBase.runActions$1(ScalatraBase.scala:169)
at org.scalatra.ScalatraBase.$anonfun$executeRoutes$6(ScalatraBase.scala:181)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.scalatra.ScalatraBase.cradleHalt(ScalatraBase.scala:199)
at org.scalatra.ScalatraBase.executeRoutes(ScalatraBase.scala:181)
at org.scalatra.ScalatraBase.executeRoutes$(ScalatraBase.scala:150)
at org.apache.livy.server.JsonServlet.executeRoutes(JsonServlet.scala:39)
at org.scalatra.ScalatraBase.$anonfun$handle$1(ScalatraBase.scala:123)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
at org.scalatra.DynamicScope.withResponse(DynamicScope.scala:79)
at org.scalatra.DynamicScope.withResponse$(DynamicScope.scala:77)
at org.apache.livy.server.JsonServlet.withResponse(JsonServlet.scala:39)
at org.scalatra.DynamicScope.$anonfun$withRequestResponse$1(DynamicScope.scala:59)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
at org.scalatra.DynamicScope.withRequest(DynamicScope.scala:70)
at org.scalatra.DynamicScope.withRequest$(DynamicScope.scala:68)
at org.apache.livy.server.JsonServlet.withRequest(JsonServlet.scala:39)
at org.scalatra.DynamicScope.withRequestResponse(DynamicScope.scala:58)
at org.scalatra.DynamicScope.withRequestResponse$(DynamicScope.scala:56)
at org.apache.livy.server.JsonServlet.withRequestResponse(JsonServlet.scala:39)
at org.scalatra.ScalatraBase.handle(ScalatraBase.scala:123)
at org.scalatra.ScalatraBase.handle$(ScalatraBase.scala:119)
at org.apache.livy.server.JsonServlet.org$scalatra$servlet$ServletBase$$super$handle(JsonServlet.scala:39)
at org.scalatra.servlet.ServletBase.handle(ServletBase.scala:53)
at org.scalatra.servlet.ServletBase.handle$(ServletBase.scala:46)
at org.apache.livy.server.SessionServlet.org$scalatra$Methodoverride$$super$handle(SessionServlet.scala:45)
at org.scalatra.Methodoverride.handle(Methodoverride.scala:35)
...skipping...
21/02/05 14:15:57 INFO LineBufferedStream: java.lang.IllegalArgumentException: Unexpected client ID '40f2d9b7-6460-45b1-9521-6a92ff733766' in SASL handshake.
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.Utils.checkArgument(Utils.java:42)
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.rpc.RpcServer$SaslServerHandler.update(RpcServer.java:288)
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.rpc.SaslHandler.channelRead0(SaslHandler.java:61)
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.rpc.SaslHandler.channelRead0(SaslHandler.java:36)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.handler.codec.BytetoMessageDecoder.fireChannelRead(BytetoMessageDecoder.java:321)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.handler.codec.BytetoMessageDecoder.channelRead(BytetoMessageDecoder.java:295)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.handler.codec.BytetoMessageCodec.channelRead(BytetoMessageCodec.java:103)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
21/02/05 14:15:57 INFO LineBufferedStream: at java.lang.Thread.run(Thread.java:748)
21/02/05 14:15:57 INFO LineBufferedStream: 21/02/05 14:15:57 WARN DefaultChannelPipeline: An exceptionCaught() event was fired,and it reached at the tail of the pipeline. It usually means the last handler in the pipeline did not handle the exception.
21/02/05 14:15:57 INFO LineBufferedStream: java.lang.IllegalArgumentException: Unexpected client ID '40f2d9b7-6460-45b1-9521-6a92ff733766' in SASL handshake.
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.Utils.checkArgument(Utils.java:42)
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.rpc.RpcServer$SaslServerHandler.update(RpcServer.java:288)
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.rpc.SaslHandler.channelRead0(SaslHandler.java:61)
21/02/05 14:15:57 INFO LineBufferedStream: at org.apache.livy.rsc.rpc.SaslHandler.channelRead0(SaslHandler.java:36)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.handler.codec.BytetoMessageDecoder.fireChannelRead(BytetoMessageDecoder.java:321)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.handler.codec.BytetoMessageDecoder.channelRead(BytetoMessageDecoder.java:295)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.handler.codec.BytetoMessageCodec.channelRead(BytetoMessageCodec.java:103)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
21/02/05 14:15:57 INFO LineBufferedStream: at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
21/02/05 14:15:57 INFO LineBufferedStream: at java.lang.Thread.run(Thread.java:748)
21/02/05 14:15:58 WARN ContextLauncher: Child process exited with code 1.
21/02/05 14:15:58 ERROR rejectedExecution: Failed to submit a listener notification task. Event loop shut down?
java.util.concurrent.RejectedExecutionException: event executor terminated
at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:821)
at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:327)
at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:320)
at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:746)
at io.netty.util.concurrent.DefaultPromise.safeExecute(DefaultPromise.java:760)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:428)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122)
at org.apache.livy.rsc.ContextLauncher$ChildProcess.fail(ContextLauncher.java:417)
at org.apache.livy.rsc.ContextLauncher$ChildProcess.access$1100(ContextLauncher.java:369)
at org.apache.livy.rsc.ContextLauncher$ChildProcess$1.run(ContextLauncher.java:400)
at org.apache.livy.rsc.ContextLauncher$ChildProcess$2.run(ContextLauncher.java:451)
at java.lang.Thread.run(Thread.java:748)
同时运行的会话也不多。最多 5 个会话同时运行。工作完成后，工作人员会删除所有较旧的会话，以确保一次打开的会话不会太多。
不好的部分是它在工作人员停止创建新会话后不会自动恢复。仅在 livy 服务器重新启动后才有效。
解决方法
暂无找到可以解决该程序问题的有效方法，小编努力寻找整理中！
如果你已经找到好的解决方法，欢迎将解决方案带上本链接一起发送给小编。
小编邮箱:dio#foxmail.com (将#修改为@）