spark任务shell运行_java调用shell脚本执行spark任务错误

最新推荐文章于 2021-03-03 18:52:21 发布

欧阳紫琦

最新推荐文章于 2021-03-03 18:52:21 发布

阅读量559

点赞数

文章标签： spark任务shell运行

本文链接：https://blog.csdn.net/weixin_42405503/article/details/113541713

版权

java调用shell脚本执行spark任务

使用java通过 Runtime.getRuntime().exec(); 调用shell 脚本，在脚本中执行 spark 任务会报错，

如下：

19/05/15 15:46:47 WARN StandaloneAppClient$ClientEndpoint: Failed to connect to master 192.168.9.2:7077

org.apache.spark.SparkException: Exception thrown in awaitResult:

at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:205)

at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75)

at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:100)

at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:108)

at org.apache.spark.deploy.client.StandaloneAppClient$ClientEndpoint$$anonfun$tryRegisterAllMasters$1$$anon$1.run(StandaloneAppClient.scala:106)

at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)

at java.util.concurrent.FutureTask.run(FutureTask.java:266)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

at java.lang.Thread.run(Thread.java:748)

Caused by: java.lang.RuntimeException: java.io.StreamCorruptedException: invalid stream header: 01000B31

at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:857)

at java.io.ObjectInputStream.(ObjectInputStream.java:349)

at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.(JavaSerializer.scala:63)

at org.apache.spark.serializer.JavaDeserializationStream.(JavaSerializer.scala:63)

at org.apache.spark.serializer.JavaSerializerInstance.deserializeStream(JavaSerializer.scala:122)

at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:107)

at org.apache.spark.rpc.netty.NettyRpcEnv$$anonfun$deserialize$1$$anonfun$apply$1.apply(NettyRpcEnv.scala:259)

at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)

at org.apache.spark.rpc.netty.NettyRpcEnv.deserialize(NettyRpcEnv.scala:308)

at org.apache.spark.rpc.netty.NettyRpcEnv$$anonfun$deserialize$1.apply(NettyRpcEnv.scala:258)

at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)

at org.apache.spark.rpc.netty.NettyRpcEnv.deserialize(NettyRpcEnv.scala:257)

at org.apache.spark.rpc.netty.NettyRpcHandler.internalReceive(NettyRpcEnv.scala:577)

at org.apache.spark.rpc.netty.NettyRpcHandler.receive(NettyRpcEnv.scala:562)

at org.apache.spark.network.server.TransportRequestHandler.processRpcRequest(TransportRequestHandler.java:159)

at org.apache.spark.network.server.TransportRequestHandler.handle(TransportRequestHandler.java:107)

at org.apache.spark.network.server.TransportChannelHandler.channelRead(TransportChannelHandler.java:118)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:367)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:353)

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:346)

at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:266)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:367)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:353)

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:346)

at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:367)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:353)

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:346)

at org.apache.spark.network.util.TransportFrameDecoder.channelRead(TransportFrameDecoder.java:85)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:367)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:353)

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:346)

at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1294)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:367)

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:353)

at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:911)

at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131)

at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:652)

at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:575)

at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:489)

at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:451)

at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:140)

at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)

at java.lang.Thread.run(Thread.java:748)

at org.apache.spark.network.client.TransportResponseHandler.handle(TransportResponseHandler.java:207)

at org.apache.spark.network.server.TransportChannelHandler.channelRead(TransportChannelHandler.java:120)