【spark exception】org.apache.spark.SparkException java.lang.ArrayIndexOutOfBoundsException 造成数组越界

【spark exception】org.apache.spark.SparkException  java.lang.ArrayIndexOutOfBoundsException 造成数组越界

当执行以下spark-sql是时候

insert overwrite table hive_user_income_detail_daily partition (pday='20190620',is_data_return='1')

select rr.user_id,rr.src,rr.app_id,rr.type,rr.create_time,rr.product_code

from tmp_hive_user_income_table rr

原因分析及解决办法：

insert overwrite table的 hive_user_income_detail_daily 这个表的建表字段数目必须和tmp_hive_user_income_table的字段数目保持一致

经过确认，hive_user_income_detail_daily有7个字段（除两个分区字段外），而select 语句中只给出了6个字段，所以将缺少的那个字段手动添加上默认值就可以正常运行啦

insert overwrite table hive_user_income_detail_daily partition (pday='20190620',is_data_return='1')

select rr.user_id,rr.src,rr.app_id,rr.type,rr.create_time,rr.product_code,'none'

from tmp_hive_user_income_table rr

详细报错信息如下：

19/06/28 16:03:14 ERROR LzoCodec: Failed to load/initialize native-lzo library
[Stage 19:======>                                              (52 + 121) / 401]19/06/28 16:04:31 ERROR TaskSetManager: Task 53 in stage 19.0 failed 4 times; aborting job
19/06/28 16:04:31 ERROR SparkHiveShell: Failed: Error 
org.apache.spark.SparkException: Job aborted due to stage failure: Task 53 in stage 19.0 failed 4 times, most recent failure: Lost task 53.3 in stage 19.0 (TID 3424, 10.160.147.93): java.lang.ArrayIndexOutOfBoundsException: 11
    at org.apache.spark.sql.catalyst.expressions.GenericMutableRow.genericGet(rows.scala:254)
    at org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow$class.getAs(rows.scala:35)
    at org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow$class.isNullAt(rows.scala:36)
    at org.apache.spark.sql.catalyst.expressions.GenericMutableRow.isNullAt(rows.scala:248)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$writeToFile$1$1.apply(SaveAsHiveFile.scala:113)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$writeToFile$1$1.apply(SaveAsHiveFile.scala:110)
    at scala.collection.Iterator$class.foreach(Iterator.scala:727)
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
    at org.apache.spark.sql.hive.SaveAsHiveFile$class.writeToFile$1(SaveAsHiveFile.scala:110)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$saveAsHiveFile$5.apply(SaveAsHiveFile.scala:91)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$saveAsHiveFile$5.apply(SaveAsHiveFile.scala:91)
    at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
    at org.apache.spark.scheduler.Task.run(Task.scala:89)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)
Driver stacktrace:
    at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1475)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1463)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1462)
    at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
    at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
    at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1462)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:843)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:843)
    at scala.Option.foreach(Option.scala:236)
    at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:843)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1684)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1643)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1632)
    at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
    at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:664)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1844)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1858)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1933)
    at org.apache.spark.sql.hive.SaveAsHiveFile$class.saveAsHiveFile(SaveAsHiveFile.scala:91)
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.saveAsHiveFile(InsertIntoHiveTable.scala:50)
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult$lzycompute(InsertIntoHiveTable.scala:204)
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult(InsertIntoHiveTable.scala:129)
    at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.doExecute(InsertIntoHiveTable.scala:294)
    at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:135)
    at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:133)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
    at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:133)
    at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:55)
    at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55)
    at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:145)
    at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:130)
    at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:52)
    at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:817)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$$anonfun$org$apache$spark$sql$hive$xitong$shell$SparkHiveShell$$run$1$1.apply(SparkHiveShell.scala:172)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$$anonfun$org$apache$spark$sql$hive$xitong$shell$SparkHiveShell$$run$1$1.apply(SparkHiveShell.scala:108)
    at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
    at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$.org$apache$spark$sql$hive$xitong$shell$SparkHiveShell$$run$1(SparkHiveShell.scala:108)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$.process$1(SparkHiveShell.scala:212)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$.org$apache$spark$sql$hive$xitong$shell$SparkHiveShell$$loop$1(SparkHiveShell.scala:250)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$$anonfun$main$2.apply(SparkHiveShell.scala:58)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$$anonfun$main$2.apply(SparkHiveShell.scala:42)
    at scala.Option.map(Option.scala:145)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell$.main(SparkHiveShell.scala:42)
    at org.apache.spark.sql.hive.xitong.shell.SparkHiveShell.main(SparkHiveShell.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:601)
    at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:733)
    at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
    at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
    at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.lang.ArrayIndexOutOfBoundsException: 11
    at org.apache.spark.sql.catalyst.expressions.GenericMutableRow.genericGet(rows.scala:254)
    at org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow$class.getAs(rows.scala:35)
    at org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow$class.isNullAt(rows.scala:36)
    at org.apache.spark.sql.catalyst.expressions.GenericMutableRow.isNullAt(rows.scala:248)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$writeToFile$1$1.apply(SaveAsHiveFile.scala:113)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$writeToFile$1$1.apply(SaveAsHiveFile.scala:110)
    at scala.collection.Iterator$class.foreach(Iterator.scala:727)
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
    at org.apache.spark.sql.hive.SaveAsHiveFile$class.writeToFile$1(SaveAsHiveFile.scala:110)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$saveAsHiveFile$5.apply(SaveAsHiveFile.scala:91)
    at org.apache.spark.sql.hive.SaveAsHiveFile$$anonfun$saveAsHiveFile$5.apply(SaveAsHiveFile.scala:91)
    at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
    at org.apache.spark.scheduler.Task.run(Task.scala:89)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:724)
[Stage 19:======>                                              (52 + 122) / 401]Moved to trash: /home/spark/cache/hdp-jinke-data/.sparkStaging/application_1556532226104_2105277
19/06/28 16:04:34 ERROR YarnClientSchedulerBackend: Yarn application has already exited with state FINISHED!