SPARK报错java.long.Exception……org.codehaus.commons.compiler.CompileException: File 'generated.java'

报错信息：

 java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 61, Column 30: Index expression of type "java.lang.Object" cannot be widened to "int"
/* 001 */ import java.nio.ByteBuffer;
/* 002 */ import java.nio.ByteOrder;
/* 003 */ import scala.collection.Iterator;
/* 004 */ import org.apache.spark.sql.types.DataType;
/* 005 */ import org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder;
/* 006 */ import org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter;
/* 007 */ import org.apache.spark.sql.execution.columnar.MutableUnsafeRow;
/* 008 */
/* 009 */ public SpecificColumnarIterator generate(Object[] references) {
/* 010 */   return new SpecificColumnarIterator();
/* 011 */ }
/* 012 */
/* 013 */ class SpecificColumnarIterator extends org.apache.spark.sql.execution.columnar.ColumnarIterator {
/* 014 */
/* 015 */   private ByteOrder nativeOrder = null;
/* 016 */   private byte[][] buffers = null;
/* 017 */   private UnsafeRow unsafeRow = new UnsafeRow(5);
/* 018 */   private BufferHolder bufferHolder = new BufferHolder(unsafeRow);
/* 019 */   private UnsafeRowWriter rowWriter = new UnsafeRowWriter(bufferHolder, 5);
/* 020 */   private MutableUnsafeRow mutableRow = null;
/* 021 */
/* 022 */   private int currentRow = 0;
/* 023 */   private int numRowsInBatch = 0;
/* 024 */
/* 025 */   private scala.collection.Iterator input = null;
/* 026 */   private DataType[] columnTypes = null;
/* 027 */   private int[] columnIndexes = null;
/* 028 */
/* 029 */   private org.apache.spark.sql.execution.columnar.StringColumnAccessor accessor;
/* 030 */   private org.apache.spark.sql.execution.columnar.StringColumnAccessor accessor1;
/* 031 */   private org.apache.spark.sql.execution.columnar.StringColumnAccessor accessor2;
/* 032 */   private org.apache.spark.sql.execution.columnar.LongColumnAccessor accessor3;
/* 033 */   private org.apache.spark.sql.execution.columnar.LongColumnAccessor accessor4;
/* 034 */
/* 035 */   public SpecificColumnarIterator() {
/* 036 */     this.nativeOrder = ByteOrder.nativeOrder();
/* 037 */     this.buffers = new byte[5][];
/* 038 */     this.mutableRow = new MutableUnsafeRow(rowWriter);
/* 039 */   }
/* 040 */
/* 041 */   public void initialize(Iterator input, DataType[] columnTypes, int[] columnIndexes) {
/* 042 */     this.input = input;
/* 043 */     this.columnTypes = columnTypes;
/* 044 */     this.columnIndexes = columnIndexes;
/* 045 */   }
/* 046 */
/* 047 */
/* 048 */
/* 049 */   public boolean hasNext() {
/* 050 */     if (currentRow < numRowsInBatch) {
/* 051 */       return true;
/* 052 */     }
/* 053 */     if (!input.hasNext()) {
/* 054 */       return false;
/* 055 */     }
/* 056 */
/* 057 */     org.apache.spark.sql.execution.columnar.CachedBatch batch = (org.apache.spark.sql.execution.columnar.CachedBatch) input.next();
/* 058 */     currentRow = 0;
/* 059 */     numRowsInBatch = batch.numRows();
/* 060 */     for (int i = 0; i < columnIndexes.length; i ++) {
/* 061 */       buffers[i] = batch.buffers()[columnIndexes[i]];
/* 062 */     }
/* 063 */     accessor = new org.apache.spark.sql.execution.columnar.StringColumnAccessor(ByteBuffer.wrap(buffers[0]).order(nativeOrder));
/* 064 */     accessor1 = new org.apache.spark.sql.execution.columnar.StringColumnAccessor(ByteBuffer.wrap(buffers[1]).order(nativeOrder));
/* 065 */     accessor2 = new org.apache.spark.sql.execution.columnar.StringColumnAccessor(ByteBuffer.wrap(buffers[2]).order(nativeOrder));
/* 066 */     accessor3 = new org.apache.spark.sql.execution.columnar.LongColumnAccessor(ByteBuffer.wrap(buffers[3]).order(nativeOrder));
/* 067 */     accessor4 = new org.apache.spark.sql.execution.columnar.LongColumnAccessor(ByteBuffer.wrap(buffers[4]).order(nativeOrder));
/* 068 */
/* 069 */     return hasNext();
/* 070 */   }
/* 071 */
/* 072 */   public InternalRow next() {
/* 073 */     currentRow += 1;
/* 074 */     bufferHolder.reset();
/* 075 */     rowWriter.zeroOutNullBytes();
/* 076 */     accessor.extractTo(mutableRow, 0);
/* 077 */     accessor1.extractTo(mutableRow, 1);
/* 078 */     accessor2.extractTo(mutableRow, 2);
/* 079 */     accessor3.extractTo(mutableRow, 3);
/* 080 */     accessor4.extractTo(mutableRow, 4);
/* 081 */     unsafeRow.setTotalSize(bufferHolder.totalSize());
/* 082 */     return unsafeRow;
/* 083 */   }
/* 084 */ }


        at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
        at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
        at org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
        at org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
        at org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
        at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
        at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
        at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
        at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
        at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
        at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
        at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:905)
        at org.apache.spark.sql.execution.columnar.GenerateColumnAccessor$.create(GenerateColumnAccessor.scala:233)
        at org.apache.spark.sql.execution.columnar.GenerateColumnAccessor$.create(GenerateColumnAccessor.scala:65)
        at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:889)
        at org.apache.spark.sql.execution.columnar.InMemoryTableScanExec$$anonfun$doExecute$1.apply(InMemoryTableScanExec.scala:180)
        at org.apache.spark.sql.execution.columnar.InMemoryTableScanExec$$anonfun$doExecute$1.apply(InMemoryTableScanExec.scala:135)
        at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndexInternal$1$$anonfun$apply$24.apply(RDD.scala:815)
        at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndexInternal$1$$anonfun$apply$24.apply(RDD.scala:815)
        at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
        at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
        at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
        at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
        at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
        at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
        at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
        at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
        at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
        at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
        at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
        at org.apache.spark.scheduler.Task.run(Task.scala:99)
        at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

        at java.lang.Thread.run(Thread.java:748)

 

原因：在转换类型时 有数据NULL值存在

 

解决办法：在强制转换类型时 先过滤掉 转换字段的NULL值

 

第二种 也会报这个错，就是spark 操作hive表数据

比如：hive表字段a类型为bigint , 你在spark程序 写了 a is not null 

向上述情况也会出现这个错误

 

时隔仨月，这个问题又来了，这次直接写了udf函数  ok

460-00-343545-123      sql是切分 但是数据里总有不规范的数据所以就用了udf函数  异常解决

session.udf.register("toCgi", (ne_name: String) => {
  var ecgi = ""
  try {
    val arr = ne_name.split("-", -1)

    val str = arr(2).toLong * 256 + arr(3).toLong
    ecgi = str.toString

  } catch {
    case e:Exception => ecgi = "0"
  }
  ecgi
})