1.
java.lang.UnsupportedOperationException: org.apache.parquet.column.values.dictionary.DictionaryValuesWriter$PlainFixedLenArrayDictionaryValuesWriter
at org.apache.parquet.column.values.ValuesWriter.writeLong(ValuesWriter.java:109)
at org.apache.parquet.column.values.fallback.FallbackValuesWriter.writeLong(FallbackValuesWriter.java:174)
at org.apache.parquet.column.impl.ColumnWriterV1.write(ColumnWriterV1.java:233)
at org.apache.parquet.io.MessageColumnIO$MessageColumnIORecordConsumer.addLong(MessageColumnIO.java:327)
at org.apache.parquet.example.data.simple.LongValue.writeValue(LongValue.java:43)
at org.apache.parquet.example.data.simple.SimpleGroup.writeValue(SimpleGroup.java:229)
at org.apache.parquet.example.data.GroupWriter.writeGroup(GroupWriter.java:51)
at org.apache.parquet.example.data.GroupWriter.write(GroupWriter.java:37)
at org.apache.parquet.hadoop.example.GroupWriteSupport.write(GroupWriteSupport.java:64)
at org.apache.parquet.hadoop.example.GroupWriteSupport.write(GroupWriteSupport.java:36)
at org.apache.parquet.hadoop.InternalParquetRecordWriter.write(InternalParquetRecordWriter.java:121)
at org.apache.parquet.hadoop.ParquetWriter.write(ParquetWriter.java:288)
at com.zbj.finance.HDFSMonitor.WriteParquet.lambda$parquetWriter$0(WriteParquet.java:83)
at java.util.HashMap.forEach(HashMap.java:1289)
at com.zbj.finance.HDFSMonitor.WriteParquet.parquetWriter(WriteParquet.java:81)
at com.zbj.finance.HDFSMonitor.WriteParquet.main(WriteParquet.java:31)
原因:根据如上报错的方法名称,at org.apache.parquet.example.data.simple.LongValue.writeValue(LongValue.java:43)可以推断应该是在写一个long字段的值时报错。代码中的schemal信息如下:
MessageType schema = MessageTypeParser.parseMessageType("message hdfs_info {\n" + " required binary serial_no (UTF8);\n" + //(UTF8) " required binary name (UTF8);\n" + //(UTF8) " required int64 size ;\n" + //(UTF8) " required int32 type ;\n" + //(UTF8) " required binary user (UTF8);\n" + //(UTF8) " required binary usergroup (UTF8);\n" + //(UTF8) " required int32 create_time ;" + " required int32 insertDB_time ;" + "}\n" + "}");
其中size、create_time, insertDB_time 这三字段,java代码中是long类型,如此可推断问题在这三字段上。最终确认是create_time,insertDB_time的问题,java是long,对应schema中应该是int64,而不是32,修改后,解决问题。