parquet文件格式详解:https://www.infoq.cn/article/in-depth-analysis-of-parquet-column-storage-format
⬆️这个链接主要描述了parquet的原理。相比于应用,它有点过于底层了。用Java读写parquet文件需要的知识是它的子集,有部分压缩算法在轮子中被实现了,使用者只要调用就好。
Java读写HDFS上的parquet文件:
首先,
start-dfs.sh
然后,编写如下Java代码:
package cn.edu.nju.zyf.parquetDemo01;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.example.GroupReadSupport;
import org.apache.parquet.hadoop.example.GroupWriteSupport;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.schema.*;
import java.io.IOException;
/**
* @author zhuyuanfu
* @version 2018-01-03
* @description just a simple demo for writing and reading parquet files.
*/
public class ParquetWriteAndReadDemo {
//这是一种parse String来制造MessageType的办法
private static Logger logger = Logger.getLogger(ParquetWriteAndReadDemo.class);
private static String schemaString = "message schema {" + "optional int64 log_id;"
+ "optional binary idc_id;" + "optional int64 house_id;"
+ "optional int64 src_ip_long;" + "optional int64 dest_ip_long;"
+ "optional int64 src_port;" + "optional int64 dest_port;"
+ "optional int32 protocol_type;" + "optional binary url64;"
+ "optional binary access_time;}";
private static MessageType schema = MessageTypeParser.parseMessageType(schemaString);
private static MessageType getMessageTypeFromCode (){
MessageType messageType = Types.buildMessage()
.required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("id")
.required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("name")
.required(PrimitiveType.PrimitiveTypeName.INT32).named("age")
.requiredGroup()
.required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("test1")
.required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("test2")
.named("group1")
.named("trigger");
//System.out.println(messageType.toString());
return messageType;
}
private static void writeParquetOnDisk(String fileName){
//1、声明parquet的messageType
MessageType messageType = getMessageTypeFromCode();
//2、声明parquetWriter
Path path = new Path("/tmp/"+fileName);
System.out.println(path);
Configuration configuration = new Configuration();
GroupWriteSupport.setSchema(messageType,configuration);
GroupWriteSupport writeSupport = new GroupWriteSupport();
//3、写数据
ParquetWriter<Group> writer = null;
try{
writer = new ParquetWriter<Group>(path,
ParquetFileWriter.Mode.OVERWRITE,
writeSupport,
CompressionCodecName.UNCOMPRESSED,
128*1024*1024,
5*1024*1024,
5*1024*1024,
ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED,
ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED,
ParquetWriter.DEFAULT_WRITER_VERSION,
configuration);
//4、构建parquet数据,封装成Group。
for(int i = 0; i < 10; i ++){
Group group = new SimpleGroupFactory(messageType).newGroup();
group.append("name",i+"@ximalaya.com")
.append("id",i+"@id")
.append("age",i)