【喜加一】parquet文件读写（在HDFS上）

parquet文件格式详解：https://www.infoq.cn/article/in-depth-analysis-of-parquet-column-storage-format

⬆️这个链接主要描述了parquet的原理。相比于应用，它有点过于底层了。用Java读写parquet文件需要的知识是它的子集，有部分压缩算法在轮子中被实现了，使用者只要调用就好。

---

Java读写HDFS上的parquet文件：

首先，

start-dfs.sh

然后，编写如下Java代码：

package cn.edu.nju.zyf.parquetDemo01;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.example.GroupReadSupport;
import org.apache.parquet.hadoop.example.GroupWriteSupport;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.schema.*;

import java.io.IOException;


/**
 * @author zhuyuanfu
 * @version 2018-01-03
 * @description just a simple demo for writing and reading parquet files.
 */
public class ParquetWriteAndReadDemo {

    //这是一种parse String来制造MessageType的办法
    private static Logger logger = Logger.getLogger(ParquetWriteAndReadDemo.class);
    private static String schemaString = "message schema {" + "optional int64 log_id;"
            + "optional binary idc_id;" + "optional int64 house_id;"
            + "optional int64 src_ip_long;" + "optional int64 dest_ip_long;"
            + "optional int64 src_port;" + "optional int64 dest_port;"
            + "optional int32 protocol_type;" + "optional binary url64;"
            + "optional binary access_time;}";
    private static MessageType schema = MessageTypeParser.parseMessageType(schemaString);

    private static MessageType getMessageTypeFromCode (){
        MessageType messageType = Types.buildMessage()
                        .required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("id")
                        .required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("name")
                        .required(PrimitiveType.PrimitiveTypeName.INT32).named("age")
                        .requiredGroup()
                        .required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("test1")
                        .required(PrimitiveType.PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("test2")
                        .named("group1")
                        .named("trigger");
        //System.out.println(messageType.toString());
        return messageType;
    }

    private static void writeParquetOnDisk(String fileName){

        //1、声明parquet的messageType
        MessageType messageType = getMessageTypeFromCode();

        //2、声明parquetWriter
        Path path = new Path("/tmp/"+fileName);
        System.out.println(path);
        Configuration configuration = new Configuration();
        GroupWriteSupport.setSchema(messageType,configuration);
        GroupWriteSupport writeSupport = new GroupWriteSupport();

        //3、写数据
        ParquetWriter<Group> writer = null;
        try{
            writer = new ParquetWriter<Group>(path,
                    ParquetFileWriter.Mode.OVERWRITE,
                    writeSupport,
                    CompressionCodecName.UNCOMPRESSED,
                    128*1024*1024,
                    5*1024*1024,
                    5*1024*1024,
                    ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED,
                    ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED,
                    ParquetWriter.DEFAULT_WRITER_VERSION,
                    configuration);

            //4、构建parquet数据，封装成Group。
            for(int i = 0; i < 10; i ++){
                Group group = new SimpleGroupFactory(messageType).newGroup();
                group.append("name",i+"@ximalaya.com")
                        .append("id",i+"@id")
                        .append("age",i)