获得parquet文件的schema 合并parquet小文件

获得parquet文件的schema

import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.hadoop.util.HiddenFileFilter;
import org.apache.parquet.schema.MessageType;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;

public class ParquetSchemaPrint {

    public static void main(String[] args) throws Exception{
        String input = args[0];

        Configuration conf = new Configuration();
        ParquetMetadata metaData;

        Path path = new Path(input);
        FileSystem fs = path.getFileSystem(conf);
        Path file;
        if (fs.isDirectory(path)) {
            FileStatus[] statuses = fs.listStatus(path, HiddenFileFilter.INSTANCE);
            if (statuses.length == 0) {
                throw new RuntimeException("Directory " + path.toString() + " is empty");
            }
            file = statuses[0].getPath();
        } else {
            file = path;
        }
        metaData = ParquetFileReader.readFooter(conf, file, NO_FILTER);
        MessageType schema = metaData.getFileMetaData().getSchema();

        System.out.println(schema);
    }
}

输出内容如下：

message hive_schema {
  optional binary code (UTF8);
  optional int32 day;
  optional double kp;
  optional double zg;
  optional double zd;
  optional double sp;
  optional double cjl;
  optional double cje;
}

合并parquet文件

合并文件的前提是知道文件的schema,之前的代码中已经获得了parquet文件的schema，具体合并代码如下

/**
     * @param src　parquet文件路径
     * @param fileBatch　多少个文件合并成一个
     * @throws Exception
     */
    public static void combineParquet(String src,int fileBatch)throws Exception{

        String schemastr = "message hive_schema {\n" +
                "  optional binary code (UTF8);\n" +
                "  optional int32 day;\n" +
                "  optional double kp;\n" +
                "  optional double zg;\n" +
                "  optional double zd;\n" +
                "  optional double sp;\n" +
                "  optional double cjl;\n" +
                "  optional double cje;\n" +
                "}\n" +
                "\n";
        Configuration configuration = new Configuration();
        MessageType schema = MessageTypeParser.parseMessageType(schemastr);
        List<Path> fileList = getPaths(configuration, src);
        String outPath = src+"/combine"+System.currentTimeMillis()+"_0_"+".parquet";

        Path path = new Path(outPath);
        GroupWriteSupport writeSupport = new GroupWriteSupport();
        writeSupport.setSchema(schema,configuration);
        ParquetWriter<Group> writer = new ParquetWriter<Group>(path,configuration,writeSupport);


        for(int i=0;i<fileList.size();i++){

            if(i>0&&i%fileBatch==0){
                writer.close();
                outPath = src+"/combine"+System.currentTimeMillis()+"_"+i+"_"+".parquet";
                path = new Path(outPath);
                writer = new ParquetWriter<Group>(path,configuration,writeSupport);
            }
            System.out.println("正在合并文件:"+fileList.get(i));
            GroupReadSupport readSupport = new GroupReadSupport();
            ParquetReader.Builder<Group> reader = ParquetReader.builder(readSupport, fileList.get(i));
            ParquetReader<Group> build = reader.build();
            Group line = null;
            while ((line = build.read()) != null) {
                writer.write(line);
            }
        }

        writer.close();
        System.out.println("读取结束");

    }

    /**
     * 获得目录下的所有文件
     * @param conf
     * @param src
     * @return
     * @throws Exception
     */
    public static List<Path> getPaths(Configuration conf,String src)throws Exception{
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> locatedFileStatusRemoteIterator = fs.listFiles(new Path(src), true);
        List<Path> fileList = new ArrayList<Path>();
        while(locatedFileStatusRemoteIterator.hasNext()){
            LocatedFileStatus next = locatedFileStatusRemoteIterator.next();
            Path path = next.getPath();
            fileList.add(path);
        }

        return fileList;
    }