博客地址:http://www.fanlegefan.com
文章地址:http://www.fanlegefan.com/archives/parquet-thrfit-hive/
摘要
本文将用MapReduce方式读写parquet-thrfit格式文件,并将数据导入到hive表中进行查询操
定义person.thrift文件
namespace java com.fan.hadoop.parquet.thrift
struct Person{
1:optional string name;
2:optional i32 age;
}
生成java bean
thrift --gen java person.thrift
生成路径如下
├── gen-java
│ └── com
│ └── fan
│ └── hadoop
│ └── parquet
│ └── thrift
│ └── Person.java
└── person.thrift
将Person.java复制到项目中去
用MR写parquet-thrift格式文件
package com.fan.hadoop.parquet.thrift;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.thrift.ParquetThriftOutputFormat;
import java.io.IOException;
/**
* Created by http://www.fanlegefan.com on 17-7-21.
*/
public class ParquetThriftWriterMR {
public static class ParthriftMap extends
Mapper<LongWritable, Text, Void, Person> {
private Text word = new Text();
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String[] arr = value.toString().split(",");
Person p = new Person();
p.setName(arr[0]);
p.setAge(Integer.valueOf(arr[1].trim()));
context.write(null,p);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(ParquetThriftWriterMR.class);
job.setJobName("parquetthrfit");
String in = "hdfs://localhost:9000/test/user.txt";
String out = "hdfs://localhost:9000/test/wdthrfit";
job.setMapperClass(ParthriftMap.class);
TextInputFormat.addInputPath(job, new Path(in));
job.setInputFormatClass(TextInputFormat.class);
job.setNumReduceTasks(0);
job.setOutputFormatClass(ParquetThriftOutputFormat.class);
ParquetThriftOutputFormat.setCompression(job, CompressionCodecName.GZIP);
ParquetThriftOutputFormat.setOutputPath(job, new Path(out));
ParquetThriftOutputFormat.setThriftClass(job, Person.class);
job.waitForCompletion(true);
}
}
生成的文件如下
hadoop dfs -ls /test/wdthrfit
Found 4 items
-rw-r--r-- 3 work supergroup 0 2017-07-21 11:51 /test/wdthrfit/_SUCCESS
-rw-r--r-- 3 work supergroup 529 2017-07-21 11:51 /test/wdthrfit/_common_metadata
-rw-r--r-- 3 work supergroup 680 2017-07-21 11:51 /test/wdthrfit/_metadata
-rw-r--r-- 3 work supergroup 776 2017-07-21 11:51 /test/wdthrfit/part-m-00000.gz.parquet
读取parquet-thrift格式文件
package com.fan.hadoop.parquet.thrift;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.parquet.hadoop.thrift.ParquetThriftInputFormat;
import java.io.IOException;
/**
* Created by http://www.fanlegefan.com on 17-7-21.
*/
public class ParquetThriftReaderMR {
public static class ParthriftMap extends
Mapper<Void, Person, LongWritable, Text> {
private Text p = new Text();
public void map(Void key, Person value, Context context)
throws IOException, InterruptedException {
p.set(value.getName()+","+value.getAge());
context.write(null, p);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(ParquetThriftReaderMR.class);
job.setJobName("parquetthrfit");
String in = "hdfs://localhost:9000/test/wdthrfit";
String out = "hdfs://localhost:9000/test/wdthriftread";
job.setMapperClass(ParthriftMap.class);
job.setInputFormatClass(ParquetThriftInputFormat.class);
ParquetThriftInputFormat.setInputPaths(job, new Path(in));
job.setNumReduceTasks(0);
job.setOutputFormatClass(TextOutputFormat.class);
TextOutputFormat.setOutputPath(job, new Path(out));
job.waitForCompletion(true);
}
}
查看生成的文本文件内容
hadoop dfs -cat /test/wdthriftread/part-m-00000
leo,20
jack,10
jim,30
andy,30
用hive表读取parquet-thrift格式文件
创建parquet格式表
create table test_parquet(name string,age int) stored as parquet
查看表结构
hive> show create table test_parquet;
OK
CREATE TABLE `test_parquet`(
`name` string,
`age` int)
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION
'hdfs://localhost:9000/user/hive/warehouse/test_parquet'
TBLPROPERTIES (
'transient_lastDdlTime'='1495038003')
将之间生成的文件导入到hive表的数据目录下
hadoop dfs -cp /test/wdthrfit/* /user/hive/warehouse/test_parquet/
hive读取数据
hive> select name,age from test_parquet;
OK
leo 20
jack 10
jim 30
andy 30
Time taken: 0.049 seconds, Fetched: 4 row(s)