创建sequencefile格式hive表
create table test_seqencefile(name string,age int) stored as SEQUENCEFILE
CREATE TABLE `test_seqencefile`(
`name` string,
`age` int)
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.SequenceFileInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat'
LOCATION
'hdfs://localhost:9000/user/work/warehouse/test_seqencefile'
TBLPROPERTIES (
'transient_lastDdlTime'='1502950306')
插入测试数据
insert into table test_seqencefile select name ,age from test limit 10;
执行hive插入语句的MR解析
在执行插入语句时的MR任务的configrration(从jobhistory中得到)
Mapper:org.apache.hadoop.hive.ql.exec.mr.ExecMapper
Reducer:org.apache.hadoop.hive.ql.exec.mr.ExecReducer
mapreduce.job.output.key.class:org.apache.hadoop.io.Text
mapreduce.job.output.value.class:org.apache.hadoop.io.Text
file.outputformat:org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
从hive表结构可以知道test_seqencefile序列化用的是org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,由此可知字段分割用的是’\1’,文件输出格式是由org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat控制,查看代码知道
final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc, fs, finalOutPath,
BytesWritable.class, valueClass, isCompressed, progress);
执行MR时,Key:BytesWritable,Value:Text,这里的Key的类型和jobhistory中的不一样(mapreduce.job.output.key.class:org.apache.hadoop.io.Text),这点也是很奇怪,这点很重要,在用MR解析sequencefile需要指定Key和Value的类型
MR读取hive表中的sequencefile文件
package com.fan.hadoop.seqencefile;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import java.io.IOException;
public class SeqenceFileReadMR {
public static class SequenceFileMaper extends Mapper<BytesWritable,Text,Text,Text>{
@Override
protected void map(BytesWritable key, Text value, Context context)
throws IOException, InterruptedException {
String t = value.toString();
String[] arr = t.split("\1");
//hive默认情况下使用'\1'来分割字段
context.write(new Text(arr[0]),new Text(arr[1]));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(SeqenceFileReadMR.class);
job.setJobName("seqencefileread");
String in = "hdfs://localhost:9000/user/work/warehouse/test_seqencefile";
String out = "hdfs://localhost:9000/test/seqencefile1";
job.setMapperClass(SequenceFileMaper.class);
SequenceFileInputFormat.addInputPath(job, new Path(in));
job.setInputFormatClass(SequenceFileInputFormat.class);
//如果是在不知道key和value的类型,就用SequenceFileAsTextInputFormat,那Map的key和value的
//类型就可已都指定为Text
// job.setInputFormatClass(SequenceFileAsTextInputFormat.class);
job.setNumReduceTasks(0);
job.setMapOutputValueClass(Text.class);
job.setMapOutputKeyClass(Text.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileOutputFormat.setOutputPath(job, new Path(out));
job.waitForCompletion(true);
}
}
查看生成的文件
hadoop dfs -cat /test/seqencefile/part-m-00000
kafka 14
tensflow 98
hadoop 34
hbase 68
flume 57
kafka 99
kafka 28
flume 24
tensflow 35
flume 44
生成sequencefile文件
生成sequencefile文件,用SequenceFileOutputFormat,应为hive表中默认使用’\1’分割数据行,所以字段的值用’\1’连接,具体代码如下
package com.fan.hadoop.seqencefile;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import java.io.IOException;
import java.util.Random;
import java.util.StringTokenizer;
public class SeqenceFileWriterMR {
public static class SequenceFileMaper extends Mapper<LongWritable,Text,LongWritable,Text>{
private Text v = new Text();
private Random random = new Random();
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer token = new StringTokenizer(line);
while (token.hasMoreTokens()) {
String s = token.nextToken();
v.set(s+"\1"+random.nextInt(20));
context.write(key, v);
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(SeqenceFileWriterMR.class);
job.setJobName("seqencefileWriter");
String in = "hdfs://localhost:9000/test/wordcount.txt";
String out = "hdfs://localhost:9000/test/seqencefile2";
job.setMapperClass(SequenceFileMaper.class);
FileInputFormat.addInputPath(job, new Path(in));
job.setInputFormatClass(TextInputFormat.class);
job.setNumReduceTasks(0);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
SequenceFileOutputFormat.setOutputPath(job, new Path(out));
job.waitForCompletion(true);
}
}
将数据复制到hive的目录下
hadoop dfs -cp /test/seqencefile2/* /user/work/warehouse/test_seqencefile/
查询hive表
hive> select * from test_seqencefile limit 5;
OK
hello 18
world 6
hello 12
presto 2
hello 14