目标
实现将
HDFS
中的数据写入到
Hbase
表中。
分步实现
1.构建 FruitMapper 类,读取 HDFS 中fruit表的数据
package com.chris.hbase.mr1;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class FruitMapper extends Mapper<LongWritable, Text,LongWritable,Text> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
context.write(key,value);
}
}
2. 构建 FruitReducer 类,用于将读取到的 fruit 表中的数据写入到 fruit1 表中
package com.chris.hbase.mr1;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class FruitReducer extends TableReducer<LongWritable, Text, NullWritable> {
// private String cf1 = null;
//
// @Override
// protected void setup(Reducer<LongWritable, Text, NullWritable, Mutation>.Context context) throws IOException, InterruptedException {
//
// Configuration configuration = context.getConfiguration();
// cf1 = configuration.get("cf1");
// }
@Override
protected void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
// 1.遍历values
for (Text value : values) {
// 2.获取每一行数据
String[] fields = value.toString().split("\t");
// 3.构造put对象 (传入rowKey)
Put put = new Put(Bytes.toBytes(fields[0]));
// 4.给put对象赋值
put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),Bytes.toBytes(fields[1]));
put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("color"),Bytes.toBytes(fields[2]));
// 5.写出
context.write(NullWritable.get(),put);
}
}
}
3. 构建 驱动类Driver
package com.chris.hbase.mr1;
import com.sun.tracing.dtrace.ArgsAttributes;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class FruitDriver implements Tool {
// 定义一个configuration
private Configuration configuration = null;
@Override
public int run(String[] args) throws Exception {
// 1. 获取job对象
Job job = Job.getInstance(configuration);
// 2. 设置驱动类路径
job.setJarByClass(FruitDriver.class);
// 3.设置Mapper输出的KV类型
job.setMapperClass(FruitMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
// 4.设置Reducer类
TableMapReduceUtil.initTableReducerJob(args[1],
FruitReducer.class,job);
// 5.设置输入参数
FileInputFormat.setInputPaths(job,new Path(args[0]));
// 6.提交任务
boolean result = job.waitForCompletion(true);
return result?0:1;
}
@Override
public void setConf(Configuration conf) {
configuration = conf;
}
@Override
public Configuration getConf() {
return configuration;
}
public static void main(String[] args) {
try {
Configuration configuration = new Configuration();
int run = ToolRunner.run(configuration, new FruitDriver(), args);
System.exit(run);
} catch (Exception e) {
e.printStackTrace();
}
}
}
4. 在hbase端口创建fruit1表:
5. 打jar包,将jar包拖到hbase目录下,运行以下命令:
# com.chris...是主类名,/fruit.tsv是HDFS上的表,fruit1是hbase中要传入数据的表
yarn jar hbase_demo-1.0-SNAPSHOT.jar com.chris.hbase.mr1.FruitDriver /fruit.tsv fruit1