案例 一
1. 需求
从 Person 表中读取name列的数据 写入到 Person_mr
2.代码
Map 阶段
package com.wangt.habse.mr01;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
/**
* 从 Person 表中读取数据 写到 Person_mr 表中
*
* @author 王天赐
* @create 2019-08-02 20:08
*/
public class PersonMapper extends TableMapper<ImmutableBytesWritable, Put> {
// 输出类型是 ImmutableBytesWritable(序列化的Byte 数组) 和 Put
/**
* 从 Hbase 的 Person 表中读取数据
*
* @param key rowkey
* @param value 从 Hbase 表中读取数据的 Result 对象
* @param context
* @throws IOException
* @throws InterruptedException
*/
@Override
protected void map(ImmutableBytesWritable key, Result value, Context context)
throws IOException, InterruptedException {
// 构建 Put 对象
// 注意 : 这个key是序列化的 rowkey
Put put = new Put(key.get());
System.out.println(Bytes.toString(key.get()));
// 遍历数组
Cell[] cells = value.rawCells();
for (Cell cell : cells) {
// 获取 HBase 的列
String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
// 列族
String family = Bytes.toString(CellUtil.cloneFamily(cell));
switch (qualifier) {
// 判断字段 是否是 name
// 如果是 name 列 ,则将 name列的数据 添加到 Put 对象中 注意 一个 map 只读一条数据
case "name":
put.add(cell);
break;
}
}
if(!put.isEmpty()){
// 输出
context.write(key, put);
}
}
}
Reduce
package com.wangt.habse.mr01;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;
import java.io.IOException;
/**
* 将数据写入到 Person_mr 表中
*
* @author 王天赐
* @create 2019-08-02 20:12
*/
public class PersonReducer extends TableReducer<ImmutableBytesWritable, Put, ImmutableBytesWritable> {
@Override
protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
// 将 put 直接输出即可
// put 中封装了 存入新表的数据
for (Put value : values) {
context.write(key, value);
}
}
}
Driver 类
package com.wangt.habse.mr01;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* @author 王天赐
* @create 2019-08-02 20:12
*/
public class PersonDriver extends Configuration implements Tool {
// 官方建议的写法是 继承 Configuration 和 实现 Tool
private Configuration conf = new Configuration();
@Override
public int run(String[] args) throws Exception {
// 创建 Job 对象
Job job = Job.getInstance(conf);
// 设置 Driver class
job.setJarByClass(PersonDriver.class);
// 设置 Map 对象
TableMapReduceUtil.initTableMapperJob(
TableName.valueOf("Person"),
new Scan(),
PersonMapper.class,
ImmutableBytesWritable.class,
Put.class,
job);
// 设置 Reduce Class
TableMapReduceUtil.initTableReducerJob(
"Person_mr",
PersonReducer.class,
job);
// 提交
boolean result = job.waitForCompletion(true);
return result ? 0 : 1;
}
@Override
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Configuration getConf() {
return conf;
}
public static void main(String[] args) throws Exception {
Configuration configuration = HBaseConfiguration.create();
ToolRunner.run(configuration, new PersonDriver(), args);
}
}
案例 二
1.需求
从 hdfs 上读取 文件中的数据 写入到 Person 表
Person.csv
2001,info01,name,maomoa
2002,info01,name,maomao2
2003,info01,name,kakak
2004,info01,name,fafa
2.代码
Mapper
package com.wangt.habse.mr02;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
/**
* @author 王天赐
* @create 2019-08-04 16:20
*/
public class HDFSMapper extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// 2001,info01,name,maomoa
// 对数据进行切分
String[] fields = value.toString().split(",");
// 获取各个字段信息 :
byte[] rowkey = Bytes.toBytes(fields[0]);
byte[] family = Bytes.toBytes(fields[1]);
byte[] qualifier = Bytes.toBytes(fields[2]);
byte[] value_ = Bytes.toBytes(fields[3]);
// 创建 Put 对象
Put put = new Put(rowkey);
// 将要添加的信息封装到 Put 对象中
put.addColumn(family, qualifier, value_);
// 判断当 Put不为空时输出到 Reduce
if(!put.isEmpty()){
context.write(new ImmutableBytesWritable(rowkey), put);
}
}
}
Reduce
package com.wangt.habse.mr02;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;
import java.io.IOException;
/**
* @author 王天赐
* @create 2019-08-04 16:23
*/
public class HDFSReducer extends TableReducer <ImmutableBytesWritable, Put, NullWritable>{
@Override
protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
// 直接遍历输出即可
for (Put value : values) {
context.write(NullWritable.get(), value);
}
}
}
Driver
package com.wangt.habse.mr02;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
* @author 王天赐
* @create 2019-08-04 16:24
*/
public class HDFSDriver extends Configuration implements Tool {
// Hadoop 配置文件对象 创建 Job 对象时使用
private Configuration conf = new Configuration();
@Override
public int run(String[] strings) throws Exception {
// 创建 Job 对象
Job job = Job.getInstance(conf);
// 设置 Driver Class
job.setJarByClass(HDFSDriver.class);
// 设置 Map
job.setMapperClass(HDFSMapper.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(Put.class);
// 设置 Reduce
TableMapReduceUtil.initTableReducerJob(
"Person",
HDFSReducer.class,
job);
// 设置 输入路径
FileInputFormat.addInputPath(job, new Path(strings[0]));
// 提交
boolean result = job.waitForCompletion(true);
return result ? 0 : 1;
}
@Override
public void setConf(Configuration configuration) {
this.conf = configuration;
}
@Override
public Configuration getConf() {
return conf;
}
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
int status = ToolRunner.run(conf, new HDFSDriver(), args);
System.exit(status);
}
}
Shell 命令
hadoop jar Day03-1.0-SNAPSHOT.jar com.wangt.habse.mr02.HDFSDriver /mr/in/person.csv