需求:使用mr从hdfs读取数据处理之后写入到Hbase
package pers.nebo.mrtohbase;
/**
- @ author fnb
- @ email nebofeng@gmail.com
- @ date 2019/12/17
- @ des :
/
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
/**
*
- mr to hbase 的输入与输出 有 job设置 ,
- tableMapper 为从hbase 读取数据
- tableReduce 为往Hbase 写入数据
- 写入的表名 通过job 设置为 tablename (一个) 则 reduce 函数中 为 NullWritable.get
- 写入的表名 为多个 则 job 设置为 MultiTableOutputFormat 通过 reduce 函数指定tableName
*/
public class MR2Hbase {
public static class Map extends Mapper<LongWritable, Text, Text, LongWritable>{
//该步骤同一般Map程序
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
//将读入行转化为字符串
String line = value.toString();
//切分字符串
String[] words = StringUtils.split(line," ");
//将单词写入context
for(String word:words) {
context.write(new Text(word), new LongWritable(1));
}
}
}
public static class Reduce extends TableReducer<Text, LongWritable, NullWritable>{
@Override
protected void reduce(Text key, Iterable<LongWritable> values,Context context)
throws IOException, InterruptedException {
long count =0;
for(LongWritable value : values) {
count += value.get();
}
//实例化Put,将单词作为主键
Put put = new Put(Bytes.toBytes(key.toString()));
//列族为content,key为result,value为count
put.addColumn(Bytes.toBytes("content"), Bytes.toBytes("result"), Bytes.toBytes(String.valueOf(count)));
context.write(NullWritable.get(), put);
}
}
//创建表
public static void createTable(String tablename) throws Exception{
//设置配置文件
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "master:2181,slave1:2181,slave2:2181");
Connection conn = ConnectionFactory.createConnection(conf);
//hbase客户端实例
Admin admin = conn.getAdmin();
//指定表名
TableName name = TableName.valueOf(tablename);
//向表描述里添加表名
HTableDescriptor desc = new HTableDescriptor(name);
//指定列族名和版本数
HColumnDescriptor content = new HColumnDescriptor("content");
content.setMaxVersions(3);
//向表描述里添加列族
desc.addFamily(content);
//判断表是否存在
if(admin.tableExists(name)){
System.out.println("table exists,trying recreate table !");
admin.disableTable(name);
admin.deleteTable(name);
}
System.out.println("Create new table: "+ tablename);
//创建表
admin.createTable(desc);
admin.close();
}
public static void main(String[] args) throws Exception{
String tablename = "WC";
//创建表
createTable(tablename);
//配置文件
Configuration conf = new Configuration();
conf.set(TableOutputFormat.OUTPUT_TABLE, tablename);
Job job = Job.getInstance(conf);
//设置整个job所调用类的jar包路径
job.setJarByClass(MR2Hbase.class);
//设置该作业所使用的mapper和reducer类
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
//指定mapper输出数据的k-v类型,和reduce输出类型一样,可缺省
job.setMapOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
//指定reduce输出到Hbase
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
//指定输入数据存放路径
FileInputFormat.setInputPaths(job, new Path(args[0]));
//指定输出到HBase
job.setOutputFormatClass(TableOutputFormat.class);
//将job提交给集群运行,参数为true表示提示运行进度
System.exit(job.waitForCompletion(true)?0:1);
}
}
文章首发于 http://nebofeng.com/2021/01/12/mrtohbase/),it内容具有时效性,未避免更新后未同步,请点击查看最新内容)