MapReduce整合HBase

1、HBase 结合 MapReduce

   为什么需要用 mapreduce 去访问 hbase 的数据？

   ——加快分析速度和扩展分析能力 Mapreduce 访问 hbase 数据作分析一定是在离线分析的场景下应用

1.1、HBaseToHDFS

  从 hbase 中读取数据，分析之后然后写入 hdfs，代码实现：

package com.qyl.mapreduce;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class HbaseReader {
public static String user_info = "user_info";
static class HdfsSinkMapper extends TableMapper<Text, NullWritable> {
@Override
protected void map(ImmutableBytesWritable key, Result value,
Context context) throws IOException, InterruptedException {
byte[] bytes = key.copyBytes();
String rowkey = new String(bytes);
byte[] usernameBytes =
value.getValue("base_info".getBytes(),"name".getBytes());
String username = Bytes.toString(usernameBytes);
context.write(new Text(rowkey + "\t" + username),
NullWritable.get());
}
}
static class HdfsSinkReducer extends
Reducer<Text, NullWritable, Text, NullWritable> {
@Override
protected void reduce(Text key, Iterable<NullWritable> values,
Context context) throws IOException, InterruptedException {
context.write(key, NullWritable.get());
}
}
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
System.setProperty("HADOOP_USER_NAME", "root");
conf.set("hbase.zookeeper.quorum",
"qyl01:2181,qyl0202:2181,qyl03:2181,qyl04:2181,qyl05:2181");
   Job job = Job.getInstance(conf);
    job.setJarByClass(HbaseReader.class);
       Scan scan = new Scan();
  TableMapReduceUtil.initTableMapperJob(user_info, scan,
  HdfsSinkMapper.class, Text.class, NullWritable.class, job);
     job.setReducerClass(HdfsSinkReducer.class);
    FileOutputFormat.setOutputPath(job, new Path("/hbasetest/output"));
     job.setOutputKeyClass(Text.class);
     job.setOutputValueClass(NullWritable.class);
     job.waitForCompletion(true);
}
}

1.2、HDFSToHBase

 从 hdfs 从读入数据，处理之后写入 hbase，代码实现：

从读入数据，处理之后写入 hbase，代码实现：
package com.qyl.mapreduce;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
public class HbaseSinkerTest {
public static String TABLE_NAME = "flowbean";
static class HbaseSinkMrMapper extends Mapper<LongWritable, Text, Text,
NullWritable> {
@Override
protected void map(LongWritable key, Text value, Context context) throws
IOException, InterruptedException {
String line = value.toString();
String[] fields = line.split("\t");
String phone = fields[0];
String url = fields[1];
context.write(new Text(phone+"\t"+url), NullWritable.get());
}
}
static class HbaseSinkMrReducer extends TableReducer<Text, NullWritable,
ImmutableBytesWritable> {
@Override
protected void reduce(Text key, Iterable<NullWritable> values,
Context context) throws IOException, InterruptedException {
// Put put = new Put(key.getPhone().getBytes());
String[] keys = key.toString().split("\\t");
Put put = new Put(keys[0].getBytes());
put.add("f1".getBytes(), "url".getBytes(), keys[1].getBytes());
context.write(new ImmutableBytesWritable(keys[0].getBytes()), put);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum",
"qyl:2181,qyl02:2181,qyl03:2181,qyl04:2181,qyl05:2181");
// 以下这段代码为创建表的代码，表名 flowbean， 列簇叫 f1
HBaseAdmin hBaseAdmin = new HBaseAdmin(conf);
boolean tableExists = hBaseAdmin.tableExists(TABLE_NAME);
if (tableExists) {
hBaseAdmin.disableTable(TABLE_NAME);
hBaseAdmin.deleteTable(TABLE_NAME);
}
HTableDescriptor desc = new
HTableDescriptor(TableName.valueOf(TABLE_NAME));
HColumnDescriptor hColumnDescriptor = new
HColumnDescriptor("f1".getBytes());
desc.addFamily(hColumnDescriptor);
hBaseAdmin.createTable(desc);
Job job = Job.getInstance(conf);
job.setJarByClass(HbaseSinker.class);
job.setMapperClass(HbaseSinkMrMapper.class);
job.setReducerClass(HbaseSinkMrReducer.class);
TableMapReduceUtil.initTableReducerJob(TABLE_NAME,
HbaseSinkMrReducer.class, job);
FileInputFormat.setInputPaths(job, new Path("/data/data.txt"));
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(Mutation.class);
job.waitForCompletion(true);
}
}