在写基于hbase的mapreduce时程序时候,与以前基于hbase的稍有不同
1、Mapper、Reducer与以前继承的类有点不同,现在继承TableMapper和TableReducer
2、在Mapper中是从hbase读取的表来处理的,在Reducer中最终也是要输出到hbase中
3、在Mapper中多出了一个类型就是行健的类型,以前是偏移值,在Reducer中也是多了一个hbase的行健
4、就是在书写job任务的时候有些不同,
(1)需要连接hbase的zookeeper,
(2)往job中添加的mapper使用的类也是不一样
(3)数据输入和输出不一样,以前是从hdfs中读取,现在是从hbase中读取,所以需要构建scan
示例程序
//Mapper
import java.io.IOException;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
public class HBaseMapper extends TableMapper<Text, IntWritable>
{
//key1代表的rowkey value1代表的是该条记录
@Override
protected void map(ImmutableBytesWritable key, Result value,Context context)
throws IOException, InterruptedException {
String infos = Bytes.toString(value.getValue(Bytes.toBytes("content"),Bytes.toBytes("info")));
String[] split = infos.split(" ");
for (String string : split)
{
context.write(new Text(string), new IntWritable(1));
}
}
}
//Reducer
import java.io.IOException;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
public class HBaseReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable>
{
@Override
protected void reduce(Text key3, Iterable<IntWritable> value3,Context context)
throws IOException, InterruptedException {
int count=0;
for (IntWritable intWritable : value3)
{
count+=intWritable.get();
}
//因为是hbase的mapreduce,所以输出的时候是输出到hbase中,因此我们需要构建一个put对象,把put对象输出
Put put=new Put(Bytes.toBytes(key3.toString()));
put.addColumn(Bytes.toBytes("content"), Bytes.toBytes("result"), Bytes.toBytes(count));
context.write(new ImmutableBytesWritable(Bytes.toBytes(key3.toString())), put);
}
}
//job
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
public class HbaseJobMain
{
public static void main(String[] args) throws Exception {
//需要连接zookeeper
Configuration conf=HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "192.168.112.111");
//需要创建job
Job job=Job.getInstance(conf);
job.setJarByClass(HbaseJobMain.class);
//需要构建一个扫描器
Scan scan=new Scan();
scan.addColumn(Bytes.toBytes("content"), Bytes.toBytes("info"));
//指定map
TableMapReduceUtil.initTableMapperJob("word", scan, HBaseMapper.class, Text.class, IntWritable.class, job);
//指定reducer
TableMapReduceUtil.initTableReducerJob("stat", HBaseReducer.class, job);
//执行job
job.waitForCompletion(true);
}
}
然后需要导出jar包,然后在hadoop中执行,以前需要制定输入路径和输出路径,现在因为直接从hbase中读取和写入
所以不需制定输入路径和输出路径,直接使用hadoop jar ***.jar就可以了
但是在执行的会报一个错误如下,这个是因为我们在yarn平台上执行了hbase相关的程序,yarn平台找不到相关的jar,所以
需要我们执行一下:
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/HBaseConfiguration
at day014.hbasemr.HbaseJobMain.main(HbaseJobMain.java:16)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.util.RunJar.run(RunJar.java:318)
at org.apache.hadoop.util.RunJar.main(RunJar.java:232)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.hbase.HBaseConfiguration
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 7 more
解决办法只需执行一下下边的就可以了:
export HADOOP_CLASSPATH=$HBASE_HOME/lib/*:$CLASSPATH
然后再执行,执行完稍微有些不完美,就是和我输出的值有关系,因为hbase没有执行的类型,所以一切表现形式都是二进制
所有在查看表的时候有点问题
然后我们稍微改造一下输出值
public class HBaseReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable>
{
@Override
protected void reduce(Text key3, Iterable<IntWritable> value3,Context context)
throws IOException, InterruptedException {
int count=0;
for (IntWritable intWritable : value3)
{
count+=intWritable.get();
}
//因为是hbase的mapreduce,所以输出的时候是输出到hbase中,因此我们需要构建一个put对象,把put对象输出
Put put=new Put(Bytes.toBytes(key3.toString()));
put.addColumn(Bytes.toBytes("content"), Bytes.toBytes("result"), Bytes.toBytes(""+count));
context.write(new ImmutableBytesWritable(Bytes.toBytes(key3.toString())), put);
}
}
然后再执行就正常了