第1关:HBase的MapReduce快速入门
package com.processdata;
import java.io.IOException;
import java.util.List;
import java.util.Scanner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apdplat.word.WordSegmenter;
import org.apdplat.word.segmentation.Word;
import com.util.HBaseUtil;
import com.vdurmont.emoji.EmojiParser;
public class WorldCountMapReduce extends Configured implements Tool {
private static class MyMapper extends TableMapper<Text, IntWritable> {
private static byte[] family = "comment_info".getBytes();
private static byte[] column = "content".getBytes();
@Override
protected void map(ImmutableBytesWritable rowKey, Result result, Context context) {
try {
byte[] value = result.getValue(family, column);
String content = new String(value, "utf-8");
String[] split = content.split(" ");
for (String str : split) {
Text text = new Text(str);
IntWritable v = new IntWritable(1);
context.write(text, v);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
private static class MyReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> {
private static byte[] family = "word_info".getBytes();
private static byte[] column = "count".getBytes();
@Override
public void reduce(Text key, Iterable<IntWritable> values, Context context) {
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
Put put = new Put(Bytes.toBytes(key.toString()));
put.addColumn(family, column, Bytes.toBytes(sum));
try {
context.write(null, put);
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
@Override
public int run(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create(getConf());
String tablename = args[0];
String targetTable = args[1];
Job job = Job.getInstance(conf);
Scan scan = new Scan();
TableMapReduceUtil.initTableMapperJob(tablename, scan, MyMapper.class, Text.class, IntWritable.class, job);
TableMapReduceUtil.initTableReducerJob(targetTable, MyReducer.class, job);
job.waitForCompletion(true);
return 0;
}
}
第2关:HBase的MapReduce使用
package com.processdata;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.util.Tool;
public class WorldCountMapReduce2 extends Configured implements Tool {
private static class MyMapper extends Mapper<Object, Text, Text, IntWritable> {
@Override
public void map(Object object, Text value, Context context) throws IOException, InterruptedException {
String[] split = value.toString().split(" ");
for (String str : split) {
Text text = new Text(str.getBytes());
IntWritable v = new IntWritable(1);
context.write(text, v);
}
}
}
private static class MyReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> {
private static byte[] family = "word_info".getBytes();
private static byte[] column = "count".getBytes();
@Override
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
Put put = new Put(Bytes.toBytes(key.toString()));
put.addColumn(family, column, Bytes.toBytes(sum));
context.write(null, put);
}
}
@Override
public int run(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create(getConf());
String file = args[0];
String targetTable = args[1];
Job job = Job.getInstance(conf);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setJarByClass(WorldCountMapReduce2.class);
FileInputFormat.addInputPath(job, new Path(file));
job.setMapperClass(MyMapper.class);
TableMapReduceUtil.initTableReducerJob(targetTable, MyReducer.class, job);
job.waitForCompletion(true);
return 0;
}
}