4.MapReduce+Hbase+HDFS的整合(补充):
public class WCJob {
public static void main(String[] args) throws Exception {
// 默认加载src下的配置文件
Configuration conf = new Configuration();
// conf.set("fs.defaultFS", "hdfs://node2:8020");
// conf.set("yarn.resourcemanager.hostname", "node3");
conf.set("hbase.zookeeper.quorum", "node1,node2,node3");
conf.set("mapred.jar", "D:\\MR\\wc.jar");
Job job = Job.getInstance(conf);
job.setJarByClass(WCJob.class);
job.setMapperClass(WCMap.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path("/wc/input/wc"));
// hbase reducer
String targetTable = "wc";
TableMapReduceUtil.initTableReducerJob(
targetTable, // reduce输出的到哪一张表
WCTableReducer.class, // reducer class
job);
boolean flag = job.waitForCompletion(true);
if (flag) {
System.out.println("job success !");
}
}
}
public class WCMap extends Mapper<LongWritable, Text, Text, IntWritable> {
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
String[] strs = StringUtils.split(value.toString(), ' ');
for (String s : strs) {
context.write(new Text(s), new IntWritable(1));
}
}
}
public class WCTableReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> {
public static final byte[] CF = "cf".getBytes();
public static final byte[] COUNT = "count".getBytes();
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int i = 0;
for (IntWritable val : values) {
i += val.get();
}
Put put = new Put(Bytes.toBytes(key.toString()));
put.add(CF, COUNT, Bytes.toBytes(i));
context.write(null, put);
}
}
public class WCJob {
public static void main(String[] args) throws Exception {
// 默认加载src下的配置文件
Configuration conf = new Configuration();
// conf.set("fs.defaultFS", "hdfs://node2:8020");
// conf.set("yarn.resourcemanager.hostname", "node3");
conf.set("hbase.zookeeper.quorum", "node1,node2,node3");
conf.set("mapred.jar", "D:\\MR\\wc.jar");
Job job = Job.getInstance(conf);
job.setJarByClass(WCJob.class);
job.setMapperClass(WCMap.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path("/wc/input/wc"));
// hbase reducer
String targetTable = "wc";
TableMapReduceUtil.initTableReducerJob(
targetTable, // reduce输出的到哪一张表
WCTableReducer.class, // reducer class
job);
boolean flag = job.waitForCompletion(true);
if (flag) {
System.out.println("job success !");
}
}
}
public class WCMap extends Mapper<LongWritable, Text, Text, IntWritable> {
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
String[] strs = StringUtils.split(value.toString(), ' ');
for (String s : strs) {
context.write(new Text(s), new IntWritable(1));
}
}
}
public class WCTableReducer extends TableReducer<Text, IntWritable, ImmutableBytesWritable> {
public static final byte[] CF = "cf".getBytes();
public static final byte[] COUNT = "count".getBytes();
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int i = 0;
for (IntWritable val : values) {
i += val.get();
}
Put put = new Put(Bytes.toBytes(key.toString()));
put.add(CF, COUNT, Bytes.toBytes(i));
context.write(null, put);
}
}