23 篇文章 0 订阅

MapReduce作业按照顺序链接在一起

mapreduce-1 | mapreduce-2 | mapreduce-3 …

Job1.waitForCompletion(true);
Job2.waitForCompletion(true);

JobControl This class encapsulates a set of MapReduce jobs and its dependency.

#例子1#

package ex7;

import java.io.IOException;

private static final Text TEXT_SUM = new Text("SUM");
private static final Text TEXT_COUNT = new Text("COUNT");
private static final Text TEXT_AVG = new Text("AVG");

// 计算SumMapper，多单个Map中的数据求和
public static class SumMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

public long sum = 0;

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
sum += Long.parseLong(value.toString());
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(TEXT_SUM, new LongWritable(sum));
}
}

// 计算SumReducer，求和
public static class SumReducer extends Reducer<Text, LongWritable, Text, LongWritable> {

public long sum = 0;

public void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
for (LongWritable v : values) {
sum += v.get();
}
context.write(TEXT_SUM, new LongWritable(sum));
}
}

// 计算CountMapper，统计数量
public static class CountMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

public long count = 0;

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
count += 1;
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(TEXT_COUNT, new LongWritable(count));
}
}

// 计算CountReducer，统计数量
public static class CountReducer extends Reducer<Text, LongWritable, Text, LongWritable> {

public long count = 0;

public void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
for (LongWritable v : values) {
count += v.get();
}
context.write(TEXT_COUNT, new LongWritable(count));
}
}

// 计算平均数MR，Map
public static class AvgMapper extends Mapper<LongWritable, Text, LongWritable, LongWritable> {

public long count = 0;
public long sum = 0;

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] v = value.toString().split("\t");
if (v[0].equals("COUNT")) {
count = Long.parseLong(v[1]);
} else if (v[0].equals("SUM")) {
sum = Long.parseLong(v[1]);
}
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(new LongWritable(sum), new LongWritable(count));
}

}

// 计算平均数MR，Reduce
public static class AvgReducer extends Reducer<LongWritable, LongWritable, Text, DoubleWritable> {

public long sum = 0;
public long count = 0;

public void reduce(LongWritable key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
sum += key.get();
for (LongWritable v : values) {
count += v.get();
}
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(TEXT_AVG, new DoubleWritable(new Double(sum) / count));
}

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

String inputPath = "testdata/example_1";
String sumOutputPath = "testdata/example1_out/sum";
String countOutputPath = "testdata/example1_out/count";
String avgOutputPath = "testdata/example1_out/avg";

Job job1 = Job.getInstance(conf, "Sum");
job1.setMapperClass(SumMapper.class);
job1.setCombinerClass(SumReducer.class);
job1.setReducerClass(SumReducer.class);
job1.setOutputKeyClass(Text.class);
job1.setOutputValueClass(LongWritable.class);
FileOutputFormat.setOutputPath(job1, new Path(sumOutputPath));

Job job2 = Job.getInstance(conf, "Count");
job2.setMapperClass(CountMapper.class);
job2.setCombinerClass(CountReducer.class);
job2.setReducerClass(CountReducer.class);
job2.setOutputKeyClass(Text.class);
job2.setOutputValueClass(LongWritable.class);
FileOutputFormat.setOutputPath(job2, new Path(countOutputPath));

Job job3 = Job.getInstance(conf, "Average");
job3.setMapperClass(AvgMapper.class);
job3.setReducerClass(AvgReducer.class);
job3.setMapOutputKeyClass(LongWritable.class);
job3.setMapOutputValueClass(LongWritable.class);
job3.setOutputKeyClass(Text.class);
job3.setOutputValueClass(DoubleWritable.class);

// 将job1及job2的输出为做job3的输入
FileOutputFormat.setOutputPath(job3, new Path(avgOutputPath));

// 提交job1及job2,并等待完成
if (job1.waitForCompletion(true) && job2.waitForCompletion(true)) {
System.exit(job3.waitForCompletion(true) ? 0 : 1);
}
}
}


#例子2#

package ex7;

import java.io.IOException;

public class DependingMR {

private static final Text TEXT_SUM = new Text("SUM");
private static final Text TEXT_COUNT = new Text("COUNT");
private static final Text TEXT_AVG = new Text("AVG");

// 计算SumMapper，多单个Map中的数据求和
public static class SumMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

public long sum = 0;

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
sum += Long.parseLong(value.toString());
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(TEXT_SUM, new LongWritable(sum));
}
}

// 计算SumReducer，求和
public static class SumReducer extends Reducer<Text, LongWritable, Text, LongWritable> {

public long sum = 0;

public void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
for (LongWritable v : values) {
sum += v.get();
}
context.write(TEXT_SUM, new LongWritable(sum));
}
}

// 计算CountMapper，统计数量
public static class CountMapper extends Mapper<LongWritable, Text, Text, LongWritable> {

public long count = 0;

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
count += 1;
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(TEXT_COUNT, new LongWritable(count));
}
}

// 计算CountReducer，统计数量
public static class CountReducer extends Reducer<Text, LongWritable, Text, LongWritable> {

public long count = 0;

public void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
for (LongWritable v : values) {
count += v.get();
}
context.write(TEXT_COUNT, new LongWritable(count));
}
}

// 计算平均数MR，Map
public static class AvgMapper extends Mapper<LongWritable, Text, LongWritable, LongWritable> {

public long count = 0;
public long sum = 0;

public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] v = value.toString().split("\t");
if (v[0].equals("COUNT")) {
count = Long.parseLong(v[1]);
} else if (v[0].equals("SUM")) {
sum = Long.parseLong(v[1]);
}
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(new LongWritable(sum), new LongWritable(count));
}

}

// 计算平均数MR，Reduce
public static class AvgReducer extends Reducer<LongWritable, LongWritable, Text, DoubleWritable> {

public long sum = 0;
public long count = 0;

public void reduce(LongWritable key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
sum += key.get();
for (LongWritable v : values) {
count += v.get();
}
}

protected void cleanup(Context context) throws IOException, InterruptedException {
context.write(TEXT_AVG, new DoubleWritable(new Double(sum) / count));
}

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

String inputPath = "testdata/lab4";
String sumOutputPath = "testdata/lab4-out/sum";
String countOutputPath = "testdata/lab4-out/count";
String avgOutputPath = "testdata/lab4-out/avg";

Job job1 = Job.getInstance(conf, "Sum");
job1.setJarByClass(DependingMR.class);
job1.setMapperClass(SumMapper.class);
job1.setCombinerClass(SumReducer.class);
job1.setReducerClass(SumReducer.class);
job1.setOutputKeyClass(Text.class);
job1.setOutputValueClass(LongWritable.class);
FileOutputFormat.setOutputPath(job1, new Path(sumOutputPath));

Job job2 = Job.getInstance(conf, "Count");
job2.setJarByClass(DependingMR.class);
job2.setMapperClass(CountMapper.class);
job2.setCombinerClass(CountReducer.class);
job2.setReducerClass(CountReducer.class);
job2.setOutputKeyClass(Text.class);
job2.setOutputValueClass(LongWritable.class);
FileOutputFormat.setOutputPath(job2, new Path(countOutputPath));

Job job3 = Job.getInstance(conf, "Average");
job3.setJarByClass(DependingMR.class);
job3.setMapperClass(AvgMapper.class);
job3.setReducerClass(AvgReducer.class);
job3.setMapOutputKeyClass(LongWritable.class);
job3.setMapOutputValueClass(LongWritable.class);
job3.setOutputKeyClass(Text.class);
job3.setOutputValueClass(DoubleWritable.class);

// 将job1及job2的输出作为job3的输入
FileOutputFormat.setOutputPath(job3, new Path(avgOutputPath));

ControlledJob contlJob1=new ControlledJob(conf);
contlJob1.setJob(job1);

ControlledJob contlJob2=new ControlledJob(conf);
contlJob1.setJob(job2);

ControlledJob contlJob3=new ControlledJob(conf);
contlJob1.setJob(job3);

JobControl jobContl=new JobControl("depending job");

t.start();

if(jobContl.allFinished()) {
System.out.println(jobContl.getSuccessfulJobList());
jobContl.stop();
}

//		// �ύjob1��job2,���ȴ����
//		if (job1.waitForCompletion(true) && job2.waitForCompletion(true)) {
//			System.exit(job3.waitForCompletion(true) ? 0 : 1);
//		}
}
}


Job预处理和后处理
Mapreduce链接的方式
[MAP | REDUCE] +

MAP+ | REDUCE | MAP*
Map1 | Map2 | Reduce | Map3 | Map4

• 0
点赞
• 1
收藏
觉得还不错? 一键收藏
• 打赏
• 0
评论
10-30 487
10-17 8707
06-11 129
06-22 311
03-29 7679
08-26 1423

### “相关推荐”对你有帮助么？

• 非常没帮助
• 没帮助
• 一般
• 有帮助
• 非常有帮助

phac123

¥2 ¥4 ¥6 ¥10 ¥20

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、C币套餐、付费专栏及课程。