MR--案例1



package com.testMr;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

public class JoinDemo {

	public static class LeftJoinMap extends Mapper<LongWritable, Text, Text, Text> {
		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
				throws IOException, InterruptedException {
			String[] line = value.toString().split(" ");
			String buyerName = line[0];
			String receiverName = line[1];
			String goods = line[2];
			String price = line[3];
			context.write(new Text(buyerName + receiverName),
					new Text(buyerName + " " + receiverName + " " + goods + " " + price));
		}
	}

	public static class LeftJoinReduce extends Reducer<Text, Text, Text, Text> {

		@Override
		protected void reduce(Text arg0, Iterable<Text> arg1, Reducer<Text, Text, Text, Text>.Context arg2)
				throws IOException, InterruptedException {

			Iterator it = arg1.iterator();
			int count = 0;
			int sum = 0;
			String[] arr;
			while (it.hasNext()) {
				count++;
				String line = it.next().toString();
				if (count >= 1) {
					arr = line.split(" ");
					int s = Integer.parseInt(arr[3]);
					sum = sum + s;
				}
			}
			String Sum = Integer.toString(sum);
			String Count = Integer.toString(count);
			arg2.write(new Text(arg0), new Text(Count + " " + Sum));

		}
	}

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		System.out.println(args.length);
		for (String str : args) {
			System.out.println(str);
		}
		String input1 = args[0];
		// String input2 = args[1];
		String outputPath = args[1];

		Job job = Job.getInstance();

		job.setJobName("test");
		job.setJarByClass(JoinDemo.class);

		FileInputFormat.addInputPath(job, new Path(input1));
		// FileInputFormat.addInputPath(job, new Path(input2));
		FileOutputFormat.setOutputPath(job, new Path(outputPath));

		job.setMapperClass(LeftJoinMap.class);
		job.setReducerClass(LeftJoinReduce.class);
		job.setPartitionerClass(HashPartitioner.class);
		job.setNumReduceTasks(1);

		job.setOutputValueClass(Text.class);
		job.setOutputKeyClass(Text.class);

		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}

}


数据结构:
id name  stuID   sex   age
1 小王    1203     男    15


id   stuID   courseName   score
1    1203         语文                82
2    1203         数学                70

  • 0
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值