Left join --案例1

最新推荐文章于 2024-07-10 14:43:50 发布

淡若晨岚

最新推荐文章于 2024-07-10 14:43:50 发布

阅读量409

点赞数

分类专栏： MapReduce

本文链接：https://blog.csdn.net/C8232/article/details/53200133

版权

MapReduce 专栏收录该内容

7 篇文章 0 订阅

订阅专栏

统计xx订单个数和价格总和：

package com.testMr;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

public class LeftJoinDemo {

	public static class LeftJoinMap extends Mapper<LongWritable, Text, Text, Text> {
		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
				throws IOException, InterruptedException {
			String[] arr = value.toString().split(" ");
			FileSplit file = (FileSplit) context.getInputSplit();
			String path = file.getPath().toString();
			String buyer;

			if (path.contains("trade.txt")) {
				buyer = arr[2];
				context.write(new Text(buyer), new Text(value.toString() + " " + "a"));
			} else {
				buyer = arr[1];
				context.write(new Text(buyer), new Text(value.toString() + " " + "b"));
			}
		}
	}

	public static class LeftJoinReduce extends Reducer<Text, Text, Text, Text> {
		@Override
		protected void reduce(Text arg0, Iterable<Text> arg1, Reducer<Text, Text, Text, Text>.Context arg2)
				throws IOException, InterruptedException {

			List<String> a = new ArrayList<String>();
			List<String> b = new ArrayList<String>();
			String[] arr;
			List<String> list = new LinkedList<String>();

                        // 这个变量用来存放确认是北京的订单id，因为在order里没有城市列，所以在划分trade时，把是北京的id，记一下
			for (Text line : arg1) {
				arr = line.toString().split(" ");
				if (arr[arr.length - 1].equals("a") && arr[3].equals("北京")) {
					a.add(line.toString());
					list.add(arr[0]);
				} else {
					b.add(line.toString());
				}
			}
			int tradesize = a.size(); // 订单个数

			int priceSum = 0; // 价格总和
			String price;
			for (String order : b) {
				arr = order.split(" ");
				if (list.contains(arr[0])) {

                        // 查看这个order的id是不是在北京id的list里，是的话累加
												
					price = arr[3];
					priceSum += Integer.parseInt(price);
				}
			}
			arg2.write(arg0, new Text(tradesize + " " + priceSum)); 
		}
	}

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		System.out.println(args.length);
		for (String str : args) {
			System.out.println(str);
		}
		String input1 = args[0];
		String input2 = args[1];
		String outputPath = args[2];

		Job job = Job.getInstance();

		job.setJobName("test");
		job.setJarByClass(LeftJoinDemo.class);

		FileInputFormat.addInputPath(job, new Path(input1));
		FileInputFormat.addInputPath(job, new Path(input2));
		FileOutputFormat.setOutputPath(job, new Path(outputPath));

		job.setMapperClass(LeftJoinMap.class);
		job.setReducerClass(LeftJoinReduce.class);
		job.setPartitionerClass(HashPartitioner.class);
		job.setNumReduceTasks(1);

		job.setOutputValueClass(Text.class);
		job.setOutputKeyClass(Text.class);

		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}

}