hadoop实战之单表关联STjoin源码

最新推荐文章于 2022-05-09 14:19:56 发布

薛定谔的哈士奇啊

最新推荐文章于 2022-05-09 14:19:56 发布

阅读量290

点赞数 2

分类专栏：大数据文章标签： hadoop实战源码

本文链接：https://blog.csdn.net/sinat_36256646/article/details/80574477

版权

大数据专栏收录该内容

12 篇文章 0 订阅

订阅专栏

实例中给出child-parent表，要求输出grandchild-grandparent表：

file：

child parent
Tom Lucy
Tom Jack
Jone Lucy
Jone Jack
Lucy Mary
Lucy Ben
Jack Alice
Jack Jesse
Terry Alice
Terry Jesses
Philip Terry
Philip Alma
Mark Terry
Mark Alma

样例输出为：

child parent
Tom Lucy
Tom Jack
Jone Lucy
Jone Jack
Lucy Mary
Lucy Ben
Jack Alice
Jack Jesse
Terry Alice
Terry Jesses
Philip Terry
Philip Alma
Mark Terry
Mark Alma

一个晚上，看了一下书上的源码，直接抛开书自己写，和书中的略有不同（改进），自己用split函数写的切分部分：

package com.test.stjoin;


import java.io.IOException;
import java.util.ArrayList;
import java.util.StringTokenizer;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class STjoin {
	private static int time = 0;


	public static class TokenizerMapper extends Mapper<Object, Text, Text, Text> {
		public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
			String childname = new String();
			String parentname = new String();
			String[] str = value.toString().trim().split(" ");
			if (time == 0) {
				time++;
			} else {
				childname = str[0];
				parentname = str[1];
				context.write(new Text(parentname), new Text("1" + "+" + childname));
				context.write(new Text(childname), new Text("2" + "+" + parentname));
			}
		}
	}


	public static class TokenizerReducer extends Reducer<Text, Text, Text, Text> {
		public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
			// 类似C++中的泛型模板
			ArrayList<String> grandchild = new ArrayList<String>();
			ArrayList<String> grandparent = new ArrayList<String>();
			// String[] grandchild=new String[10];
			// String[] grandparent=new String[10];
			// int grandchildnum=0;
			// int grandparentnum=0;
			// 处理第一行
			if (time == 0) {
				context.write(new Text("grandchild"), new Text("grandparent"));
				time++;
			}
			// foreach方法迭代每一行
			for (Text val : values) {
				String[] str = val.toString().split("\\+");
				String relationtype = new String(str[0]);
				String realtion = new String(str[1]);
				if (relationtype.equals("1"))
					grandchild.add(realtion);
				else
					grandparent.add(realtion);
				// context.write(key,new Text(""));
			}
			// 笛卡尔积的方法求关系
			if (grandchild.size() != 0 && grandparent.size() != 0) {
				for (int i = 0; i < grandchild.size(); i++)
					for (int j = 0; j < grandparent.size(); j++)
						context.write(new Text(grandchild.get(i)), new Text(grandparent.get(j)));
			}
		}
	}


	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf, "STjoin");
		// Job job=new Job(conf,"word count");
		job.setJarByClass(STjoin.class);
		job.setMapperClass(TokenizerMapper.class);
		// job.setCombinerClass(TokenizerReducer.class);
		job.setReducerClass(TokenizerReducer.class);
		// job.setPartitionerClass(Partition.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
		FileInputFormat.addInputPath(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		System.exit(job.waitForCompletion(true) ? 0 : 1);


	}


}

运行过程：