java map reduce简单模板

2019独角兽企业重金招聘Python工程师标准>>>

输入文件内容：

a       a1
b       b2
c       c3
d       d4
a       a1
b       b2
c       c3
d       d4

输出：

a    a1|0    a1|20
b    b2|5    b2|25
c    c3|10    c3|30
d    d4|15    d4|35

代码：

import java.io.IOException;  
import java.util.StringTokenizer;  
       
import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.util.GenericOptionsParser;  
       
public class WordCount {  
       
	public static class TokenizerMapper extends Mapper<LongWritable, Text, Text, Text>{  
           
		public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			String[] oriSegs = value.toString().split("\t");
			String str = oriSegs[1] + "|" + key;
			context.write(new Text(oriSegs[0]), new Text(str));
		}
	}  
         
  	public static class IntSumReducer extends Reducer<Text, Text, Text, Text> {  
       
    		public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {  
			String out = "";
			for (Text val: values) {
				if (!out.equals("")) {
					out += '\t';
				}
				out += val.toString();
			}
			context.write(key, new Text(out));
    		}  
	}  
       
	public static void main(String[] args) throws Exception {  
		
		Configuration conf = new Configuration();
		conf.set("mapred.job.queue.name", "platform");
		String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();  
		if (otherArgs.length != 2) {  
			System.err.println("Usage: wordcount <in> <out>");  
			System.exit(2);  
    		}
		
		Job job = new Job(conf, "word count");  
	       job.setJarByClass(WordCount.class);  
   		job.setMapperClass(TokenizerMapper.class);  
		job.setCombinerClass(IntSumReducer.class);  
		job.setReducerClass(IntSumReducer.class);  
		job.setOutputKeyClass(Text.class);  
		job.setOutputValueClass(Text.class);
		job.setNumReduceTasks(1); //set reducer number
		FileInputFormat.addInputPath(job, new Path(otherArgs[0]));  
		FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));  
		System.exit(job.waitForCompletion(true) ? 0 : 1);  
	}  
}

编译：make.sh    编译成jar文件

javac -classpath /home/hadoop/hadoop-0.20.2-cdh3u0/hadoop-core-0.20.2-cdh3u0.jar:/home/hadoop/hadoop-0.20.2-cdh3u0/lib/commons-cli-1.2.jar -d wordcount_class WordCount.java
jar -cvf WordCount.jar -C wordcount_class/ .

执行map reduce任务：exec.sh

IN=/user/zhumingliang/tanx_rtb_account/input
OUT=/user/zhumingliang/tanx_rtb_account/output/test
hadoop jar WordCount.jar WordCount $IN $OUT

注意：

mapper的输入key在针对文件输入时，是一行起始位置在文件中的字符序号；而mapper的输入value则为整行内容。

reducer的输入key则为mapper的输出key; reducer的输入value则为mapper的输出value。

转载于:https://my.oschina.net/zmlblog/blog/201915