新版hadoop MultipleOutputs多文件输出

转载请标明出处：http://blackwing.iteye.com/blog/2191454

网上虽然有不少关于MultipleOutputs实现多文件输出的文章，但发现要不还是使用mapred.lib旧接口，要不就是说明不清楚。

Mapper

package com.yy.hiido.itemcf.hadoop.mapper;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class ReadFilesMapper extends
		Mapper<LongWritable, Text, Text, Text> {
	private final String SPLITTER="\\s+";
	private Text outkey;
	private Text values;
	@Override
	protected void setup(Context context)
			throws IOException, InterruptedException {
		super.setup(context);
		outkey = new Text();
		values = new Text();
	}

	@Override
	protected void map(LongWritable key, Text value,Context context)
			throws IOException, InterruptedException {
		String line = value.toString();
		//System.out.println("line : "+line);
		String [] fields = line.split(this.SPLITTER);
		outkey.set(fields[0]);
		values.set(fields[1]);
		System.out.println("key : "+fields[0]+"  |  values : "+fields[1]);
		context.write(outkey,values);
	}
}

Reducer

package com.yy.hiido.itemcf.hadoop.reducer;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ReadFilesReducer extends
		Reducer<Text, Text, Text, Text> {
	private MultipleOutputs<Text,Text> mos;
	@Override
	protected void setup(Context context)
			throws IOException, InterruptedException {
		mos = new MultipleOutputs<Text,Text>(context);
	}

	@Override
	protected void reduce(Text key, Iterable<Text> values,Context context)
			throws IOException, InterruptedException {
		String temp="";
		for(Text t : values)
			temp+=t.toString()+" | ";
		//mos.write(key.toString(), key, new Text(temp));//这样需要预定义named output
		mos.write(key, new Text(temp), key.toString());//这样不需要与定义named output
	}

	@Override
	protected void cleanup(Context context)
			throws IOException, InterruptedException {
		mos.close();;
	}
}

main

package com.yy.hiido.itemcf.hadoop.job;

import java.io.IOException;
import java.net.URI;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import com.yy.hiido.itemcf.hadoop.mapper.ReadFilesMapper;
import com.yy.hiido.itemcf.hadoop.reducer.ReadFilesReducer;

/**
 * @author BlackWing 测试multioutput
 * */

public class TestMultiOutputJob extends Configured implements Tool {
	// 配置文件名
	public final static String propertyFileName = "config.xml";
	private static final Log LOG = LogFactory.getLog(TestMultiOutputJob.class);
	private static Configuration conf = HBaseConfiguration.create();

	public int myjob() {
		// 读配置文件
		conf.addResource(propertyFileName);
		String input = conf.get("file.to.read");
		String outputDir = conf.get("test.output");
		System.out.println("input : "+input);
		System.out.println("output : "+outputDir);

		// 若输出目录存在,则删除
		try {
			FileSystem fs = FileSystem.get(URI.create(outputDir),
					new Configuration());
			fs.delete(new Path(outputDir), true);
			fs.close();
		} catch (Exception e) {
			e.printStackTrace();
		}

		Job myjob = null;

		try {
			myjob = new Job(conf);
		} catch (IOException e) {
			e.printStackTrace();
		}
		myjob.setJarByClass(ReadFilesMapper.class);

		try {
			FileInputFormat.setInputPaths(myjob, input);
			FileOutputFormat.setOutputPath(myjob, new Path(outputDir));
		} catch (IOException e1) {
			e1.printStackTrace();
		}
		myjob.setMapperClass(ReadFilesMapper.class);
		myjob.setInputFormatClass(TextInputFormat.class);
//		myjob.setOutputFormatClass(TextOutputFormat.class);
		LazyOutputFormat.setOutputFormatClass(myjob, TextOutputFormat.class);
		myjob.setReducerClass(ReadFilesReducer.class);
		myjob.setOutputKeyClass(Text.class);
		myjob.setOutputValueClass(Text.class);

		//MultipleOutputs.addNamedOutput(myjob, "moshouzhengba", TextOutputFormat.class, Text.class, Text.class);
		//MultipleOutputs.addNamedOutput(myjob, "maoxiandao", TextOutputFormat.class, Text.class, Text.class);
		//MultipleOutputs.addNamedOutput(myjob, "yingxionglianmen", TextOutputFormat.class, Text.class, Text.class);

		boolean succeeded = false;
		try {
			succeeded = myjob.waitForCompletion(true);
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}
		if (!succeeded)
			return -1;
		// status记录job运行状态
		LOG.info("Job complete !");
		return 1;
	}

	@Override
	public int run(String[] as) throws Exception {
		// 读配置文件
		conf.addResource(propertyFileName);
		myjob();
		return 0;
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		long start = System.currentTimeMillis();
		try {
			ToolRunner.run(new TestMultiOutputJob(), args);
		} catch (Exception e) {
			e.printStackTrace();
		}
		long end = System.currentTimeMillis();
		System.out.println("run the program costs time:" + (end - start)
				/ 60000 + "Minutes");
	}
}

其中注意的是：
1.reducer中调用时，要调用MultipleOutputs以下接口：

write

public void write(KEYOUT key,
                  VALUEOUT value,
                  String baseOutputPath)
           throws IOException,
                  InterruptedException

如果调用

write

public <K,V> void write(String namedOutput,
                        K key,
                        V value)
           throws IOException,
                  InterruptedException

则需要在job中，预先声明named output（如下），不然会报错：named output xxx not defined：

MultipleOutputs.addNamedOutput(myjob, "moshouzhengba", TextOutputFormat.class, Text.class, Text.class);
		MultipleOutputs.addNamedOutput(myjob, "maoxiandao", TextOutputFormat.class, Text.class, Text.class);
		MultipleOutputs.addNamedOutput(myjob, "yingxionglianmen", TextOutputFormat.class, Text.class, Text.class);

2.默认情况下，输出目录会生成part-r-00000或者part-m-00000的空文件，需要如下设置后，才不会生成：

//		myjob.setOutputFormatClass(TextOutputFormat.class);
		LazyOutputFormat.setOutputFormatClass(myjob, TextOutputFormat.class);

就是去掉job设置outputFormatClass，改为通过LazyOutputFormat设置

这里只是以Text的输入输出格式说明。

官方的文档：
https://hadoop.apache.org/docs/current2/api/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.html#write(java.lang.String, K, V)