java中纵表转横表_Hadoop MapReduce纵表转横表 与 横表转纵表

package seg;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.GenericOptionsParser;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

/**

* @author zhf

* @email zhf.thu@gmail.com

* @version 创建时间：2014年8月24日 上午9:56:45

*/

public class Vertical2Horizontal extends Configured implements Tool{

public static void main(String[] args) throws Exception {

int exitCode = ToolRunner.run(new Vertical2Horizontal(), args);

System.exit(exitCode);

}

@Override

public int run(String[] arg0) throws Exception {

String[] args = new GenericOptionsParser(arg0).getRemainingArgs();

if(args.length != 2){

System.out.println("Usage:seg.Horizontal2Vertical ");

System.exit(1);

}

Configuration conf = new Configuration();

FileSystem fs = FileSystem.get(conf);

if(fs.exists(new Path(args[1])))

fs.delete(new Path(args[1]),true);

Job job = new Job(conf);

job.setJarByClass(getClass());

job.setMapperClass(HVMapper.class);

job.setReducerClass(HVReducer.class);

job.setMapOutputKeyClass(Text.class);

job.setMapOutputValueClass(Text.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(Text.class);

FileInputFormat.addInputPath(job, new Path(args[0]));

FileOutputFormat.setOutputPath(job, new Path(args[1]));

return job.waitForCompletion(true) ? 0:1;

}

public static class HVMapper extends Mapper{

private Text text = new Text();

private Text clazz = new Text();

public void map(LongWritable key,Text value,Context context) throws IOException, InterruptedException{

String line = value.toString();

String params[] = line.split("\t");

text.set(params[0]);

clazz.set(params[1]);

context.write(clazz,text);

}

}

public static class HVReducer extends Reducer{

private Text result = new Text();

public void reduce(Text key,Iterable values,Context context) throws IOException, InterruptedException{

String tmp = "";

for(Text val : values){

tmp += val + "\t";

}

result.set(tmp.trim());

context.write(key, result);

}

}

}