Data deduplication

题目：Data deduplication

描述

你的程序要求读入输入文件，在去掉所有数据中的重复数据后输出结果。在输入文件中每一行是一个元数据。

输入

输入是一组文本文件，在每个输入文件中每一行是一个数据。每一个元数据都是一个字符串。

输出文件

输出文件的每一行都是在输入文件中出现过的一个数据，并且输出文件中的每一行都不相同。

输入样例

input1:
2006-6-9 a
2006-6-10 b
2006-6-11 c
2006-6-12 d
2006-6-13 a
2006-6-14 b
2006-6-15 c
2006-6-11 c
input2:
2006-6-9 b
2006-6-10 a
2006-6-11 b
2006-6-12 d
2006-6-13 a
2006-6-14 c
2006-6-15 d
2006-6-11 c

输出样例:
2006-6-10 a 
2006-6-10 b 
2006-6-11 b 
2006-6-11 c 
2006-6-12 d 
2006-6-13 a 
2006-6-14 b 
2006-6-14 c 
2006-6-15 c 
2006-6-15 d 
2006-6-9 a 
2006-6-9 b

注意:
1 输出结果是按照字典顺序排序的；
2 每一行都是一个元数据；
3 重复数据在输出文件中也要输出一次。

整整干了一下午加晚上，悲催的。。。。

程序应该有漏洞，只是在单机下小数据文件可以测试通过。欢迎指正~~

代码如下：

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;


public class TextPair implements WritableComparable<TextPair> {

public  Text first;
public  Text second;

public  TextPair(){
this.first=new Text();
this.second=new Text();
    }

public TextPair(Text first, Text second) {
//super();
        this.first = first;
this.second = second;
    }
public TextPair(String first,String second){
this.first=new Text(first);
this.second=new Text(second);

    }

public Text getFirst() {
return first;
    }

public void setFirst(Text first) {
this.first = first;
    }

public Text getSecond() {
return second;
    }

public void setSecond(Text second) {
this.second = second;
    }
public void set(Text first,Text second){
this.first=first;
this.second=second;
    }

    @Override
public int hashCode() {
// TODO Auto-generated method stub
        return first.hashCode()*163+second.hashCode();
    }

    @Override
public boolean equals(Object obj) {
// TODO Auto-generated method stub
        if(obj instanceof TextPair){
            TextPair tp=(TextPair)obj;
return first.equals(tp.getFirst())&&second.equals(tp.getSecond());
        }
return false;
    }

    @Override
public String toString() {
// TODO Auto-generated method stub
        return first+"\t"+second;
    }

    @Override
public void readFields(DataInput in) throws IOException {
// TODO Auto-generated method stub
        first.readFields(in);
        second.readFields(in);
    }

    @Override
public void write(DataOutput out) throws IOException {
// TODO Auto-generated method stub
        first.write(out);
        second.write(out);
    }




    @Override
public int compareTo(TextPair tp) {
// TODO Auto-generated method stub
        int cmp=first.compareTo(tp.getFirst());
if(cmp!=0)
return cmp;
return second.compareTo(tp.getSecond());

    }


}

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Partitioner;


public class FirstPartitioner extends Partitioner<TextPair, Text> {

    @Override
public int getPartition(TextPair key, Text value, int numPartitions) {
// TODO Auto-generated method stub
        return Math.abs(key.getFirst().hashCode()&Integer.MAX_VALUE)%numPartitions;
    }

}

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;


public class GroupComparator extends WritableComparator {


public GroupComparator() {
super(TextPair.class, true);
// TODO Auto-generated constructor stub
    }


    @Override
public int compare(WritableComparable a, WritableComparable b) {
// TODO Auto-generated method stub
        TextPair p1=(TextPair)a;
        TextPair p2=(TextPair)b;
return p1.getFirst().compareTo(p2.getFirst());
    }


}

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Mapper;

public class DatadeduplicationMapper extends Mapper<LongWritable,Text,TextPair,Text>{

    @Override
protected void map(LongWritable key, Text value,
            org.apache.hadoop.mapreduce.Mapper.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
        System.out.println("mapper");
        String line=value.toString();
        StringTokenizer token=new StringTokenizer(line);
        String []str=new String[2];
int i=0;
        TextPair tp=new TextPair();
while(token.hasMoreTokens()){
            str[i]=token.nextToken();
            i++;
        }

        tp.set(new Text(str[0]), new Text(str[1]));
        System.out.println(tp);
        System.out.println(str[1]);
        context.write(tp, new Text(str[1]));

    }


}

import java.awt.List;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.Vector;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


public class DatadeduplicationReducer extends
        Reducer<TextPair, Text, Text, Text> {
    @Override  
protected void reduce(TextPair key, Iterable<Text> values,
            Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
        System.out.println("reducer");

        Set set=new HashSet();
        Vector<String>list=new Vector();

for(Text value:values){
//set.add(value.toString());
            if(!list.contains(value.toString())){
                list.add(value.toString());
            }
            System.out.println(key);
            System.out.println(value);
        }
for(int i=0;i<list.size();i++)
        {
//System.out.println(it.next());
            System.out.println((String)list.get(i));
            context.write(key.getFirst(), new Text(list.get(i).toString()));
        }
//    context.write(key, values)
    }

}

import java.io.IOException;



import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class Datadeduplication {

/**
     * @param args
     * @throws IOException 
     * @throws ClassNotFoundException 
     * @throws InterruptedException 
*/
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
// TODO Auto-generated method stub
        System.out.println("nihao");
        Job job=new Job();
        System.out.println("nihao");
        job.setJobName("Datadeduplication");
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.setMapperClass(DatadeduplicationMapper.class);
        job.setMapOutputKeyClass(TextPair.class);
        job.setMapOutputValueClass(Text.class);


        job.setPartitionerClass(FirstPartitioner.class);

        job.setGroupingComparatorClass(GroupComparator.class);
//job.setSortComparatorClass(FirstGroupingComparator.class);
//job.setSortComparatorClass(GroupComparator.class);
//job.setGroupingComparatorClass(KeyComparator.class);
        
        job.setReducerClass(DatadeduplicationReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);


        job.waitForCompletion(false);

    }

}

          

转载于:https://www.cnblogs.com/dlutxm/archive/2011/09/26/2192154.html