Hadoop 使用MapReduce完成倒排索引

所需数据：

a.txt：

 welcome    tom
welcome    tomm
welcome    kitty
welcome    james
welcome    tom

b.txt

welcome    james
welcome    tom
welcome    kitty

c.txt

welcome    tom
welcome    tomm
welcome    james

 

结果需求：

james    a.txt->1b.txt->1c.txt->1
kitty    a.txt->1b.txt->1c.txt->0
tom    a.txt->2b.txt->1c.txt->1
tomm    a.txt->1b.txt->0c.txt->1
welcome    a.txt->5b.txt->3c.txt->3

 

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class Daoxu extends WritableComparator {



    static {
        System.setProperty("hadoop.home.dir","D:\\soft\\hadoop\\hadoop-2.9.2");
    }

    public static class MyMapper extends Mapper<LongWritable,Text,Text,Text> {
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            String s = value.toString();
            String FileName = ((FileSplit) context.getInputSplit()).getPath().getName();
            String[] split = s.split("\\s");
            if ("a.txt".equals(FileName)) {
                context.write(new Text(split[0]), new Text("a.txt"));
                context.write(new Text(split[1]), new Text("a.txt"));
            } else if ("b.txt".equals(FileName)) {
                context.write(new Text(split[0]), new Text("b.txt"));
                context.write(new Text(split[1]), new Text("b.txt"));
            } else if ("c.txt".equals(FileName)) {
                context.write(new Text(split[0]), new Text("c.txt"));
                context.write(new Text(split[1]), new Text("c.txt"));
            }
        }
    }
    public static class MyReducer extends Reducer<Text,Text,Text,Text> {
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            long a=0l;
            long b=0l;
            long c=0l;

            for (Text value:values){
                String s = value.toString();
                if ("a.txt".equals(s)){
                    a+=1;
                }else if ("b.txt".equals(s)){
                    b+=1;
                }else if ("c.txt".equals(s)){
                    c+=1;
                }
            }
            context.write(new Text(key),new Text("a.txt->"+a+"b.txt->"+b+"c.txt->"+c));
        }
    }

    public static void main(String[] args) throws Exception {
        //初始化一个作业
        Configuration conf = new Configuration();
        //给作业取个名字
        Job job = Job.getInstance(conf, "Three");
        //输入文件路径
        FileInputFormat.addInputPaths(job,args[0]);
        //map并行计算
        job.setMapperClass(MyMapper.class);
        //shuffle流程
        //reduce计算
        job.setReducerClass(MyReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        //输出文件路径
        FileOutputFormat.setOutputPath(job,new Path(args[1]));
        //判断文件是否存在
        FileSystem fs = FileSystem.get(conf);
        if (fs.exists(new Path(args[1]))){
            fs.delete(new Path(args[1]),true);
        }
        //判断是否成功
        boolean b = job.waitForCompletion(true);
        System.out.println(b ? 1 : 0);
    }

}