mr 实现 显示一个字段在各个文件中的数量

a.txt

 hello tom

hello jerry

b.txt

hello jerry
hello jerry
tom jerry

c.txt

hello jerry
hello tom

最终结果 hello a.txt 2  b.txt 2  c.txt  2

思路 将word+filename 作为一个key传入 经过mrchuli得下面的文件

得到文件名

//获取文件名
FileSplit inputSplit = (FileSplit) context.getInputSplit();
String fileName = inputSplit.getPath().getName();

hello--a.txt    2
hello--b.txt    2
hello--c.txt    2
jerry--a.txt    1
jerry--b.txt    3
jerry--c.txt    1
tom--a.txt    1
tom--b.txt    1
tom--c.txt    1

再经过一次mr处理分割计数就可以的到结果了

map切分，reduce合并计数等

map分割按-- 分割后，reduce stringbuffer添加输出

一个mr不行就用两个

package com.zz.bd;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.net.URISyntaxException;

public class InverseIndex {
    static class mapp extends Mapper<LongWritable, Text, Text, IntWritable> {
        Text text = new Text();
        IntWritable intWritable = new IntWritable();

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            String[] words = line.split(" ");
            // FileSplit inputSplit = (FileSplit) context.getInputSplit();
            //获取文件名
            FileSplit inputSplit = (FileSplit) context.getInputSplit();
            String fileName = inputSplit.getPath().getName();
            for (String word : words) {
                text.set(word + "--" + fileName);
                intWritable.set(1);
                context.write(text, intWritable);
            }

        }
    }

    static class reducee extends Reducer<Text, IntWritable, Text, LongWritable> {
        LongWritable longWritable = new LongWritable();

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            long count = 0;
            for (IntWritable value : values) {
                count += value.get();
            }
            longWritable.set(count);
            context.write(key, longWritable);
        }
    }

    public static void main(String[] args) throws URISyntaxException, InterruptedException, IOException, ClassNotFoundException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(InverseIndex.class);

        job.setMapperClass(mapp.class);
        job.setReducerClass(reducee.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        FileInputFormat.setInputPaths(job, new Path("/home/lzq/input/"));
        FileOutputFormat.setOutputPath(job, new Path("/home/lzq/output1"));

        boolean b = job.waitForCompletion(true);
        System.exit(b ? 0 : 1);
    }
}

第二次mr

package com.zz.bd;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.net.URISyntaxException;

public class InverseSeconIndex {
    static class mapp extends Mapper<LongWritable, Text, Text, Text> {
        Text text = new Text();

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            String[] fields = line.split("--");
            context.write(new Text(fields[0]), new Text(fields[1]));
        }
    }

    static class reducee extends Reducer<Text, Text, Text, Text> {

        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            StringBuffer sb = new StringBuffer();
            for (Text v : values) {
                sb.append(v.toString()+"\t");
            }
            context.write(key, new Text(String.valueOf(sb)));
        }


        public static void main(String[] args) throws URISyntaxException, InterruptedException, IOException, ClassNotFoundException {
            Configuration conf = new Configuration();
            Job job = Job.getInstance(conf);

            job.setJarByClass(InverseSeconIndex.class);

            job.setMapperClass(InverseIndex.mapp.class);
            job.setReducerClass(InverseIndex.reducee.class);

            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(IntWritable.class);

            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(LongWritable.class);

            FileInputFormat.setInputPaths(job, new Path("/home/lzq/input/"));
            FileOutputFormat.setOutputPath(job, new Path("/home/lzq/output1"));

            boolean b = job.waitForCompletion(true);
            System.exit(b ? 0 : 1);
        }
    }
}