MapReduce的简单代码实现

需求：统计某个文档里面每个城市访问该网站的访客数

package mapperDemo1;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;


public class MapperDemo extends Configured implements Tool  {

  // Mapper方法
    public static class MapperDrv  extends Mapper<LongWritable,Text,Text,Text>{
        private Text outPutKey = new Text();
        private Text outPutValue = new Text();

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            String[] split = line.split("\t");
            if (split.length<36){
                return;
            }

            if (StringUtils.isBlank(split[24])){
                return;
            }
            else {
                if (StringUtils.isBlank(split[5])){
                    return;
                }
                else {
                   outPutKey.set(split[24]);
                   outPutValue.set(split[5]);
                   context.write(outPutKey,outPutValue);
                }

            }

        }
    }

  // Reduce方法
  // 这里注意hashset里面不能用hadoop里面的类型，否则会有错，我之前把hashset放到全局下，虽然程序没有出错，但是结果出错了，因为每次都把所有结果放到了hashset里面了
    public static class ReduceDiv extends Reducer<Text,Text,Text,IntWritable>{

        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            Set<String> hashSet = new HashSet<String>();
            for (Text value:values) {
               hashSet.add(value.toString());
            }
            int sum = hashSet.size();
            //outPutValue.set(hashSet.size());
            context.write(key,new IntWritable(sum));
        }
    }
    @Override
    public int run(String[] args) throws Exception {
        //job
        Job job = Job.getInstance(this.getConf(),"UVTEST");
        job.setJarByClass(MapperDemo.class);
        //input
        Path inPath = new Path(args[0]);
        FileInputFormat.setInputPaths(job,inPath);
        //mapper
        job.setMapperClass(MapperDrv.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        //shufflie，这里我还没有实现

        //reduce
        job.setReducerClass(ReduceDiv.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        job.setNumReduceTasks(1);

        Path path = new Path(args[1]);
        FileOutputFormat.setOutputPath(job,path);

        return job.waitForCompletion(true)?0:1;
    }

    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        try {
            int run = ToolRunner.run(configuration, new MapperDemo(), args);
            System.exit(run);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

然后将该项目打包成jar包，放到Linux里面，最后执行该jar包
输出结果：