MapReduce--词频统计

 

 

复制一段：

a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e
a,v,s,adg,gw,q3,fga,a,a,eg,e

 

---

导入依赖

 

​​​​​​​<dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.6</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.10</version>
    </dependency>
</dependencies>

 

编写map reduce代码

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


import java.io.IOException;

public class tongj {
    public static void main(String[] args) {
        class map extends Mapper<LongWritable,Text,Text,IntWritable> {
            @Override
            protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
                String data = value.toString();
                String[] datas = data.split(",");
                for (int i=0;i<datas.length;i++){
                    context.write(new Text(datas[i]),new IntWritable(1));
                }

            }
        }
        class reduce extends Reducer<Text,IntWritable,Text,IntWritable>{
            @Override
            protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
                int count = 0;
                for (IntWritable su:values){
                    count +=su.get();

                }
                context.write(key,new IntWritable(count));
            }
        }
        if (args.length<2){
            System.exit(0);
        }
        try {
            //Configuration：map/reduce的j配置类，向hadoop框架描述map-reduce执行的工作
            Configuration conf = new Configuration();
            Job job=Job.getInstance(conf);
            job.setJobName("text");//设置一个用户定义的job名称
            job.setJarByClass(tongj.class);//加载主类
            //为job设置输入路径
            FileInputFormat.addInputPath(job,new Path(args[0]));
            //为job设置输出路径
            FileOutputFormat.setOutputPath(job,new Path(args[1]));
            job.setOutputKeyClass(Text.class);//为job的输出数据设置Key类
            job.setOutputValueClass(IntWritable.class); //为job输出设置value类

            job.setMapperClass(map.class);//为job设置Mapper类
            job.setReducerClass(reduce.class);//为job设置Reducer类

            System.exit(job.waitForCompletion(true)?0:1);//运行job
        }catch (Exception ex){

            ex.printStackTrace();
        }

    }
}

 

打包，运行

hadoop jar mapreduce-1.0-SNAPSHOT.jar tongj data/* ss/

结果

[root@b1 ~]# hdfs dfs -cat ss/*
a	27
adg	9
e	9
eg	9
fga	9
gw	9
q3	9
s	9
v	9