hadoop使用mapreduce统计词频_hadoop实现词频统计并排序

package sort;

import java.io.IOException;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Partitioner;

import org.apache.hadoop.mapreduce.Reducer;

import

org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import

org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;

import

org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

class WordsFrequenciesReducer extends Reducer {

@Override

protected void reduce(Text key, Iterable values,

Context context)

throws IOException, InterruptedException {

long counts = 0;

for (LongWritable value : values) {

counts += value.get();

}

context.write(key, new LongWritable(counts));

}

}

class WordsFrequenciesCombinerextends

org.apache.hadoop.mapreduce.Reducer{

@Override

protected void reduce(Text key, Iterable values,

Context context)

throws IOException, InterruptedException {

long counts = 0;

for (LongWritable value : values) {

counts += value.get();

}

context.write(key, new LongWritable(counts));

}

}

class WordsFrequenciesMapper extends Mapper {

@Override

protected void map(LongWritable key, Text value, Context

context)

throws IOException, InterruptedException {

String line = value.toString().trim();

if (line.length() == 0)

{

return;

}

String[] wordStrings = line.split("\\s+");

for (String word : wordStrings) {

context.write(new Text(word.toLowerCase()), new

LongWritable(1));

}

}

}

class WordsFrequenciesMapper2 extends Mapper {

@Override

protected void map(Text key, Text value, Context

context)

throws IOException, InterruptedException {

String newKey = String.format("0d",

Long.parseLong(value.toString())) + key.toString();

context.write(new Text(newKey), key);

}

}

class WordsFrequenciesPartitioner extends Partitioner {

@Override

public int getPartition(Text key, LongWritable value, int

numOfReducer) {

// 本例设置reducer个数为25,所以比如长度为26的单词会和长度为1的单词分配到同一个分区

return key.toString().length() % numOfReducer;

}

}

class WordsFrequenciesReducer2extends Reducer {

@Override

protected void reduce(Text key, Iterable values,

Context context)

throws IOException, InterruptedException

{

String newValueString = key.toString().substring(0,10);

context.write(values.iterator().next(), new

LongWritable(Long.parseLong(newValueString)));

}

}

public class WordsFrequenciesRunner extends Configured

implements Tool {

public static void main(String[] args) throws Exception

{

int exitCode = ToolRunner.run(new WordsFrequenciesRunner(),

args);

System.exit(exitCode);

}

@Override

public int run(String[] args) throws Exception {

if (args.length < 3) {

System.err.printf("Args missing. Input path and output path is

required.");

return -1;

}

Configuration conf = getConf();

// 设置数据文件分割大小。设置的数值越小，所需读mapper就越多。

//conf.setStrings("mapred.max.split.size", "20000");

Job job = new Job(conf, "sort");

// 设置reducer读个数。每个reducer最终会产生一个输出文件

job.setNumReduceTasks(25);

// 自定义分区类

// 本例中，长度相同的单词会被同一个reducer处理，最终也会出现在同一个输出文件中

job.setPartitionerClass(WordsFrequenciesPartitioner.class);

job.setJobName("Calculate temp words frequencies");

job.setJarByClass(getClass());

FileInputFormat.addInputPath(job, new Path(args[0]));

FileOutputFormat.setOutputPath(job, new Path(args[1]));

job.setMapperClass(WordsFrequenciesMapper.class);

job.setReducerClass(WordsFrequenciesReducer.class);

job.setCombinerClass(WordsFrequenciesCombiner.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(LongWritable.class);

int code = job.waitForCompletion(true) ? 0 : 1;

if (code == 0)

{

Configuration conf2 = getConf();

Job job2 = new Job(conf2, "sort");

job2.setNumReduceTasks(1);

job2.setJobName("Calculate final words frequencies");

job2.setJarByClass(getClass());

job2.setInputFormatClass(KeyValueTextInputFormat.class);

KeyValueTextInputFormat.addInputPath(job2, new

Path(args[1]));

FileOutputFormat.setOutputPath(job2, new Path(args[2]));

job2.setMapperClass(WordsFrequenciesMapper2.class);

job2.setReducerClass(WordsFrequenciesReducer2.class);

job2.setMapOutputKeyClass(Text.class);

job2.setMapOutputValueClass(Text.class);

job2.setOutputKeyClass(Text.class);

job2.setOutputValueClass(LongWritable.class);

code = job2.waitForCompletion(true) ? 0 : 1;

}

return code;

}

}