Java/Hadoop wordcount

public class FileWordCount {

public void count() throws IOException{

BufferedReader reader = new BufferedReader(new FileReader("D:\\test\\english.txt"));

StringBuffer buffer = new StringBuffer();

String line = null;

while( (line = reader.readLine()) != null ){

buffer.append(line); }

reader.close();

Pattern expression = Pattern.compile("[a-zA-Z]+"); //定义正则表达式匹配单词

String string = buffer.toString();

Matcher matcher = expression.matcher(string);

Map<string integer=""> map = new TreeMap<string integer="">();

String word = "";

int n = 0;

Integer times = 0;

while(matcher.find()){

//是否匹配单词 word = matcher.group();

//得到一个单词，树映射中的键 n++;

if( map.containsKey(word) ){

//如果该键存在，则表示单词出现过 times = map.get(word);

//得到单词出现的次数 map.put(word, times+1);

} else {

map.put(word, 1);

//否则单词是第一次出现，直接放入map

}

}

List<Map.Entry<string integer="">> list = new ArrayList<Map.Entry<string integer="">>(map.entrySet()); Comparator<Map.Entry<string integer="">> comparator = new Comparator<Map.Entry<string integer="">>(){

@Override public int compare(Entry<string integer=""> left, Entry<string integer=""> right) {

return (left.getValue()).compareTo(right.getValue());

}

};

Collections.sort(list, comparator);

System.out.println("统计分析如下：");

System.out.println("t 文章中单词总数" + n + "个");

System.out.println("具体的信息在原文件目录的result.txt文件中");

BufferedWriter bufw = new BufferedWriter(new FileWriter("D:\\test\\result.txt"));

for(Entry<string integer=""> me : list){

bufw.write(me+"");

bufw.newLine();

}

bufw.write("english.txt中的单词总数" + n + "个");

 

public class WordCount {

//输入键类型、输入值类型、输出键类型和输出值类型，在本例中分别为普通对象（Object）、字符串（Text）、字符串和整数型（IntWritable） public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {

private final static IntWritable one = new IntWritable(1);

private Text word = new Text();

//输入键key、输入值value和环境变量context。输入键默认为行号，输入值为每一行的文本字符串

public void map(Object key, Text value, Context context) throws IOException, InterruptedException {

StringTokenizer itr = new StringTokenizer(value.toString());

while (itr.hasMoreTokens()) {

word.set(itr.nextToken());

//设置键的值

context.write(word, one);

//函数context.write()输出键为单词值为11的键值对

}

}

}

//输入键类型、输入值类型、输出键类型和输出值类型，在本例中分别为字符串、整数型、字符串和整数型

public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

private IntWritable result = new IntWritable();

//输入键key、输入值列表values和环境变量

context public void reduce(Text key, Iterable<IntWritable> values, Context context)throws IOException, InterruptedException {

int sum = 0;

for (IntWritable val : values) {

sum += val.get();

}

result.set(sum);

context.write(key, result);

//输出键为单词值为单词出现次数的键值对

}

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();

//初始化hadoop集群组件配置

String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();

//解析命令行参数

if (otherArgs.length < 2) {

System.err.println("Usage: wordcount <in> [<in>...] <out>");

System.exit(2);

}

@SuppressWarnings("deprecation")

Job job = new Job(conf, "word count");

//Job命名为"word count"

job.setJarByClass(WordCount.class);

//设置主类 job.setMa