这里我们写一个更全面的WordCount例子,它使用了我们已经讨论过的很多Map/Reduce框架提供的功能。运行这个例子需要HDFS的某些功能,特别是 DistributedCache相关功能。因此这个例子只能运行在 伪分布式 或者 完全分布式模式的 Hadoop上。
同样下面的例子的先决条件是Hadoop被正确安装、配置和正常运行中。
1.1 源码
1. package org.myorg;
3. import java.io.*;
4. import java.util.*;
6. import org.apache.hadoop.fs.Path;
7. import org.apache.hadoop.filecache.DistributedCache;
8. import org.apache.hadoop.conf.*;
9. import org.apache.hadoop.io.*;
10. import org.apache.hadoop.mapred.*;
11. import org.apache.hadoop.util.*;
13. public class WordCount extends Configured implements Tool {
15. public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
17. static enum Counters { INPUT_WORDS }
19. private final static IntWritable one = new IntWritable(1);
20. private Text word = new Text();
22. private boolean caseSensitive = true;
23. private Set<String> patternsToSkip = new HashSet<String>();
25. private long numRecords = 0;
26. private String inputFile;
28. public void configure(JobConf job) {
29. caseSensitive = job.getBoolean("wordcount.case.sensitive", true);
30. inputFile = job.get("map.input.file");
32. if (job.getBoolean("wordcount.skip.patterns", false)) {
33. Path[] patternsFiles = new Path[0];
34. try {
35. patternsFiles = DistributedCache.getLocalCacheFiles(job);
36. } catch (IOException ioe) {
37. System.err.println("Caught exception while getting cached files: " + StringUtils.stringifyException(ioe));
38. }
39. for (Path patternsFile : patternsFiles) {
40. parseSkipFile(patternsFile);
41. }
42. }
43. }
45. private void parseSkipFile(Path patternsFile) {
46. try {
47 BufferedReader fis = new BufferedReader(new FileReader(patternsFile.toString()));
48. String pattern = null;
49. while ((pattern = fis.readLine()) != null) {
50. patternsToSkip.add(pattern);
51. }
52. } catch (IOException ioe) {
53. System.err.println("Caught exception while parsing the cached file '" + patternsFile + "' : " + StringUtils.stringifyException(ioe));
54. }
55. }
57. public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
58. String line = (caseSensitive) ? value.toString() : value.toString().toLowerCase();
60. for (String pattern : patternsToSkip) {
61. line = line.replaceAll(pattern, "");
62. }
64. StringTokenizer tokenizer = new StringTokenizer(line);
65. while (tokenizer.hasMoreTokens()) {
66. word.set(tokenizer.nextToken());
67. output.collect(word, one);
68. reporter.incrCounter(Counters.INPUT_WORDS, 1);
69. }
71. if ((++numRecords % 100) == 0) {
72. reporter.setStatus("Finished processing " + numRecords + " records " + "from the input file: " + inputFile);
73. }
74. }
75. }
77. public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
78. public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
79. int sum = 0;
80. while (values.hasNext()) {
81. sum += values.next().get();
82. }
83. output.collect(key, new IntWritable(sum));
84. }
85. }
87. public int run(String[] args) throws Exception {
88. JobConf conf = new JobConf(getConf(), WordCount.class);
89. conf.setJobName("wordcount");
91. conf.setOutputKeyClass(Text.class);
92. conf.setOutputValueClass(IntWritable.class);
94. conf.setMapperClass(Map.class);
95. conf.setCombinerClass(Reduce.class);
96. conf.setReducerClass(Reduce.class);
98. conf.setInputFormat(TextInputFormat.class);
99. conf.setOutputFormat(TextOutputFormat.class);
101. List<String> other_args = new ArrayList<String>();
102. for (int i=0; i < args.length; ++i) {
103. if ("-skip".equals(args[i])) {
104. DistributedCache.addCacheFile(new Path(args[++i]).toUri(), conf);
105. conf.setBoolean("wordcount.skip.patterns", true);
106. } else {
107. other_args.add(args[i]);
108. }
109. }
111. FileInputFormat.setInputPaths(conf, new Path(other_args.get(0)));
112. FileOutputFormat.setOutputPath(conf, new Path(other_args.get(1)));
114. JobClient.runJob(conf);
115. return 0;
116. }
118. public static void main(String[] args) throws Exception {
119. int res = ToolRunner.run(new Configuration(), new WordCount(), args);
120. System.exit(res);
121. }
122. }
1.2运行样例
输入样例:
$ bin/hadoop dfs -ls /usr/joe/wordcount/input/
/usr/joe/wordcount/input/file01
/usr/joe/wordcount/input/file02
$ bin/hadoop dfs -cat /usr/joe/wordcount/input/file01
Hello World, Bye World!
$ bin/hadoop dfs -cat /usr/joe/wordcount/input/file02
Hello Hadoop, Goodbye to hadoop.
运行程序:
$ bin/hadoop jar /usr/joe/wordcount.jar org.myorg.WordCount /usr/joe/wordcount/input /usr/joe/wordcount/output
输出:
$ bin/hadoop dfs -cat /usr/joe/wordcount/output/part-00000
Bye 1
Goodbye 1
Hadoop, 1
Hello 2
World! 1
World, 1
hadoop. 1
to 1
注意此时的输入与第一个版本的不同,输出的结果也有不同。
现在通过DistributedCache插入一个模式文件,文件中保存了要被忽略的单词模式。
$ hadoop dfs -cat /user/joe/wordcount/patterns.txt
\.
\,
\!
to
再运行一次,这次使用更多的选项:
$ bin/hadoop jar /usr/joe/wordcount.jar org.myorg.WordCount -Dwordcount.case.sensitive=true /usr/joe/wordcount/input /usr/joe/wordcount/output -skip /user/joe/wordcount/patterns.txt
应该得到这样的输出:
$ bin/hadoop dfs -cat /usr/joe/wordcount/output/part-00000
Bye 1
Goodbye 1
Hadoop 1
Hello 2
World 2
hadoop 1
再运行一次,这一次关闭大小写敏感性(case-sensitivity):
$ bin/hadoop jar /usr/joe/wordcount.jar org.myorg.WordCount -Dwordcount.case.sensitive=false /usr/joe/wordcount/input /usr/joe/wordcount/output -skip /user/joe/wordcount/patterns.txt
输出:
$ bin/hadoop dfs -cat /usr/joe/wordcount/output/part-00000
bye 1
goodbye 1
hadoop 2
hello 2
world 2
1.3程序要点
通过使用一些Map/Reduce框架提供的功能,WordCount的第二个版本在原始版本基础上有了如下的改进:
- 展示了应用程序如何在Mapper (和Reducer)中通过configure方法 修改配置参数(28-43行)。
- 展示了作业如何使用DistributedCache 来分发只读数据。这里允许用户指定单词的模式,在计数时忽略那些符合模式的单词(104行)。
- 展示Tool接口和GenericOptionsParser处理Hadoop命令行选项的功能 (87-116, 119行)。
- 展示了应用程序如何使用Counters(68行),如何通过传递给map(和reduce)方法的Reporter实例来设置应用程序的状态信息(72行)。