hadoop 初学

创建文件夹 hadoop fs -mkdir /input

hadoop fs -ls output

博主最近在学hadoop，而且在本实验室一位大神的指导下，我已配置好hadoop2.4.1开发环境，还没有配置或者不会配置的，请看链接http://dblab.xmu.edu.cn/blog/powerxing/install-hadoop-2-4-1-single-node/。由于之前运行的都是hadoop自带的实例，但是对于个人学习而言，肯定是要自己编写实现类以及编译运行实现类，因此博主就撰写了这篇文章，希望对学习hadoop的同道中人有所帮助。

• 编写实现类

首先在hadoop的根目录下新建一个工作目录workspace，即在/usr/local/hadoop执行命令

接着在workspace目录下编写WordCount.java类

WordCount.java代码如下，各位可以自己复制粘帖，在hadoop官方网站上也可以找到。

WordCount.java代码

Java

import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class WordCount { public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{ private final static IntWritable one = new IntWritable(1); private Text word = new Text(); public void map(Object key, Text value, Context context ) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(value.toString()); while (itr.hasMoreTokens()) { word.set(itr.nextToken()); context.write(word, one); } } } public static class IntSumReducer extends Reducer<Text,IntWritable,Text,IntWritable> { private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable<IntWritable> values, Context context ) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } result.set(sum); context.write(key, result); } } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); } }

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61

import java . io . IOException ; import java . util . StringTokenizer ;   import org . apache . hadoop . conf . Configuration ; import org . apache . hadoop . fs . Path ; import org . apache . hadoop . io . IntWritable ; import org . apache . hadoop . io . Text ; import org . apache . hadoop . mapreduce . Job ; import org . apache . hadoop . mapreduce . Mapper ; import org . apache . hadoop . mapreduce . Reducer ; import org . apache . hadoop . mapreduce . lib . input . FileInputFormat ; import org . apache . hadoop . mapreduce . lib . output . FileOutputFormat ;   public class WordCount {      public static class TokenizerMapper        extends Mapper < Object , Text , Text , IntWritable > {        private final static IntWritable one = new IntWritable ( 1 ) ;      private Text word = new Text ( ) ;        public void map ( Object key , Text value , Context context                      ) throws IOException , InterruptedException {        StringTokenizer itr = new StringTokenizer ( value . toString ( ) ) ;        while ( itr . hasMoreTokens ( ) ) {          word . set ( itr . nextToken ( ) ) ;          context . write ( word , one ) ;        }      }    }      public static class IntSumReducer        extends Reducer < Text , IntWritable , Text , IntWritable > {      private IntWritable result = new IntWritable ( ) ;        public void reduce ( Text key , Iterable <IntWritable> values ,                        Context context                        ) throws IOException , InterruptedException {        int sum = 0 ;        for ( IntWritable val : values ) {          sum += val . get ( ) ;        }        result . set ( sum ) ;        context . write ( key , result ) ;      }    }      public static void main ( String [ ] args ) throws Exception {      Configuration conf = new Configuration ( ) ;      Job job = Job . getInstance ( conf , "word count" ) ;      job . setJarByClass ( WordCount . class ) ;      job . setMapperClass ( TokenizerMapper . class ) ;      job . setCombinerClass ( IntSumReducer . class ) ;      job . setReducerClass ( IntSumReducer . class ) ;      job . setOutputKeyClass ( Text . class ) ;      job . setOutputValueClass ( IntWritable . class ) ;      FileInputFormat . addInputPath ( job , new Path ( args [ 0 ] ) ) ;      FileOutputFormat . setOutputPath ( job , new Path ( args [ 1 ] ) ) ;      System . exit ( job . waitForCompletion ( true ) ? 0 : 1 ) ;    } }

• 编译运行

接着将该WordCount.java文件编译为class文件，在编译之前，还需添加以下环境变量

1.首先添加JAVA_HOME，因为我用的是系统自带的jdk，所以路径如下图所示：

2.将jdk目录下的bin文件夹添加到环境变量：

3.接着将hadoop_classpath添加到环境变量：

执行上述步骤后，即可开始编译WordCount.java文件，编译java文件的命令为javac，截图如下：

此时，在workspace文件夹下将会出现生成三个class文件，

编译成功后，即可将三个class文件打包成jar文件，

执行成功后，在workspace文件下生成了WordCount.jar文件，

接着，在/usr/local/hadoop文件夹下新建一个input文件夹，用于存放数据，

接着cd 到input文件下，执行以下命令，就是将’Hello World Bye World’写进file01文件，将’Hello Hadoop Goodbye Hadoop’ 写进file02文件

最后运行程序，

期间可以到程序执行的过程，

最后，将output目录下的文件输出，即可到程序运行的结果，

到此，编写一个mapreduce实现类以及编译运行就成功实现了。

注：每次运行程序之前，要将之前运行的产生的output删除，不然程序会报错！