Hadoop部署及运行

开启hadoop

1、运行cmd窗口，执行“hdfs namenode -format”
2、子hadoop的sbin目录，执行“start-all.cmd”
此时hadoop服务器已开启

---

操作HDFS

我们来创建输入目录（创建目录要确保服务器已开启状态才行）

hadoop fs -mkdir hdfs://localhost:9000/user/
hadoop fs -mkdir hdfs://localhost:9000/user/wcinput

上传文件到目录

hadoop fs -put D:\Study_soft\file1.txt hdfs://localhost:9000/user/wcinput

hadoop fs -put D:\Study_soft\file2.txt hdfs://localhost:9000/user/wcinput

查看文件

hadoop fs -ls hdfs://localhost:9000/user/wcinput

文件上传成功

在eclipse连接hadoop

1、填写连接的参数

连接成功

注意：如果连接时出现An internal error occurred during: “Map/Reducelocation status updater”.java.lang.NullPointerException，是因为配置部署的Hadoop还没创建输入和输出目录，所以导致空指针

2、创建Map/Redure Project，右键 –> New –> Other –> Map/Redure Project

package hadoop;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.GenericOptionsParser;


public class WordCount {
    // 继承Mapper接口，设置map的输入类型为<Object,Text>
    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
        // one表示单词出现一次
        private final static IntWritable one = new IntWritable(1);
        // word存储切换下的单词
        private Text word = new Text();

        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());// 对输入的切换词
            while (itr.hasMoreTokens()) {
                word.set(itr.nextToken());// 切下的单词存入word
                context.write(word, one);
            }
        }
    }

    // 继承Reduce接口，设置reduce的输入类型<Text，IntWritable>
    // 输出类型为<Text，IntWritable>
    public static class IntSumReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
        // result记录单词的频数
        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<IntWritable> values, Context context)
                throws IOException, InterruptedException {
            int sum = 0;
            // 对获取的<key,value-list>计算value的和
            for (IntWritable val : values) {
                sum += val.get();
            }
            // 将频数设置到result中
            result.set(sum);
            // 收集结果
            context.write(key, result);
        }

    }

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
       Configuration conf=new Configuration();
       //检查运行命令
      String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();
      if(otherArgs.length!=2){
          System.err.println("Usage:wordcount<int><out>");
          System.exit(2);
      }
      Job job=new Job(conf,"word count");
      //配置作业各个类
      job.setJarByClass(WordCount.class);
      job.setMapperClass(TokenizerMapper.class);
      job.setCombinerClass(IntSumReduce.class);
      job.setReducerClass(IntSumReduce.class);
      job.setOutputKeyClass(Text.class);
      job.setOutputValueClass(IntWritable.class);
      FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
      FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
      System.exit(job.waitForCompletion(true)?0:1);
    }
}

3、点击WordCount类，右键Run As –> Run Configurations ，点击Arguments，填写输入目录，输出目录参数

hdfs://localhost:9000/user/wcinput
hdfs://localhost:9000/user/wcoutput

4、运行
输出结果

成功了。。。