自己编写的WordCound如何在Linux运行

最新推荐文章于 2021-02-01 09:35:38 发布

qq_42039996

最新推荐文章于 2021-02-01 09:35:38 发布

阅读量124

点赞数

本文链接：https://blog.csdn.net/qq_42039996/article/details/83592186

版权

首先写上自己的代码，在eclipse编译完成

package com.jie;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {

    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{

        private final IntWritable one = new IntWritable(1);

        private Text word = new Text();

        @Override
        public void map(Object key, Text value, Context context
                ) throws IOException, InterruptedException {
          String str = value.toString();
          int len = str.length();
          for (int i = 0; i < len; i++) {
              char c = str.charAt(i);
              if(!Character.isWhitespace(c)){
                  word.set(String.valueOf(c));
                  context.write(word, one);
              }
          }
        }

    }

    public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable>{

        private IntWritable result = new IntWritable();

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values,
                Context context) throws IOException, InterruptedException {
            int sum = 0;
            for (IntWritable one : values) {
                sum += one.get();
            }
            result.set(sum);
            context.write(key, result);
        }

    }

    public static void main(String[] args) throws Exception{

        Configuration cfg = new Configuration();
        if(args.length < 2){
            System.err.println("Usage: <in> <out>");
            System.exit(2);
        }

        Job job = Job.getInstance(cfg, "myCharCount");
        job.setJarByClass(WordCount.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumReducer.class);
        job.setReducerClass(IntSumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1 );

    }

}

编译完之后打包

上传文件后

运行

hadoop jar hp.jar /usr/test/word /usr/test/output