关于在idea运行本地MapReduce的问题

在idea运行一个很普通的wordcount

所需环境,反正也不知道是不是这样,就都配了

 

没配之前一直报错

Could not locate executabl ....\hadoop-2.7.3\hadoop-2.7.3\bin\winutils.exe in the Hadoop binaries

配置完毕之后重启电脑就OK了.

下面是pom依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>MapReduce</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>

    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.3</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.16.18</version>
        </dependency>

    </dependencies>



</project>

完整代码

package com.czx.mr;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import java.io.IOException;

/**
 * Created with IntelliJ IDEA.
 *
 * @Author: czx
 * @Date: 2021/12/29/12:13
 * @Description:
 */
public class WordCount {
    public static class WordCountMapper extends Mapper<LongWritable, Text, Text, LongWritable>{
        /**
         * 单词
         */
        Text keyOut = new Text();

        /**
         * 数值
         */
        LongWritable valueOut = new LongWritable();

        @Override
        protected void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            System.out.println("mapper input==>" + key.get() + ", " + value.toString());
            // 获取一行的字符串
            // one world
            String line = value.toString();

            // one, world
            String[] arr = line.split(" ");
            for(String word : arr){
                keyOut.set(word);
                valueOut.set(1);
                // 通过该方法输出数据  one,1
                context.write(keyOut, valueOut);
                System.out.println("mapper output==>" + keyOut.toString() + ", " + valueOut.get());
            }

        }
    }
    /*
     * reducer的KEYIN, VALUEIN 和 mapper输出的  KEYOUT, VALUEOUT 类型一致
     * KEYIN：单词 ----》String类型-----》 Hadoop自带的Text, 里面是String类型
     * VALUEIN：数值 ---》 long类型存 -----》 Hadoop自带的 LongWritable, 里面是long类型
     *
     *
     * KEYOUT, VALUEOUT
     * reducer输出啥取决于业务需求
     * 统计单词的数量
     * KEYOUT：单词 ----》String类型-----》 Hadoop自带的Text, 里面是String类型
     * VALUEOUT：数值---》 long类型存 -----》 Hadoop自带的 LongWritable, 里面是long类型
     *
     */
    public static class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable>{
        LongWritable valueOut = new LongWritable();
        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
            // one, [1,1,1,1]  ---> one,4
            long sum = 0L;

            StringBuilder sb = new StringBuilder("reducer input==>");
            sb.append(key).append(", [");
            for(LongWritable w : values){
                sb.append(w.get()).append(",");
                sum += w.get();
            }

            sb.deleteCharAt(sb.length()-1).append("]");
            System.out.println(sb.toString());

            valueOut.set(sum);
            context.write(key, valueOut);

            System.out.println("reducer output==>" + key + ", " + sum);
        }
    }
    // /tmp/mr/input /tmp/mr/output
    public static void main(String[] args) throws Exception {
        // 加载 core-default.xml 和 core-site.xml
        Configuration conf = new Configuration();
        // 创建运行mapreduce任务的Job对象
        Job job = Job.getInstance(conf, "wordcount");
        // 设置运行的类(linux 运行用)
        job.setJarByClass(WordCount.class);

        // 设置mapperclass
        job.setMapperClass(WordCountMapper.class);
        // 设置reducerclass
        job.setReducerClass(WordCountReducer.class);

        // 设置reducer个数， 不设置默认是1
        job.setNumReduceTasks(1);

        // 设置mapper输出keyclass
        job.setMapOutputKeyClass(Text.class);
        // 设置mapper输出valueclass
        job.setMapOutputValueClass(LongWritable.class);
        // 设置reducer输出keyclass
        job.setOutputKeyClass(Text.class);
        // 设置reducer输出的valueclass
        job.setOutputValueClass(LongWritable.class);

        // 设置读取的输入文件的inputformatclass，默认是文本，可以不设置
        job.setInputFormatClass(TextInputFormat.class);
        // 设置写入文件的outputformatclass，默认是文本，可以不设置
        job.setOutputFormatClass(TextOutputFormat.class);

        // 设置输入目录
        FileInputFormat.addInputPath(job, new Path(args[0]));

        Path outputPath = new Path(args[1]);
        // 设置输出目录
        FileOutputFormat.setOutputPath(job, outputPath);

        // 自动删除输出目录
        FileSystem fs = FileSystem.get(conf);
        // 如果输出目录存在，就递归删除输出目录
        if(fs.exists(outputPath)){
            // 递归删除输出目录
            fs.delete(outputPath, true);
            System.out.println("delete outputPath==> 【" + outputPath.toString() + "】 success!");
        }

        // 提交job
        boolean status = job.waitForCompletion(false);

        System.exit(status ? 0 : 1);

    }
}

运行代码

设置参数

运行结果