Yarn_Tool 接口案例

最新推荐文章于 2024-10-17 09:33:14 发布

你是厉害的！！

最新推荐文章于 2024-10-17 09:33:14 发布

阅读量84

点赞数

分类专栏： Hadoop 文章标签：开发语言 yarn hadoop

本文链接：https://blog.csdn.net/weixin_61959079/article/details/131394087

版权

Hadoop 专栏收录该内容

9 篇文章 0 订阅

订阅专栏

文章介绍了如何在Hadoop环境中，通过实现Yarn的Tool接口和使用Maven构建项目，实现WordCount程序动态传参并运行。主要步骤包括配置pom.xml，编写WordCount类、Mapper和Reducer，以及WordCountDriver，通过ToolRunner运行程序，允许动态修改集群参数。

摘要由CSDN通过智能技术生成

回顾：

Hadoop运行wordcount的jar程序代码如下
hadoop jar wc.jar com/hadoop/mapreduce/wordcount/WordCountDriver /input /output

1、需求

可以动态的传参，而命令行运行代码不报错

2、需求分析

分析Yarn Tool接口的需求，并提供一个案例，说明如何使用该接口来动态修改集群参数。我们希望解决的主要痛点是，在不需要重启服务的情况下，能够动态修改集群参数。

3、代码实现

（1）配置pom.xml文件

新建 Maven 项目 YarnDemo ， pom 如下：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.hadoop</groupId>
    <artifactId>YarnDemo</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.1.3</version>
        </dependency>
    </dependencies>


</project>

（2）WordCount类

package com.hadoop.yarn;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;

import java.io.IOException;

/**
 * @author codestart
 * @create 2023-06-26 10:31
 */
public class WordCount implements Tool {

    private Configuration conf;

    //核心驱动信息（conf需要传入）
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(conf);

        //2、配置驱动jar位置
        job.setJarByClass(WordCountDriver.class);

        //3、关联Mappr和Reducer
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReducer.class);

        //4、Map的输出K-V
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        //5、最后输出的K-v
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //6、路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        //7、提交
        return job.waitForCompletion(true) ? 0 : 1;
    }

    @Override
    public void setConf(Configuration conf) {
        this.conf = conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }

    //Mapper类
    public static class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        private Text outK = new Text();
        private IntWritable outV = new IntWritable(1);

        @Override
        protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context) throws IOException, InterruptedException {
            //读取一行
            String line = value.toString();
            //切割
            String[] split = line.split(" ");
            //封装
            for (String s : split) {
                outK.set(s);

                //写入
                context.write(outK, outV);
            }
        }
    }

    //Reducer类
    public static class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

        private IntWritable outV = new IntWritable();

        //Reducer 方法
        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
            //求和
            int sum = 0;
            for (IntWritable value : values) {
                sum += value.get();
            }
            //封装
            outV.set(sum);
            //写入
            context.write(key, outV);
        }
    }
}

（3）WordCountDriver类

package com.hadoop.yarn;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.util.Arrays;


/**
 * @author codestart
 * @create 2023-06-26 11:03
 */
public class WordCountDriver {
    private static Tool tool;

    public static void main(String[] args) throws Exception {
        //处理tool接口的处理信息

        //创建配置
        Configuration conf = new Configuration();

        //判断是否有tool接口
        switch (args[0]){
            case "wordcount":
                tool = new WordCount();
                break;
            default:
                throw new RuntimeException("No such tool:" + args[0]);

        }

        //执行程序
        int b = ToolRunner.run(conf, tool, Arrays.copyOfRange(args, 1, args.length));
        System.exit(b);

    }
}