SpringBoot下YarnClient使用

最新推荐文章于 2024-08-12 03:15:40 发布

仙道Bob

最新推荐文章于 2024-08-12 03:15:40 发布

阅读量3.9k

点赞数 4

分类专栏： # yarn # spark

本文链接：https://blog.csdn.net/jsbylibo/article/details/109286147

版权

yarn 同时被 2 个专栏收录

2 篇文章

订阅专栏

spark

2 篇文章

订阅专栏

本文介绍如何使用Java通过YARN API查询并终止运行中的任务。包括配置Maven依赖、编写Java代码实现任务查询与终止功能，以及环境配置说明。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1.pom.xml

<properties>
        <!-- hadoop version-->
        <hadoop.version>2.6.0</hadoop.version>
    </properties>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <!-- hadoop yarn api -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-api</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <!-- hadoop yarn client -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-yarn-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

2.代码

package com.**.dfp.rtsync.task;

import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;

import java.io.IOException;
import java.util.EnumSet;
import java.util.List;

/**
 * Description
 *
 * @author Bob
 * @date 2020/10/22
 **/
@Slf4j
public class YarnTest {
    public static void main(String[] args){
        Configuration conf = new YarnConfiguration();
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(conf);
        yarnClient.start();
        try {
            //查询运行状态所有任务
            List<ApplicationReport> applicationReportList = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
            for (ApplicationReport applicationReport : applicationReportList) {
                // 获取Name
                String name = applicationReport.getName();
                // 获取ApplicationType
                String applicationType = applicationReport.getApplicationType();
                ApplicationId applicationId;
                log.info("-----------applicationId:{},name:{},quene:{},user:{},type:{}", applicationReport.getApplicationId(), applicationReport.getName(), applicationReport.getQueue(), applicationReport.getUser(), applicationReport.getApplicationType());
                // 检查是否实时集成任务（Apache Spark）
                if (name.equals(applicationName)) {
                    if ("Apache Spark".equalsIgnoreCase(applicationType) || "Spark".equalsIgnoreCase(applicationType)) {
                        // ApplicationId
                        applicationId = applicationReport.getApplicationId();
                        // Kill applicationId
                        //yarnClient.killApplication(applicationId);
                        log.info("==========applicationId:{} is killed!", applicationName);
                        break;
                    } else {
                        log.warn("The app {} is not valid spark job! ", applicationName);
                    }
                }
            }
        } catch (YarnException | IOException e) {
            log.error("kill killYarnApplication error:", e);
            throw new BusinessException("停止Yarn任务失败!");
        } finally {
            if (yarnClient != null) {
                try {
                    // 停止YarnClient
                    yarnClient.stop();
                    // 关闭YarnClient
                    yarnClient.close();
                } catch (IOException e) {
                    log.error("关闭Yarn Client失败！", e);
                }
            }
        }
    }
}

3.yarn-site.xml

注意：

方式一：将yarn-site.xml文件放到resources目录下。

方式二：指定yarn-site.xml，代码如下：

Configuration conf = new YarnConfiguration();
conf.addResource(new File("D:\spark\hadoop-2.6.0\yarn-site.xml").toURI().toURL());
yarnClient.init(conf);