yarn使用笔记

一、 REST API

1. 相关参考

YARN REST API 总结-阿里云开发者社区

Apache Hadoop 3.3.6 – Hadoop YARN - Introduction to the web services REST APIs

2. python获取应用信息

import requests
from functools import reduce
import json
import os
     

# 激活状态的app信息
def get_apps(yarn_url):
    # 访问URL
    url = "http://%s:8088/ws/v1/cluster/apps"%(yarn_url)
    print('get_apps访问url:' + url)
    # 参数
    params = {
        #"minEndDate": min_end_date,
        #"maxEndDate": max_end_date,
        "state": "running",
        #"limit": "3",
    }
    return  requests.get(url, params = params, timeout = 30).json()['apps']['app']


# 格式化并过滤app
def format_filter_apps(apps, timeout):
    format_apps = []
    for app in apps:
        if app['state'] == 'RUNNING' and app['elapsedTime'] > timeout and app['applicationType'].lower() == 'spark' :
              format_apps.append({
                'id' : app['id'],
                'user' : app['user'],
                'name' : app['name'],
                'user' : app['user'],
                'type' : app['applicationType'],
                'start_time' : app['startedTime'],
                'state' : app['state'],
                'queue' : app['queue'],
                'elapsed_time' : app['elapsedTime'],
                'allocated_mb' : app['allocatedMB'],
                'reserved_mb' : app['reservedMB'],
                'running_containers' : app['runningContainers'],
                })
    return format_apps


# 设定参数
yarn_url='aa.bb.cc.dd'
timeout=30*60*1000

# 获取结果
format_apps=format_filter_apps( get_apps(yarn_url),timeout)
for app in format_apps:
    print(app)