用python获取yarn资源fair-scheduler.xml里的队列信息

yarn资源队列信息一般都保存在一个xml文件里。具体路径可以在yarn-site.xml中查找。
具体格式如下：

<property>
      <name>yarn.scheduler.fair.allocation.file</name>
      <value>/etc/hadoop/conf/fair-scheduler.xml</value>
    </property>

fair-scheduler.xml格式如下：

<?xml version="1.0"?>
<allocations>
    <queue name="root">
        <aclSubmitApps></aclSubmitApps>
        <aclAdministerApps></aclAdministerApps>
        <defaultFairSharePreemptionTimeout>600</defaultFairSharePreemptionTimeout>
        <defaultFairSharePreemptionThreshold>0.8</defaultFairSharePreemptionThreshold>>
        
        <queue name="production">
            <minResources>8192mb,8vcores</minResources>
            <maxResources>419840mb,125vcores</maxResources>
            <maxRunningApps>60</maxRunningApps>
            <schedulingMode>fair</schedulingMode>
            <weight>7.5</weight>
            <aclSubmitApps>*</aclSubmitApps>
            <aclAdministerApps>production</aclAdministerApps>
        </queue>
        <queue name="spark">
            <minResources>8192mb,8vcores</minResources>
            <maxResources>376480mb,110vcores</maxResources>
            <maxRunningApps>50</maxRunningApps>
            <schedulingMode>fair</schedulingMode>
            <weight>1</weight>
            <aclSubmitApps>*</aclSubmitApps>
            <aclAdministerApps>spark</aclAdministerApps>
        </queue>
        <queue name="default">
            <minResources>8192mb,8vcores</minResources>
            <maxResources>202400mb,20vcores</maxResources>
            <maxRunningApps>20</maxRunningApps>
            <schedulingMode>FIFO</schedulingMode>
            <weight>0.5</weight>
            <aclSubmitApps>*</aclSubmitApps>
            <aclAdministerApps>*</aclAdministerApps>
        </queue>
        <queue name="hbase">
            <minResources>8192mb,8vcores</minResources>
            <maxResources>69120mb,16vcores</maxResources>
            <maxRunningApps>20</maxRunningApps>
            <schedulingMode>fair</schedulingMode>
            <aclSubmitApps>*</aclSubmitApps>
            <weight>1</weight>
            <aclAdministerApps>hbase</aclAdministerApps>
        </queue>
    </queue>

python解析xml文件可以直接导入模块xml.sax

import xml.sax

class YarnQueue(xml.sax.ContenHandler):
    def __init__(self):
        self.CurrentData = ""
        self.minResources = ""
        self.maxResources =""
        self.aclSubmitApps =""
        self.aclAdminsterApps = ""
    #获取每个队列的队列名    
    def get_queuename(self, tag, attributes):
        self.CurrentData = tag
        if tag == "queue":
            name = attributes["name"]
            #区分每个队列
            print("[yarn_{}]".format(name))
            print("name = root.{}".format(name))
    def get_queuevlu(self, tag):
        if self.CurrentData == "minResources":
            if len(self.minResources.split()) == 4:
                min_cores = self.minResources.split()[2]
                min_memory = int(self.minResources.split()[0])/1024
                print("min_cores = {}".format(min_cores))
                #int()是为了不带小数点
                print("min_memory = {}g".format(int(min_memory)))
        elif self.CurrentData == "maxResources":
            if len(self.maxResources.split()) == 4:
                max_cores = self.maxResources.split()[2]
                max_memory = int(self.maxResources.split()[0])/1024
                print("max_cores = {}".format(max_cores))
                #int()是为了不带小数点
                print("max_memory = {}g".format(int(max_memory)))
        elif self.CurrentData == "aclSubmitApps":
            print("aclSubmitApps = {}".format(aclSubmitApps))
        elif self.CurrentData == "aclAdminsterApps":
            print("aclAdminsterApps = {}".format(aclAdminsterApps))
        self.CurrentData = ""
    #处理内容
    def queue_res(self, content):
        if self.CurrentData == "minResources":
            self.minResources = content
        elif self.CurrentData == "maxResources":
            self.maxResources = content
        elif self.CurrentData == "aclSubmitApps":
            self.aclSubmitApps = content
        elif self.CurrentData == "aclAdminsterApps":
            self.aclAdminsterApps = content
if (__name__ == "__main__"):
    #创建一个xml reader
    parser = xml.sax.make_parser()
    parser.setFeature(xml.sax.handler.feature_namespaces, 0)
    #重写ContenHandler
    Handler = YarnQueue()
    parser.setContentHandler(Handler)
    parser.parse("fair-scheduler.xml")