yarn资源队列信息一般都保存在一个xml文件里。具体路径可以在yarn-site.xml中查找。
具体格式如下:
<property>
<name>yarn.scheduler.fair.allocation.file</name>
<value>/etc/hadoop/conf/fair-scheduler.xml</value>
</property>
fair-scheduler.xml格式如下:
<?xml version="1.0"?>
<allocations>
<queue name="root">
<aclSubmitApps></aclSubmitApps>
<aclAdministerApps></aclAdministerApps>
<defaultFairSharePreemptionTimeout>600</defaultFairSharePreemptionTimeout>
<defaultFairSharePreemptionThreshold>0.8</defaultFairSharePreemptionThreshold>>
<queue name="production">
<minResources>8192mb,8vcores</minResources>
<maxResources>419840mb,125vcores</maxResources>
<maxRunningApps>60</maxRunningApps>
<schedulingMode>fair</schedulingMode>
<weight>7.5</weight>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>production</aclAdministerApps>
</queue>
<queue name="spark">
<minResources>8192mb,8vcores</minResources>
<maxResources>376480mb,110vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
<schedulingMode>fair</schedulingMode>
<weight>1</weight>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>spark</aclAdministerApps>
</queue>
<queue name="default">
<minResources>8192mb,8vcores</minResources>
<maxResources>202400mb,20vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
<schedulingMode>FIFO</schedulingMode>
<weight>0.5</weight>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
</queue>
<queue name="hbase">
<minResources>8192mb,8vcores</minResources>
<maxResources>69120mb,16vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
<schedulingMode>fair</schedulingMode>
<aclSubmitApps>*</aclSubmitApps>
<weight>1</weight>
<aclAdministerApps>hbase</aclAdministerApps>
</queue>
</queue>
python解析xml文件可以直接导入模块xml.sax
import xml.sax
class YarnQueue(xml.sax.ContenHandler):
def __init__(self):
self.CurrentData = ""
self.minResources = ""
self.maxResources =""
self.aclSubmitApps =""
self.aclAdminsterApps = ""
#获取每个队列的队列名
def get_queuename(self, tag, attributes):
self.CurrentData = tag
if tag == "queue":
name = attributes["name"]
#区分每个队列
print("[yarn_{}]".format(name))
print("name = root.{}".format(name))
def get_queuevlu(self, tag):
if self.CurrentData == "minResources":
if len(self.minResources.split()) == 4:
min_cores = self.minResources.split()[2]
min_memory = int(self.minResources.split()[0])/1024
print("min_cores = {}".format(min_cores))
#int()是为了不带小数点
print("min_memory = {}g".format(int(min_memory)))
elif self.CurrentData == "maxResources":
if len(self.maxResources.split()) == 4:
max_cores = self.maxResources.split()[2]
max_memory = int(self.maxResources.split()[0])/1024
print("max_cores = {}".format(max_cores))
#int()是为了不带小数点
print("max_memory = {}g".format(int(max_memory)))
elif self.CurrentData == "aclSubmitApps":
print("aclSubmitApps = {}".format(aclSubmitApps))
elif self.CurrentData == "aclAdminsterApps":
print("aclAdminsterApps = {}".format(aclAdminsterApps))
self.CurrentData = ""
#处理内容
def queue_res(self, content):
if self.CurrentData == "minResources":
self.minResources = content
elif self.CurrentData == "maxResources":
self.maxResources = content
elif self.CurrentData == "aclSubmitApps":
self.aclSubmitApps = content
elif self.CurrentData == "aclAdminsterApps":
self.aclAdminsterApps = content
if (__name__ == "__main__"):
#创建一个xml reader
parser = xml.sax.make_parser()
parser.setFeature(xml.sax.handler.feature_namespaces, 0)
#重写ContenHandler
Handler = YarnQueue()
parser.setContentHandler(Handler)
parser.parse("fair-scheduler.xml")