OOZIE依赖解析并用mxGraph画出依赖图

 

方式一：

在hive中创建表，通过UDTF解析hdfs上的coordinate.xml获取应用间的依赖关系，并以父子维的方式存储在hive表中，以后可以在java中直接访问hive表，并用mxgraph画出依赖图

create_hadoop_app_info.hql

use pad_hdp;
drop table hadoop_app_info;
create table hadoop_app_info(
appname           string comment '应用名',
frequency         string comment '调度频率',
startTime         string comment '调度时间',
dataInStr         string comment '依赖的应用们'
)comment 'HADOOP应用解析结果表' row format delimited fields terminated by '\001' stored as rcfile;

 

CoordinatorParse

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
public class CoordinatorParse {

	/**
	 * @param args
	 */
	/*
	 * public static void main(String[] args) throws Exception{ // TODO
	 * Auto-generated method stub List<OozieApp> appRelation =new
	 * ArrayList<OozieApp>(); String
	 * svnPath="D:/CCShare/PAD-HADOOP1.87.0/src/main/resources/config/hduser0401"
	 * ;
	 * 
	 * List<String> svnPathList=getCoordinatorList(svnPath); for (int
	 * i=0;i<svnPathList.size();i++){
	 * singleFileParseForGraph(svnPathList.get(i).toString(),appRelation); }
	 * for(OozieApp op:appRelation){
	 * //System.out.println(op.getDataIn()+","+op.getDataOut()); } List<String>
	 * jsList=new ArrayList<String>(); jsList.add(
	 * "var root = graph.insertVertex(parent, null, 'root', 200, 200, w, h);");
	 * //System.out.println(
	 * "var root = graph.insertVertex(parent, null, 'root', 200, 200, w, h);");
	 * for(OozieApp op:appRelation){ String
	 * str1="var "+op.getDataIn().replace("-",
	 * "_")+" = graph.insertVertex(parent, null, '"
	 * +op.getDataIn()+"', 200, 200, w, h);"; String
	 * str2="var "+op.getDataOut().replace("-",
	 * "_")+" = graph.insertVertex(parent, null, '"
	 * +op.getDataOut()+"', 200, 200, w, h);"; String
	 * str3="var "+op.getDataIn().replace("-", "_")+op.getDataOut().replace("-",
	 * "_")+" = graph.insertEdge(parent, null, '',"+op.getDataIn().replace("-",
	 * "_")+" ,"+op.getDataOut().replace("-", "_")+");";
	 * 
	 * if(!jsList.contains(str1)){ jsList.add(str1); //
	 * System.out.println(str1); } if(!jsList.contains(str2)){ jsList.add(str2);
	 * //System.out.println(str2); } jsList.add(str3);
	 * //System.out.println(str3); } }
	 */
	public static void main(String[] args) throws Exception{ 
		Map<String, OozieApp> oozieAppMap = new HashMap<String, OozieApp>();
		List<String> svnPathList =new CoordinatorParse().getCoordinatorList("hdfs://dev-l002781.app.paic.com.cn:9000/apps/hduser0401");

	
	
		for (int i = 0; i < svnPathList.size(); i++) {
			try {
				new CoordinatorParse().singleFileParseForGraph(svnPathList.get(i).toString(),
						oozieAppMap);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		//return oozieAppMap;
	  }

	public  Map<String, OozieApp> getOozieAppMap(String svnPath) {
		Map<String, OozieApp> oozieAppMap = new HashMap<String, OozieApp>();
		List<String> svnPathList = this.getCoordinatorList(svnPath);

		for (int i = 0; i < svnPathList.size(); i++) {
			try {
				this.singleFileParseForGraph(svnPathList.get(i).toString(),
						oozieAppMap);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		return oozieAppMap;
	}
	
	  
	public List<String> getCoordinatorList(String str) {
		List<String> list = new ArrayList<String>();
		System.out.println(str);
		BufferedReader br = null;
		FileSystem fs;
		try {
			fs = FileSystem.get(new Configuration());
			FileStatus[] status = fs.listStatus(new Path(str));
			for (FileStatus file : status) {

				String strTmp = file.getPath().getName();
				//System.out.println(strTmp);
				if (strTmp.indexOf("pad-dp-") >= 0
						&& strTmp.indexOf("svn") < 0
						&& strTmp.indexOf("-mis-") < 0
						&& strTmp.indexOf("pad-dp-check") < 0
						&& strTmp.indexOf("pad-dp-clean") < 0
						&& strTmp.indexOf("pad-dp-bak") < 0) {
					//System.out.println(file.getPath());
					FileSystem subfs = FileSystem.get(new Configuration());
					FileStatus[] substatus = subfs.listStatus(file.getPath());
					if (substatus!=null){
						//System.out.println(subfs.listStatus(new Path(strTmp)));
						for (FileStatus subfile : substatus) {
							//SSystem.out.println(subfile.getPath());
							if (subfile.getPath().getName().equals("coordinator.xml")) {
								//System.out.println(subfile.getPath());
								list.add(subfile.getPath().toString());
							}
						}
					}
					
					//System.out.println(file.getPath().getName());
				}
				}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return list;
	}

	public  Map<String,OozieApp> singleFileParseForGraph(String str,
			Map<String,OozieApp> appRelationMap) throws Exception {
		
		BufferedReader br = null;
		FileSystem fs;
		
			fs = FileSystem.get(new Configuration());
			FileStatus file =fs.getFileStatus(new Path(str));
			System.out.println(file.getPath());
			FSDataInputStream inputStream = fs.open(file.getPath());
			br =	  new BufferedReader(new InputStreamReader(inputStream));
			
			String line = null;
			 String strs="";
			while (null != (line = br.readLine())) {
				strs= strs+line;
				//System.out.println(line);
			}
		//InputStream is = new ByteArrayInputStream(text.getBytes());

		//System.out.println("singleFileParseForGraph=============================");
		// ???DOM??????????????
		DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
		// ??DOM?????л??DOM??????

		DocumentBuilder dbBuilder = dbFactory.newDocumentBuilder();
		// ?????File????????????
		Document doc = null;
		
		// 字符串转XML
		
		StringReader sr = new StringReader(strs);
		InputSource is = new InputSource(sr);
		//DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
		//DocumentBuilder builder=factory.newDocumentBuilder();
		//Document doc = builder.parse(is);



		doc = dbBuilder.parse(is);
		// ??????????Student?????????б?
		NodeList listIn = doc.getElementsByTagName("data-in");
		NodeList listOut = doc.getElementsByTagName("data-out");
		String frequency = ((Element) doc.getElementsByTagName("coordinator-app").item(0)).getAttribute("frequency");
		
		
		System.out.println("frequency:"+frequency);
		String datasetOut = "";
		String datasetIn = "";
//		OozieApp oozieApp = new OozieApp();
		System.out.println("singleFileParseForGraph");
		
		OozieApp oozieApp = new OozieApp();
		List<String> dataInList =new ArrayList<String>();
		
		for (int i = 0; i < listOut.getLength(); i++) {

			String str_tmp_1 = ((Element) listOut.item(i)).getAttribute("dataset");
			str_tmp_1=str_tmp_1.replace("-dataset", "");
			
			if (str_tmp_1.contains("pad-dp-")) {
				datasetOut=str_tmp_1;
				for (int j = 0; j < listIn.getLength(); j++) {
					
					datasetIn = ((Element) listIn.item(j))
							.getAttribute("dataset");
					datasetIn=datasetIn.replace("-dataset", "");
                    dataInList.add(datasetIn);
                    
					//appRelation.add(oozieApp);
				}
			}
		}
		System.out.println("str:"+str);
		String startTime = getAppStartTime(str);
		System.out.println("getAppStartTime_startTime:"+startTime);
	
		oozieApp.setDataInList(dataInList);
		//System.out.println("dataInList:"+dataInList.size());
		//System.out.println("datasetOut:"+datasetOut);
		//System.out.println("frequency:"+frequency);
		//System.out.println("startTime:"+startTime);
		
		oozieApp.setDataOut(datasetOut);
		oozieApp.setStartTime(startTime);
		
		oozieApp.setFrequency(frequency);
		
		//System.out.println("frequency:"+oozieApp.getFrequency());
		//System.out.println("startTime:"+oozieApp.getStartTime());
		appRelationMap.put(datasetOut, oozieApp);
		
		System.out.println("appRelationMap:"+appRelationMap.size());
		
		
		//System.out.println("startTime:"+startTime);


		return appRelationMap;

	}
	
	
	
	
	
	public  String getAppStartTime(String str) throws Exception {
		Properties props = new Properties();
		//System.out.println("getAppStartTime");
		String[] strArray = str.split("/");
		//System.out.println("strArray:"+strArray[strArray.length - 2]);
		//System.out.println(str);
		//str=str.replace("ordinator", "fdsafsd");
		System.out.println(str);
		str=str.replaceAll("coordinator.xml", strArray[strArray.length - 2]
				+ ".properties");
		System.out.println(str);
		
		BufferedReader br = null;
		FileSystem fs;
		
			fs = FileSystem.get(new Configuration());
			FileStatus file =fs.getFileStatus(new Path(str));
			System.out.println(file.getPath());
			FSDataInputStream inputStream = fs.open(file.getPath());
			br =	  new BufferedReader(new InputStreamReader(inputStream));
			
			String line = null;
			 String strs="";
			while (null != (line = br.readLine())) {
				strs= strs+"\r\n"+line;
				//System.out.println(line);
			}
			// 
			
		
		//System.out.println(str);
		try {
			InputStream is = new ByteArrayInputStream(strs.getBytes());

            
			//InputStream in = new BufferedInputStream(new FileInputStream(str));
			InputStream in=is;
			System.out.println("================================================");
			props.load(is);
			String value = props.getProperty("job_start");
			System.out.println("job_start:" + value);
			int byteRead;
           /* while ((byteRead = is.read()) != -1) {
                System.out.print((char)byteRead);