OOZIE依賴解析並用mxGraph畫出依賴圖
阿新 • • 發佈：2019-01-25
方式一：
在hive中建立表，通過UDTF解析hdfs上的coordinate.xml獲取應用間的依賴關係，並以父子維的方式儲存在hive表中，以後可以在java中直接訪問hive表，並用mxgraph畫出依賴圖
create_hadoop_app_info.hql
use pad_hdp; drop table hadoop_app_info; create table hadoop_app_info( appname string comment '應用名', frequency string comment '排程頻率', startTime string comment '排程時間', dataInStr string comment '依賴的應用們' )comment 'HADOOP應用解析結果表' row format delimited fields terminated by '\001' stored as rcfile;
CoordinatorParse
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; public class CoordinatorParse { /** * @param args */ /* * public static void main(String[] args) throws Exception{ // TODO * Auto-generated method stub List<OozieApp> appRelation =new * ArrayList<OozieApp>(); String * svnPath="D:/CCShare/PAD-HADOOP1.87.0/src/main/resources/config/hduser0401" * ; * * List<String> svnPathList=getCoordinatorList(svnPath); for (int * i=0;i<svnPathList.size();i++){ * singleFileParseForGraph(svnPathList.get(i).toString(),appRelation); } * for(OozieApp op:appRelation){ * //System.out.println(op.getDataIn()+","+op.getDataOut()); } List<String> * jsList=new ArrayList<String>(); jsList.add( * "var root = graph.insertVertex(parent, null, 'root', 200, 200, w, h);"); * //System.out.println( * "var root = graph.insertVertex(parent, null, 'root', 200, 200, w, h);"); * for(OozieApp op:appRelation){ String * str1="var "+op.getDataIn().replace("-", * "_")+" = graph.insertVertex(parent, null, '" * +op.getDataIn()+"', 200, 200, w, h);"; String * str2="var "+op.getDataOut().replace("-", * "_")+" = graph.insertVertex(parent, null, '" * +op.getDataOut()+"', 200, 200, w, h);"; String * str3="var "+op.getDataIn().replace("-", "_")+op.getDataOut().replace("-", * "_")+" = graph.insertEdge(parent, null, '',"+op.getDataIn().replace("-", * "_")+" ,"+op.getDataOut().replace("-", "_")+");"; * * if(!jsList.contains(str1)){ jsList.add(str1); // * System.out.println(str1); } if(!jsList.contains(str2)){ jsList.add(str2); * //System.out.println(str2); } jsList.add(str3); * //System.out.println(str3); } } */ public static void main(String[] args) throws Exception{ Map<String, OozieApp> oozieAppMap = new HashMap<String, OozieApp>(); List<String> svnPathList =new CoordinatorParse().getCoordinatorList("hdfs://dev-l002781.app.paic.com.cn:9000/apps/hduser0401"); for (int i = 0; i < svnPathList.size(); i++) { try { new CoordinatorParse().singleFileParseForGraph(svnPathList.get(i).toString(), oozieAppMap); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } //return oozieAppMap; } public Map<String, OozieApp> getOozieAppMap(String svnPath) { Map<String, OozieApp> oozieAppMap = new HashMap<String, OozieApp>(); List<String> svnPathList = this.getCoordinatorList(svnPath); for (int i = 0; i < svnPathList.size(); i++) { try { this.singleFileParseForGraph(svnPathList.get(i).toString(), oozieAppMap); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } return oozieAppMap; } public List<String> getCoordinatorList(String str) { List<String> list = new ArrayList<String>(); System.out.println(str); BufferedReader br = null; FileSystem fs; try { fs = FileSystem.get(new Configuration()); FileStatus[] status = fs.listStatus(new Path(str)); for (FileStatus file : status) { String strTmp = file.getPath().getName(); //System.out.println(strTmp); if (strTmp.indexOf("pad-dp-") >= 0 && strTmp.indexOf("svn") < 0 && strTmp.indexOf("-mis-") < 0 && strTmp.indexOf("pad-dp-check") < 0 && strTmp.indexOf("pad-dp-clean") < 0 && strTmp.indexOf("pad-dp-bak") < 0) { //System.out.println(file.getPath()); FileSystem subfs = FileSystem.get(new Configuration()); FileStatus[] substatus = subfs.listStatus(file.getPath()); if (substatus!=null){ //System.out.println(subfs.listStatus(new Path(strTmp))); for (FileStatus subfile : substatus) { //SSystem.out.println(subfile.getPath()); if (subfile.getPath().getName().equals("coordinator.xml")) { //System.out.println(subfile.getPath()); list.add(subfile.getPath().toString()); } } } //System.out.println(file.getPath().getName()); } } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return list; } public Map<String,OozieApp> singleFileParseForGraph(String str, Map<String,OozieApp> appRelationMap) throws Exception { BufferedReader br = null; FileSystem fs; fs = FileSystem.get(new Configuration()); FileStatus file =fs.getFileStatus(new Path(str)); System.out.println(file.getPath()); FSDataInputStream inputStream = fs.open(file.getPath()); br = new BufferedReader(new InputStreamReader(inputStream)); String line = null; String strs=""; while (null != (line = br.readLine())) { strs= strs+line; //System.out.println(line); } //InputStream is = new ByteArrayInputStream(text.getBytes()); //System.out.println("singleFileParseForGraph============================="); // ???DOM?????????????? DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); // ??DOM?????л??DOM?????? DocumentBuilder dbBuilder = dbFactory.newDocumentBuilder(); // ?????File???????????? Document doc = null; // 字串轉XML StringReader sr = new StringReader(strs); InputSource is = new InputSource(sr); //DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); //DocumentBuilder builder=factory.newDocumentBuilder(); //Document doc = builder.parse(is); doc = dbBuilder.parse(is); // ??????????Student?????????б? NodeList listIn = doc.getElementsByTagName("data-in"); NodeList listOut = doc.getElementsByTagName("data-out"); String frequency = ((Element) doc.getElementsByTagName("coordinator-app").item(0)).getAttribute("frequency"); System.out.println("frequency:"+frequency); String datasetOut = ""; String datasetIn = ""; // OozieApp oozieApp = new OozieApp(); System.out.println("singleFileParseForGraph"); OozieApp oozieApp = new OozieApp(); List<String> dataInList =new ArrayList<String>(); for (int i = 0; i < listOut.getLength(); i++) { String str_tmp_1 = ((Element) listOut.item(i)).getAttribute("dataset"); str_tmp_1=str_tmp_1.replace("-dataset", ""); if (str_tmp_1.contains("pad-dp-")) { datasetOut=str_tmp_1; for (int j = 0; j < listIn.getLength(); j++) { datasetIn = ((Element) listIn.item(j)) .getAttribute("dataset"); datasetIn=datasetIn.replace("-dataset", ""); dataInList.add(datasetIn); //appRelation.add(oozieApp); } } } System.out.println("str:"+str); String startTime = getAppStartTime(str); System.out.println("getAppStartTime_startTime:"+startTime); oozieApp.setDataInList(dataInList); //System.out.println("dataInList:"+dataInList.size()); //System.out.println("datasetOut:"+datasetOut); //System.out.println("frequency:"+frequency); //System.out.println("startTime:"+startTime); oozieApp.setDataOut(datasetOut); oozieApp.setStartTime(startTime); oozieApp.setFrequency(frequency); //System.out.println("frequency:"+oozieApp.getFrequency()); //System.out.println("startTime:"+oozieApp.getStartTime()); appRelationMap.put(datasetOut, oozieApp); System.out.println("appRelationMap:"+appRelationMap.size()); //System.out.println("startTime:"+startTime); return appRelationMap; } public String getAppStartTime(String str) throws Exception { Properties props = new Properties(); //System.out.println("getAppStartTime"); String[] strArray = str.split("/"); //System.out.println("strArray:"+strArray[strArray.length - 2]); //System.out.println(str); //str=str.replace("ordinator", "fdsafsd"); System.out.println(str); str=str.replaceAll("coordinator.xml", strArray[strArray.length - 2] + ".properties"); System.out.println(str); BufferedReader br = null; FileSystem fs; fs = FileSystem.get(new Configuration()); FileStatus file =fs.getFileStatus(new Path(str)); System.out.println(file.getPath()); FSDataInputStream inputStream = fs.open(file.getPath()); br = new BufferedReader(new InputStreamReader(inputStream)); String line = null; String strs=""; while (null != (line = br.readLine())) { strs= strs+"\r\n"+line; //System.out.println(line); } // //System.out.println(str); try { InputStream is = new ByteArrayInputStream(strs.getBytes()); //InputStream in = new BufferedInputStream(new FileInputStream(str)); InputStream in=is; System.out.println("================================================"); props.load(is); String value = props.getProperty("job_start"); System.out.println("job_start:" + value); int byteRead; /* while ((byteRead = is.read()) != -1) { System.out.print((char)byteRead); } System.out.println("");*/ //is.read() is.close(); return value; } catch (Exception e) { e.printStackTrace(); return ""; } } public List<OozieApp> getOozieAppList(Map<String, OozieApp> oozieAppMap) { List<OozieApp> oozieAppList = new ArrayList<OozieApp>(); for (Object o : oozieAppMap.keySet()) { OozieApp oozieApp = oozieAppMap.get(o); String dataOut = oozieApp.getDataOut(); String startTime = oozieApp.getStartTime(); String frequency = oozieApp.getFrequency(); List<String> dataInList = oozieApp.getDataInList(); if (dataInList.size() > 0) { for (int i = 0; i < dataInList.size(); i++) { String dataIn = dataInList.get(i); OozieApp oozieAppRelation = new OozieApp(); oozieAppRelation.setDataIn(dataIn); oozieAppRelation.setDataOut(dataOut); oozieAppRelation.setStartTime(startTime); oozieAppRelation.setFrequency(frequency); oozieAppList.add(oozieAppRelation); } } else { String dataIn = ""; OozieApp oozieAppRelation = new OozieApp(); oozieAppRelation.setDataIn(dataIn); oozieAppRelation.setDataOut(dataOut); oozieAppRelation.setStartTime(startTime); oozieAppRelation.setFrequency(frequency); oozieAppList.add(oozieAppRelation); } } return oozieAppList; } public List<OozieApp> filterOozieAppList(List<OozieApp> oozieAppList, String appname) throws Exception { List<OozieApp> resultOozieAppList = new ArrayList<OozieApp>(); List<String> alreadyFindList = new ArrayList<String>(); // System.out.println("appname:"+appname+" "+oozieAppList.size()); filterSourceOozieAppList(oozieAppList, appname, resultOozieAppList, alreadyFindList); alreadyFindList = new ArrayList<String>(); filterTargetOozieAppList(oozieAppList, appname, resultOozieAppList, alreadyFindList); return resultOozieAppList; } public void filterSourceOozieAppList(List<OozieApp> oozieAppList, String appname, List<OozieApp> resultOozieAppList, List<String> alreadyFindList) throws Exception { String dataIn; String dataOut; if (appname != null && appname != "") { for (int i = 0; i < oozieAppList.size(); i++) { dataIn = oozieAppList.get(i).getDataIn(); dataOut = oozieAppList.get(i).getDataOut(); if (dataOut.equals(appname)) { resultOozieAppList.add(oozieAppList.get(i)); alreadyFindList.add(appname); // System.out.println("appname:"+appname+" SalreadyFindList:"+alreadyFindList.size()); // System.out.println("appname:"+appname+" resultOozieAppList:"+resultOozieAppList.size()); // System.out.println("appname:"+appname+" oozieAppList:"+oozieAppList.size()); // System.out.println("appname:"+appname+" dataOut:"+dataOut); if (!dataIn.equals(dataOut) && !alreadyFindList.contains(dataIn)) { filterSourceOozieAppList(oozieAppList, dataIn, resultOozieAppList, alreadyFindList); } } } } } protected void filterTargetOozieAppList(List<OozieApp> oozieAppList, String appname, List<OozieApp> resultOozieAppList, List<String> alreadyFindList) throws Exception { String dataIn; String dataOut; if (appname != null && appname != "") { for (int i = 0; i < oozieAppList.size(); i++) { dataIn = oozieAppList.get(i).getDataIn(); dataOut = oozieAppList.get(i).getDataOut(); if (dataIn.equals(appname)) { // System.out.println("appname:"+appname+" dataOut:"+dataOut); resultOozieAppList.add(oozieAppList.get(i)); alreadyFindList.add(appname); // System.out.println("appname:"+appname+" TalreadyFindList:"+alreadyFindList.size()); if (!dataIn.equals(dataOut) && !alreadyFindList.contains(dataOut)) { filterTargetOozieAppList(oozieAppList, dataOut, resultOozieAppList, alreadyFindList); } } } } } }
OozieApp
public class OozieApp { List<String> dataInList; String dataIn; String dataOut; String startTime; String frequency; public String getFrequency() { String frequency=this.frequency; if(this.frequency.indexOf(":")<this.frequency.indexOf("}")){ frequency=this.frequency.substring(this.frequency.indexOf(":")+1, this.frequency.indexOf("}")); } return frequency; } public void setFrequency(String frequency) { this.frequency = frequency; } public String getDataIn() { return dataIn; } public void setDataIn(String dataIn) { this.dataIn = dataIn; } public String getStartTime() { //System.out.println("OozieApp_startTime:"+startTime); String startTime=this.startTime; //System.out.println("this.frequency:"+this.dataOut+this.frequency); //System.out.println("this.startTime:"+this.dataOut+this.startTime); //System.out.println("this.frequency.indexOf(days):"+this.dataOut+this.frequency.indexOf("days")); if(this.frequency.indexOf("days")>=0){ if(this.startTime.indexOf("T")<this.startTime.indexOf("+")){ startTime=this.startTime.substring(this.startTime.indexOf("T")+1, this.startTime.indexOf("+")); } //System.out.println("days"); }else if(this.frequency.indexOf("months")>=0){ //System.out.println("month"); if(this.startTime.indexOf("-")<this.startTime.indexOf("+")){ //System.out.println("month"); startTime=this.startTime.substring(this.startTime.indexOf("-",7)+1, this.startTime.indexOf("+")); } } //System.out.println("end"); return startTime; } public void setStartTime(String startTime) { //System.out.println("OozieApp_set_startTime:"+startTime); this.startTime = startTime; } public void setDataInList(List<String> dataInList) { this.dataInList = dataInList; } public void setDataOut(String dataOut) { this.dataOut = dataOut; } public List<String> getDataInList() { List<String> dataInList=new ArrayList<String>(); for(int i=0;i<this.dataInList.size();i++){ dataInList.add(this.dataInList.get(i).replace("-dataset", "")); } //System.out.println("OozieApp_dataInList:"+dataInList.size()); //System.out.println("this.OozieApp_dataInList:"+this.dataInList.size()); return dataInList; } public String getDataOut() { return dataOut.replace("-dataset", ""); } }
UDTFHadoopAppParse
import org.apache.hadoop.hiveimport org.apache.hadoop.hivepublic class UDTFHadoopAppParse extends GenericUDTF{
@Override
public void close() throws HiveException {
// TODO Auto-generated method stub
}
@Override
public StructObjectInspector initialize(ObjectInspector[] args)
throws UDFArgumentException {
if (args.length != 1) {
throw new UDFArgumentLengthException("UDTFHadoopAppParse takes only one argument");
}
if (args[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
throw new UDFArgumentException("UDTFHadoopAppParse takes string as a parameter");
}
ArrayList<String> fieldNames = new ArrayList<String>();
ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
fieldNames.add("col1");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldNames.add("col2");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldNames.add("col3");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldNames.add("col4");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames,fieldOIs);
}
@Override
public void process(Object[] args) throws HiveException {
String input = args[0].toString();
Map<String, OozieApp> oozieAppMap = new HashMap<String, OozieApp>();
String svnPath = input;
CoordinatorParse cp = new CoordinatorParse();
// List<String> svnPathList = cp.getCoordinatorList(svnPath);
oozieAppMap = cp.getOozieAppMap(svnPath);
List<OozieApp> oozieAppList = new ArrayList<OozieApp>();
for (Object o : oozieAppMap.keySet()) {
OozieApp oozieApp = oozieAppMap.get(o);
String dataOut = oozieApp.getDataOut();
String startTime = oozieApp.getStartTime();
String frequency = oozieApp.getFrequency();
List<String> dataInList = oozieApp.getDataInList();
String dataInStr="";
for (int i=0;i<dataInList.size();i++){
if(dataInStr!=""){
dataInStr=dataInStr+","+dataInList.get(i);
}else{
dataInStr=dataInStr+dataInList.get(i);
}
}
try {
String[] result = new String[4];
result[0]=dataOut;
result[1]=frequency;
result[2]=startTime;
result[3]=dataInStr;
forward(result);
} catch (Exception e) {
continue;
}
}
}
}方式二：
直接解析本地SVN程式碼，使用者查詢後直接解析：
CoordinatorParse
public class CoordinatorParse {
/**
* @param args
*/
public Map<String, OozieApp> getOozieAppMap(String svnPath) {
Map<String, OozieApp> oozieAppMap = new HashMap<String, OozieApp>();
List<String> svnPathList = this.getCoordinatorList(svnPath);
for (int i = 0; i < svnPathList.size(); i++) {
try {
this.singleFileParseForGraph(svnPathList.get(i).toString(),
oozieAppMap);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return oozieAppMap;
}
// ��ȡ��Ч��coordinate·��
public List<String> getCoordinatorList(String str) {
List<String> list = new ArrayList<String>();
File file = new File(str);
File[] lf = file.listFiles();
// System.out.println("data-in,data-out");
for (int i = 0; i < lf.length; i++) {
// System.out.println(lf[i].toString());
if (lf[i].toString().indexOf("pad-dp-") > 0
&& lf[i].toString().indexOf("svn") <= 0
&& lf[i].toString().indexOf("-mis-") <= 0
&& lf[i].toString().indexOf("pad-dp-check") <= 0
&& lf[i].toString().indexOf("pad-dp-clean") <= 0
&& lf[i].toString().indexOf("pad-dp-bak") <= 0) {
File subDir = new File(lf[i].toString());
File[] sublf = subDir.listFiles();
// System.out.println(lf[i].toString());
for (int j = 0; j < sublf.length; j++) {
// System.out.println(sublf[j]);
if (sublf[j].toString().contains("coordinator.xml")) {
// System.out.println(sublf[j]);
list.add(sublf[j].toString());
}
}
}
}
return list;
}
public Map<String,OozieApp> singleFileParseForGraph(String str,
Map<String,OozieApp> appRelationMap) throws Exception {
//System.out.println("singleFileParseForGraph=============================");
// �õ�DOM�������Ĺ���ʵ��
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
// ��DOM������DOM������
DocumentBuilder dbBuilder = dbFactory.newDocumentBuilder();
// ����ΪFileΪ��ʶ��������
Document doc = null;
doc = dbBuilder.parse(str);
// �õ��ĵ����ΪStudent��Ԫ�صĽڵ��б�
NodeList listIn = doc.getElementsByTagName("data-in");
NodeList listOut = doc.getElementsByTagName("data-out");
String frequency = ((Element) doc.getElementsByTagName("coordinator-app").item(0)).getAttribute("frequency");
//System.out.println("frequency:"+frequency);
String datasetOut = "";
String datasetIn = "";
// OozieApp oozieApp = new OozieApp();
//System.out.println("singleFileParseForGraph");
OozieApp oozieApp = new OozieApp();
List<String> dataInList =new ArrayList<String>();
for (int i = 0; i < listOut.getLength(); i++) {
String str_tmp_1 = ((Element) listOut.item(i)).getAttribute("dataset");
str_tmp_1=str_tmp_1.replace("-dataset", "");
if (str_tmp_1.contains("pad-dp-")) {
datasetOut=str_tmp_1;
for (int j = 0; j < listIn.getLength(); j++) {
datasetIn = ((Element) listIn.item(j))
.getAttribute("dataset");
datasetIn=datasetIn.replace("-dataset", "");
dataInList.add(datasetIn);
//appRelation.add(oozieApp);
}
}
}
String startTime = getAppStartTime(str);
//System.out.println("getAppStartTime_startTime:"+startTime);
oozieApp.setDataInList(dataInList);
//System.out.println("dataInList:"+dataInList.size());
//System.out.println("datasetOut:"+datasetOut);
//System.out.println("frequency:"+frequency);
//System.out.println("startTime:"+startTime);
oozieApp.setDataOut(datasetOut);
oozieApp.setStartTime(startTime);
oozieApp.setFrequency(frequency);
//System.out.println("frequency:"+oozieApp.getFrequency());
//System.out.println("startTime:"+oozieApp.getStartTime());
appRelationMap.put(datasetOut, oozieApp);
//System.out.println("appRelationMap:"+appRelationMap.size());
//System.out.println("startTime:"+startTime);
return appRelationMap;
}
public String getAppStartTime(String str) throws Exception {
Properties props = new Properties();
//System.out.println("getAppStartTime");
String[] strArray = str.split("\\\\");
//System.out.println("strArray:"+strArray[strArray.length - 2]);
//System.out.println(str);
//str=str.replace("ordinator", "fdsafsd");
//System.out.println(str);
str=str.replaceAll("coordinator.xml", strArray[strArray.length - 2]
+ ".properties");
//System.out.println(str);
try {
InputStream in = new BufferedInputStream (new FileInputStream(str));
props.load(in);
String value = props.getProperty("job_start");
//System.out.println("job_start:" + value);
return value;
} catch (Exception e) {
e.printStackTrace();
return "";
}
}
public List<OozieApp> getOozieAppList(Map<String, OozieApp> oozieAppMap) {
List<OozieApp> oozieAppList = new ArrayList<OozieApp>();
for (Object o : oozieAppMap.keySet()) {
OozieApp oozieApp = oozieAppMap.get(o);
String dataOut = oozieApp.getDataOut();
String startTime = oozieApp.getStartTime();
String frequency = oozieApp.getFrequency();
List<String> dataInList = oozieApp.getDataInList();
if (dataInList.size() > 0) {
for (int i = 0; i < dataInList.size(); i++) {
String dataIn = dataInList.get(i);
OozieApp oozieAppRelation = new OozieApp();
oozieAppRelation.setDataIn(dataIn);
oozieAppRelation.setDataOut(dataOut);
oozieAppRelation.setStartTime(startTime);
oozieAppRelation.setFrequency(frequency);
oozieAppList.add(oozieAppRelation);
}
} else {
String dataIn = "";
OozieApp oozieAppRelation = new OozieApp();
oozieAppRelation.setDataIn(dataIn);
oozieAppRelation.setDataOut(dataOut);
oozieAppRelation.setStartTime(startTime);
oozieAppRelation.setFrequency(frequency);
oozieAppList.add(oozieAppRelation);
}
}
return oozieAppList;
}
public List<OozieApp> filterOozieAppList(List<OozieApp> oozieAppList,
String appname) throws Exception {
List<OozieApp> resultOozieAppList = new ArrayList<OozieApp>();
List<String> alreadyFindList = new ArrayList<String>();
// System.out.println("appname:"+appname+" "+oozieAppList.size());
filterSourceOozieAppList(oozieAppList, appname, resultOozieAppList,
alreadyFindList);
alreadyFindList = new ArrayList<String>();
filterTargetOozieAppList(oozieAppList, appname, resultOozieAppList,
alreadyFindList);
return resultOozieAppList;
}
public void filterSourceOozieAppList(List<OozieApp> oozieAppList,
String appname, List<OozieApp> resultOozieAppList,
List<String> alreadyFindList) throws Exception {
String dataIn;
String dataOut;
if (appname != null && appname != "") {
for (int i = 0; i < oozieAppList.size(); i++) {
dataIn = oozieAppList.get(i).getDataIn();
dataOut = oozieAppList.get(i).getDataOut();
if (dataOut.equals(appname)) {
resultOozieAppList.add(oozieAppList.get(i));
alreadyFindList.add(appname);
// System.out.println("appname:"+appname+" SalreadyFindList:"+alreadyFindList.size());
// System.out.println("appname:"+appname+" resultOozieAppList:"+resultOozieAppList.size());
// System.out.println("appname:"+appname+" oozieAppList:"+oozieAppList.size());
// System.out.println("appname:"+appname+" dataOut:"+dataOut);
if (!dataIn.equals(dataOut)
&& !alreadyFindList.contains(dataIn)) {
filterSourceOozieAppList(oozieAppList, dataIn,
resultOozieAppList, alreadyFindList);
}
}
}
}
}
protected void filterTargetOozieAppList(List<OozieApp> oozieAppList,
String appname, List<OozieApp> resultOozieAppList,
List<String> alreadyFindList) throws Exception {
String dataIn;
String dataOut;
if (appname != null && appname != "") {
for (int i = 0; i < oozieAppList.size(); i++) {
dataIn = oozieAppList.get(i).getDataIn();
dataOut = oozieAppList.get(i).getDataOut();
if (dataIn.equals(appname)) {
// System.out.println("appname:"+appname+" dataOut:"+dataOut);
resultOozieAppList.add(oozieAppList.get(i));
alreadyFindList.add(appname);
// System.out.println("appname:"+appname+" TalreadyFindList:"+alreadyFindList.size());
if (!dataIn.equals(dataOut)
&& !alreadyFindList.contains(dataOut)) {
filterTargetOozieAppList(oozieAppList, dataOut,
resultOozieAppList, alreadyFindList);
}
}
}
}
}
}
ActionRel
public class ActionRel {
String action;
String nextAction;
public String getAction() {
return action;
}
public void setAction(String action) {
this.action = action;
}
public String getNextAction() {
return nextAction;
}
public void setNextAction(String nextAction) {
this.nextAction = nextAction;
}
}
OozieApp
public class OozieApp {
List<String> dataInList;
String dataIn;
String dataOut;
String startTime;
String frequency;
public String getFrequency() {
String frequency=this.frequency;
if(this.frequency.indexOf(":")<this.frequency.indexOf("}")){
frequency=this.frequency.substring(this.frequency.indexOf(":")+1, this.frequency.indexOf("}"));
}
return frequency;
}
public void setFrequency(String frequency) {
this.frequency = frequency;
}
public String getDataIn() {
return dataIn;
}
public void setDataIn(String dataIn) {
this.dataIn = dataIn;
}
public String getStartTime() {
//System.out.println("OozieApp_startTime:"+startTime);
String startTime=this.startTime;
//System.out.println("this.frequency:"+this.dataOut+this.frequency);
//System.out.println("this.startTime:"+this.dataOut+this.startTime);
//System.out.println("this.frequency.indexOf(days):"+this.dataOut+this.frequency.indexOf("days"));
if(this.frequency.indexOf("days")>=0){
if(this.startTime.indexOf("T")<this.startTime.indexOf("+")){
startTime=this.startTime.substring(this.startTime.indexOf("T")+1, this.startTime.indexOf("+"));
}
//System.out.println("days");
}else if(this.frequency.indexOf("months")>=0){
//System.out.println("month");
if(this.startTime.indexOf("-")<this.startTime.indexOf("+")){
//System.out.println("month");
startTime=this.startTime.substring(this.startTime.indexOf("-",7)+1, this.startTime.indexOf("+"));
}
}
//System.out.println("end");
return startTime;
}
public void setStartTime(String startTime) {
//System.out.println("OozieApp_set_startTime:"+startTime);
this.startTime = startTime;
}
public void setDataInList(List<String> dataInList) {
this.dataInList = dataInList;
}
public void setDataOut(String dataOut) {
this.dataOut = dataOut;
}
public List<String> getDataInList() {
List<String> dataInList=new ArrayList<String>();
for(int i=0;i<this.dataInList.size();i++){
dataInList.add(this.dataInList.get(i).replace("-dataset", ""));
}
//System.out.println("OozieApp_dataInList:"+dataInList.size());
//System.out.println("this.OozieApp_dataInList:"+this.dataInList.size());
return dataInList;
}
public String getDataOut() {
return dataOut.replace("-dataset", "");
}
}
TestServlet
/**
* Servlet implementation class TestServlet
*/
@WebServlet("/mxGraph/examples/TestServlet")
public class TestServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
/**
* @see HttpServlet#HttpServlet()
*/
public TestServlet() {
super();
// TODO Auto-generated constructor stub
}
/**
* @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse
* response)
*/
protected void doGet(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {
// TODO Auto-generated method stub
response.setCharacterEncoding("UTF-8");
request.setCharacterEncoding("UTF-8");
response.setContentType("text/html");
PrintWriter out = response.getWriter();
this.doPost(request, response);
out.flush();
out.close();
}
/**
* @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse
* response)
*/
protected void doPost(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {
// TODO Auto-generated method stub
response.setCharacterEncoding("UTF-8");
request.setCharacterEncoding("UTF-8");
response.setContentType("text/html");
PrintWriter out = response.getWriter();
List<List<String>> listArray;
// System.out.println(request.getParameter("appname"));
String appname;
if (request.getParameter("appname") != null) {
appname = request.getParameter("appname");
} else {
appname = "";
}
try {
listArray = this.getJavaScriptList(appname);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
listArray = new ArrayList<List<String>>();
}
List<String> javaScriptVertexList = listArray.get(0);
List<String> javaScriptRelationList = listArray.get(1);
List<String> javaScriptTableList = listArray.get(2);
request.setAttribute("javaScriptVertexList", javaScriptVertexList);
request.setAttribute("javaScriptRelationList", javaScriptRelationList);
request.setAttribute("javaScriptTableList", javaScriptTableList);
request.getRequestDispatcher("graphlayout-jsp.jsp").forward(request,
response);
out.flush();
out.close();
}
protected List<List<String>> getJavaScriptList(String appname)
throws Exception {
List<OozieApp> oozieAppList;
Map<String, OozieApp> oozieAppMap = new HashMap<String, OozieApp>();
String svnPath = "D:/CCShare/PAD-HADOOP1.89.0/src/main/resources/config/hduser0401";
CoordinatorParse cp = new CoordinatorParse();
// List<String> svnPathList = cp.getCoordinatorList(svnPath);
oozieAppMap = cp.getOozieAppMap(svnPath);
oozieAppList = cp.getOozieAppList(oozieAppMap);
List<OozieApp> resultOozieAppList;
if (appname != null && appname.toUpperCase().equals("ALL")) {
System.out.println("all_appname:" + appname);
resultOozieAppList = oozieAppList;
} else {
System.out.println("null_appname:" + appname);
resultOozieAppList = cp.filterOozieAppList(oozieAppList, appname);
}
List<String> jsListVertex = new ArrayList<String>();
List<String> jsListRelation = new ArrayList<String>();
List<String> jsListTable = new ArrayList<String>();
String bgcolorGrap = "rgb(202,250,250)";
String vertexInVar = "";
String vertexOutVar = "";
String vertexInName = "";
String vertexOutName = "";
String vertexInDesc = "";
String vertexOutDesc = "";
String vertexInStartTime = "";
String vertexOutStartTime = "";
// String startTime = "";
String frequency = "";
String vertexInStr = "";
String vertexOutStr = "";
String edgeStr = "";
String trStr = "";
String trStrIn = "";
String trStrOut = "";
String startTimeStr = "";
String frequencyStr = "";
for (OozieApp op : resultOozieAppList) {
vertexInName = op.getDataIn();
vertexOutName = op.getDataOut();
vertexInVar = vertexInName.replace("-", "_");
vertexOutVar = vertexOutName.replace("-", "_");
vertexOutStartTime = oozieAppMap.get(vertexOutName).getStartTime();
vertexOutDesc = vertexOutName;
//vertexOutDesc = vertexOutName + "(" + vertexOutStartTime + ")";
if (!vertexInName.equals("")) {
vertexInStartTime = oozieAppMap.get(vertexInName)
.getStartTime();
vertexInDesc = vertexInName;
//vertexInDesc = vertexInName + "(" + vertexInStartTime + ")";
edgeStr = "var " + vertexInVar + vertexOutVar
+ " = graph.insertEdge(parent, null, ''," + vertexInVar
+ " ," + vertexOutVar + ");";
if (vertexInName.equals(appname)) {
// System.out.println("jgkjh:"+vertexInDesc+oozieAppMap.get(op.getDataIn()));
vertexInStr = "var "
+ vertexInVar
+ " = graph.insertVertex(parent, null, '"
+ vertexInDesc
+ "', 200, 200, w, h,\"fillColor="
+ bgcolorGrap
+ ";rounded=true;perimeter=ellipsePerimeter;arcSize=60;\");";
trStrIn = "<td class=\"alt\" bgcolor=\"#0007C0\">"
+ vertexInDesc + "</td>";
} else {
vertexInStr = "var "
+ vertexInVar
+ " = graph.insertVertex(parent, null, '"
+ vertexInDesc
+ "', 200, 200, w, h,\"rounded=true;perimeter=ellipsePerimeter;arcSize=60;\");";
trStrIn = "<td>" + vertexInDesc + "</td>";
}
} else {
vertexInStr = "";
edgeStr = "";
trStrIn = "<td></td>";
}
if (vertexOutName.equals(appname)) {
vertexOutStr = "var "
+ vertexOutVar
+ " = graph.insertVertex(parent, null, '"
+ vertexOutDesc
+ "', 200, 200, w, h,\"fillColor="
+ bgcolorGrap
+ ";rounded=true;perimeter=ellipsePerimeter;arcSize=60;\");";
trStrOut = "<td class=\"alt\" bgcolor=\"#0007C0\">"
+ vertexOutDesc
+ "</td>";
} else {
vertexOutStr = "var "
+ vertexOutVar
+ " = graph.insertVertex(parent, null, '"
+ vertexOutDesc
+ "', 200, 200, w, h,\"rounded=true;perimeter=ellipsePerimeter;arcSize=60;\");";
trStrOut = "<td>" + vertexOutDesc + "</td>";
}
// System.out.println("jgkjh:"+op.getDataOut());
frequency = op.getFrequency();
frequencyStr = "<td>" + frequency + "</td>";
startTimeStr = "<td>" + vertexOutStartTime + "</td>";
trStr = "<tr>" + trStrOut + frequencyStr + startTimeStr + trStrIn
+ "</tr>";
if (!vertexInStr.equals("") && !jsListVertex.contains(vertexInStr)) {
jsListVertex.add(vertexInStr);
}
if (!edgeStr.equals("") && !jsListRelation.contains(edgeStr)) {
jsListRelation.add(edgeStr);
}
if (!jsListVertex.contains(vertexOutStr)) {
jsListVertex.add(vertexOutStr);
}
if (!jsListTable.contains(trStr)) {
jsListTable.add(trStr);
}
}
List<List<String>> listArray = new ArrayList<List<String>>();
listArray.add(jsListVertex);
listArray.add(jsListRelation);
listArray.add(jsListTable);
return listArray;
}
}
graphlayout-jsp.jsp
<%@ page language="java" contentType="text/html; charset=UTF-8"
import="java.util.List" pageEncoding="UTF-8"%>
<%@ taglib prefix="c" uri="http://java.sun.com/jstl/core_rt"%>
<%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>Hadoop dependent query</title>
</HEAD>
<!-- Sets the basepath for the library if not in same directory -->
<script LANGUAGE="JavaScript">
mxBasePath = '../src';
</script>
<!-- Loads and initializes the library -->
<script LANGUAGE="JavaScript" src="../src/js/mxClient.js"></script>
<!-- Example code -->
<script LANGUAGE="JavaScript">
function main(container) {
if (!mxClient.isBrowserSupported()) {
mxUtils.error('Browser is not supported!', 200, false);
} else {
var graph = new mxGraph(container);
graph.setEnabled(false);
//graph.setVertexLabelsMovable(true);
//graph.setTooltips(true);
var parent = graph.getDefaultParent();
var layout = new mxFastOrganicLayout(graph);
layout.forceConstant = 300;
var animate = document.getElementById('animate');
graph.getModel().beginUpdate();
var w = 300;
var h = 20;
try {
<c:forEach items="${javaScriptVertexList}" var="javaScriptVertexList">
<c:out value="${javaScriptVertexList}" escapeXml="false"/>
</c:forEach>
//========
<c:forEach items="${javaScriptRelationList}" var="javaScriptRelationList">
<c:out value="${javaScriptRelationList}" escapeXml="false"/>
</c:forEach>
layout.execute(parent);
} finally {
graph.getModel().endUpdate();
}
}
};
</script>
</head>
<style type="text/css">
body {
font: normal 11px auto "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
color: #4f6b72;
background: #E6EAE9;
}
a {
color: #c75f3e;
}
#mytable {
width: 700px;
padding: 0;
margin: 0;
}
caption {
padding: 0 0 5px 0;
width: 700px;
font: italic 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
text-align: right;
}
th {
font: bold 11px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
color: #4f6b72;
border-right: 1px solid #C1DAD7;
border-bottom: 1px solid #C1DAD7;
border-top: 1px solid #C1DAD7;
letter-spacing: 2px;
text-transform: uppercase;
text-align: left;
padding: 6px 6px 6px 12px;
background: #CAE8EA no-repeat;
}
th.nobg {
border-top: 0;
border-left: 0;
border-right: 1px solid #C1DAD7;
background: none;
}
td {
border-right: 1px solid #C1DAD7;
border-bottom: 1px solid #C1DAD7;
background: #fff;
font-size:11px;
padding: 6px 6px 6px 12px;
color: #4f6b72;
}
td.alt {
background: #CAF5F5;
color: #0008C0;
}
th.spec {
border-left: 1px solid #C1DAD7;
border-top: 0;
background: #fff no-repeat;
font: bold 10px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
}
th.specalt {
border-left: 1px solid #C1DAD7;
border-top: 0;
background: #f5fafa no-repeat;
font: bold 10px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
color: #797268;
}
html>body td{ font-size:11px;}
body,td,th {
font-family: 宋體, Arial;
font-size: 12px;
}
</style>
<body onload="main(document.getElementById('graphContainer'))">
<!-- main(document.getElementById('graphContainer'))-->
<br>
<h1>HADOOP資料平臺應用依賴關係查詢</h1>
<form id="form1" action="/TestWeb/mxGraph/examples/TestServlet"
method="post">
請輸入應用名稱:<input type="text" id="appid" name="appname" /><input
type="button" id="query" value="查詢"
onclick="document.getElementById('form1').submit();" />
</form>
<br>
<p>
使用說明：
輸入HADOOP應用名(如：pad-dp-view-pac-quick-upload-cd)查詢該應用的相關資訊，並向上、向下追溯依賴的及被依賴的應用，如果要查詢所有HADOOP應用資訊，請輸入“ALL”；
</p>
<table id="mytable" cellspacing="0" >
<caption> </caption>
<tr>
<th>應用名稱</th>
<th>排程頻率</th>
<th>排程時間</th>
<th>依賴於</th>
</tr>
<c:forEach items="${javaScriptTableList}" var="javaScriptTableList">
<c:out value="${javaScriptTableList}" escapeXml="false" />
</c:forEach>
</table>
<div id="graphContainer"
style="overflow: visible; width: 821px; height: 641px;"></div>
<br>
</body>
</html>
|
應用名稱 |
排程頻率 | 排程時間 | 依賴於 |
| pad-dp-view-kyc2-invest-median-cd | months(1) | 03T16:00 | |
| pad-dp-integrate-pac-epcis-cd | days(1) | 7:03 | pad-dp-integrate-public-cd |
| pad-dp-exp-nets2-pac-upload-cd | days(1) | 0:10 | pad-dp-view-pac-upload-cd |
| pad-dp-exp-sas-pac-vehicleowner-xb-cd | days(1) | 3:56 | pad-dp-view-pac-vehicleowner-xb-cd |
| pad-dp-imp-sql-invest-ods-init-cd | days(1) | 19:20 | pad-dp-imp-sql-invest-src-init-cd |
| pad-dp-integrate-invest-deep-cd | days(1) | 7:03 | pad-dp-integrate-invest-deep-pre-cd |
| pad-dp-view-life-complete-listno-cd | days(1) | 9:00 | pad-dp-imp-netslife-src-cd |
| pad-dp-view-life-complete-listno-cd | days(1) | 9:00 | pad-dp-integrate-life-netslife-cd |
| pad-dp-imp-sql-loan-sms-pre-src-cd | days(1) | 0:10 | |
| pad-dp-imp-padus-ods-cd | days(1) | 3:30 | pad-dp-imp-padus-src-cd |
| pad-dp-imp-fusion-pab-src-cd | days(1) | 4:00 | |
| pad-dp-imp-nbiz-loan-src-cd | days(1) | 4:00 | |
| pad-dp-imp-claim-pac-ods-cd | days(1) | 5:00 | pad-dp-imp-claim-pac-src-cd |
| pad-dp-exp-sql-invest-deep-kettle-cd | days(7) | 23:05 | pad-dp-integrate-invest-deep-cd |
| pad-dp-imp-tcims-loan-nbiz-upload-src-cd | days(1) | 2:30 | |
| pad-dp-monitor-hadoop-listno-cycle-cd | days(1) | 12:00 | |
| pad-dp-integrate-loan-sms-deep-cd | days(1) | 0:10 | pad-dp-integrate-loan-sms-cd |
| pad-dp-integrate-loan-sms-deep-cd | days(1) | 0:10 | pad-dp-imp-nbiz-loan-ods-cd |
| pad-dp-integrate-loan-sms-deep-cd | days(1) | 0:10 | pad-dp-imp-fusion-pab-ods-cd |
| pad-dp-exp-eshop-bbc-cd | days(1) | 2:03 | pad-dp-imp-elis-ods-cd |
| pad-dp-exp-eshop-bbc-cd | days(1) | 2:03 | pad-dp-exp-eshop-bbc-cd |
| pad-dp-exp-sql-life-elis-kettle-cd | days(1) | 21:00 |