package org.cipres.test; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.text.DecimalFormat; import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Observable; import java.util.Observer; import java.util.Properties; import java.util.TimeZone; import org.apache.log4j.Logger; import org.jdom.Document; import org.jdom.Element; import org.jdom.JDOMException; import org.jdom.input.SAXBuilder; import org.jdom.output.Format; import org.jdom.output.XMLOutputter; /** * define the *.xml format in the following ways: * note that analysis name is not really used, replaced by toolInfer, * but you must make sure the analysis name are unique * * * * * * * **/ public class TestCipresAction implements Observer { private static final Logger LOGGER = Logger.getLogger(TestCipresAction.class); public TestCipresAction() {} private static String inputXml = "input.xml"; private static final String nexusinfoFile = "nexusinfo.txt"; private static final String doneFile = "done"; private static final String df = "MM/dd/yy HH:mm:ss"; private static String remoteRegistry, cipresBuildDir, cipreslib, commandToolDir, cipresCp, datadirectory; private static String toNexus_datadirectory = "to_nexusfiles/"; private static String resultsDir = "results/"; private static String _resultsDir = null; private static String xmlfile = "scripts/NexusFiles.xml"; private static final String webcountDir = "../webcount.properties"; private static final String configFile = System.getProperty("user.dir") + "/scripts/configuration.xml"; private static final String mylib = System.getenv("M2_REPO") + "/org/cipres/webapp/cipres-test/1.0-SNAPSHOT/cipres-test-1.0-SNAPSHOT.jar"; private static final String jdomlib = System.getenv("M2_REPO") + "/jdom/jdom/1.0/jdom-1.0.jar"; private static final String observer = "org.cipres.test.TestCipresAction"; private static final String cipresAppName = "org.cipres.portal.RunServiceApp"; private static Document doc = null; private static String currentXmlFile = null; /** * main program to kick off the run * * @param args * @throws Exception */ public static void main( String[] args ) throws Exception { currentXmlFile = xmlfile; // read the configuration file processConfigFile(); // print variable printVars(); // get the xml file and convert to Document type doc = xml2doc(); // update the xml file to get list of tools to run //getTools(doc); // convert all the file to nexus, determine the datatype, and ntaxa // run this only if we are doing end-to-end testings //if (remoteRegistry != null && remoteRegistry.length() > 0) preprocessData(); preprocessData(); // run the next analysis runNextAnalysis(doc); } // end main private static void printVars() { System.out.println("*****************************************************"); System.out.println("remoteRegistry\t = " + remoteRegistry); System.out.println("cipresBuildDir\t = " + cipresBuildDir); System.out.println("commandToolDir\t = " + commandToolDir); System.out.println("datadirectory\t = " + datadirectory); System.out.println("resultsDir\t = " + resultsDir); System.out.println("xmlfile\t = " + xmlfile); System.out.println("webcountDir\t = " + webcountDir); System.out.println("configFile\t = " + configFile); System.out.println("cipreslib\t = " + cipreslib); System.out.println("cipresCp\t = " + cipresCp); System.out.println("observer\t = " + observer); System.out.println("cipresAppName\t = " + cipresAppName); System.out.println("*****************************************************"); } /** * get variables from configuration.xml * **/ private static void processConfigFile() { FileInputStream stream = null; try { // load the configuration file stream = new FileInputStream(configFile); Properties properties = new Properties(); properties.loadFromXML(stream); remoteRegistry = properties.getProperty("remoteRegistry"); cipresBuildDir= properties.getProperty("cipresBuildDir"); commandToolDir = properties.getProperty("commandToolDir"); datadirectory = properties.getProperty("nexusDir"); // update variables cipreslib = cipresBuildDir + "lib/cipres/cipres-classpath.jar"; cipresCp = cipreslib + ":" + mylib + ":" + jdomlib; } catch (Exception ex) { LOGGER.debug("Error in process configuration.xml properties file:" + ex); } finally { if (stream != null) try {stream.close();} catch (Exception ex) {;} } } /** * get the next analysis to run, and record the starting time in NexusFiles.xml * * @param doc */ private static void runNextAnalysis(Document doc) { String fileRun = null; String analysisRun = null; String treeInfer = null; // name of the paup infer file String toolFile= null; // name of the command tool file String timeout = null; // timeout in mins int exitValue = 0; List list = doc.getRootElement().getChildren(); for (Element element: list) // list of { fileRun = element.getAttributeValue("name"); List analyses = element.getChildren(); // list of for (Element analysis: analyses) { analysisRun = analysis.getAttributeValue("name"); String start = analysis.getAttributeValue("start"); String stop = analysis.getAttributeValue("stop"); treeInfer = analysis.getAttributeValue("treeInfer"); toolFile = analysis.getAttributeValue("toolFile"); timeout = analysis.getAttributeValue("timeout"); String RunServiceApp = analysis.getAttributeValue("RunServiceApp"); // find the next analysis to run if (RunServiceApp == null) { start = getCurrentTime(df); addAttribute(analysis, "start", start); LOGGER.debug("==== Analysis to run " + analysisRun + " for " + fileRun + "====="); // get the datatype and ndata exitValue = startAnalysis(doc, fileRun, analysisRun, treeInfer, toolFile, timeout, start); LOGGER.debug("Exit Value return = " + exitValue); if (exitValue == 0) { addAttribute(analysis, "RunServiceApp", "0"); // analysis is done, process data received from cipres processData(fileRun, analysisRun); } else { stop = getCurrentTime(df); addAttribute(analysis, "stop", stop); addAttribute(analysis, "RunServiceApp", "-1"); LOGGER.debug("Exit Value for startAnalysis returns error " + exitValue); } } } // end for } // end for LOGGER.debug("All done. No more analysis to run"); System.exit(0); } private static void preprocessData() { String fileRun = null; String analysisRun = null; int exitValue = 0; String cipresbin = cipresBuildDir + "bin" + "/"; List list = doc.getRootElement().getChildren(); for (Element element: list) // list of { // parse the file to nexus fileRun = element.getAttributeValue("name"); // user paup as the parser //String command = "python " + System.getenv("CIPRES_HOME") +"/CIPRES-and-deps/cipres/framework/etc/to_nexus.py"; // command += " --file=" + toNexus_datadirectory + fileRun + " " + datadirectory + fileRun; String[] commandArray1 = {"python", cipresbin+"to_nexus.py", "--phylip-to-nexus="+cipresbin+"phylip_to_nexus", "--file="+toNexus_datadirectory+fileRun, datadirectory+fileRun}; exitValue = submitJob(commandArray1); String value = (exitValue == 0) ? (fileRun) : (""); addAttribute(element, "toNexus", value); if (exitValue != 0) continue; // skip RunNexusReader // determine the datatype, ntaxa of the file String propertyFilePath = resultsDir + nexusinfoFile; String[] commandArray2 = new String[] { "java", "-cp", cipreslib, "-DoriginalNexusName=" + fileRun, "-DnexusFile=" + toNexus_datadirectory + fileRun, "-DpropertyFile=" + propertyFilePath, "-DtreeInferTool=null", "org.cipres.portal.RunNexusReader" }; exitValue = submitJob(commandArray2); String datatype = ""; String ntaxa = ""; if (exitValue == 0) { try { // locate the nexusinfo.txt properties file InputStream stream = new FileInputStream(propertyFilePath); Properties properties = new Properties(); properties.load(stream); datatype = properties.getProperty("datatype"); ntaxa = properties.getProperty("ntaxa"); } catch (Exception ex) { LOGGER.debug("Error in process nexusinfo properties file:" + ex); } finally { if (datatype == null) datatype=""; if (ntaxa == null) ntaxa=""; addAttribute(element, "datatype", datatype); addAttribute(element, "ntaxa", ntaxa); } } else { } } // end for } private static int submitJob(String[] commandArray) { int exitValue = 0; LOGGER.debug("submitJob with command " + Arrays.toString(commandArray)); try { Runtime rt = Runtime.getRuntime(); Process process = rt.exec(commandArray); // any error message? StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream(), "ERROR"); // any output? StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream(), "OUTPUT"); // start reading outputGobbler.start(); errorGobbler.start(); exitValue = process.waitFor(); LOGGER.debug("Process returns exitValue = " + exitValue); } catch(Exception ex) { ex.printStackTrace(); } LOGGER.debug("Return Code is " + exitValue); return exitValue; } /** * update the xml file * */ private static void updateXml() { FileOutputStream fo = null; try { // update the xml file Format format = Format.getPrettyFormat(); XMLOutputter output = new XMLOutputter(format); fo = new FileOutputStream(currentXmlFile); output.output(doc, fo); } catch (IOException ex) { ex.printStackTrace(); } finally { if (fo!= null) try {fo.close();} catch (Exception ex) {} } } /** * return the analysis node in NexusFiles.xml given the file name and analysis name * * @param attrName * @param attrValue */ private static Element getAnalysisNode(Document doc, String fileName, String analysisName) { String file = null; String analysis = null; boolean found = false; Element foundElmenet = null; // find the appropriate node, and add the attribute List list = doc.getRootElement().getChildren(); for (Element element: list) { if (found) break; file = element.getAttributeValue("name"); if (!file.equals(fileName)) continue; List analyses = element.getChildren("analysis"); for (Element e: analyses) { analysis = e.getAttributeValue("name"); if (analysis.equals(analysisName)) { found = true; foundElmenet = e; break; } // end if } // end for } // end for return foundElmenet; } /** * add an attribute a specified element * * @param element * @param attrName * @param attrValue */ private static void addAttribute(Element element,String attrName, String attrValue) { element.setAttribute(attrName, attrValue); updateXml(); } /** * start running the job here * * @param doc * @param fileName * @param analysisName * @param start */ private static int startAnalysis(Document doc, String fileName, String analysisName, String treeInfer, String toolFile, String timeout, String start) { _resultsDir = resultsDir + toolFile + "_X_" + treeInfer + "/"; // make sure results directory is create new File(_resultsDir).mkdir(); // read count and increment it int count = getWebCount(); // get the current date and attach to groupLabel Date now = new Date(); SimpleDateFormat sdf = new SimpleDateFormat("MM-dd-yyyy"); //String paupInferFile = (analysisName.indexOf("paup") >=0) ? // "paup_infer.xml" : "quick_paup_infer.xml"; // if (treeInfer != null) paupInferFile = treeInfer; // if preprocessData() is not executed, we need to get the // nexus file from datadirectory //if (remoteRegistry == null || remoteRegistry.length() <= 0) //{ //toNexus_datadirectory = datadirectory; //} String[] commandArray = new String[] { "java", "-cp", cipresCp, "-DoriginalNexusName=" + fileName, "-DnexusFile=" + toNexus_datadirectory + fileName, "-DtoolFile=" + commandToolDir + toolFile, "-DtreeInferTool=" + commandToolDir + treeInfer, "-DtoolName=" + analysisName, "-DinputType=" + "nexus", "-DresultsDir=" + _resultsDir, "-DgroupLabel=" + "Job-" + count + "(lcchan@sdsc.edu)", "-Dobserver=" + observer, "-DremoteRegistry=" + remoteRegistry, "-Dtimeout=" + timeout, cipresAppName }; LOGGER.debug("Command to run analysis: " + Arrays.toString(commandArray)); // save the inputs saveInputs(fileName, analysisName, treeInfer, toolFile, start); int exitValue = submitJob(commandArray); return exitValue; } /** * save the information so we can use it later to retrieve the starting time for a job * * @param fileName * @param analysisName * @param start */ private static void saveInputs(String fileName, String analysisName, String treeInfer, String toolFile, String start) { Properties properties = new Properties(); properties.setProperty("file", fileName); properties.setProperty("analysis", analysisName); properties.setProperty("start", start); String filename = _resultsDir + inputXml; try { LOGGER.debug("Storing input.xml in " + filename); properties.storeToXML(new FileOutputStream(filename), null); } catch (IOException ex) { LOGGER.debug("Error in writing data to input.xml: " + ex.toString()); } } /** * get list of the tools to run for this node * * @param nodeName * @return */ private static void addAnalysisNode(Element element, String tool) { Element e = new Element("analysis"); e.setAttribute("name", tool); element.addContent(e); updateXml(); } /** * get the list of analysese to run using the nexus file information * * @param doc */ private static void getTools(Document doc) { String ntaxa_str = null; List list = doc.getRootElement().getChildren(); // for each of element for(Element element: list) { if (element.getChildren("analysis").size() > 0 ) continue; String node = element.getAttributeValue("name"); //************** PAUP *********************** // base method addAnalysisNode(element, "paup"); // recidcm3 boosted version addAnalysisNode(element, "rid3-paup-12.5-pct"); addAnalysisNode(element, "rid3-paup-25-pct"); addAnalysisNode(element, "rid3-paup-50-pct"); //************** GARLI ************************ // base method addAnalysisNode(element, "garli-gtr-g"); // recidcm3 boosted versoin addAnalysisNode(element, "rid3-garli-gtr-g-12.5-pct"); addAnalysisNode(element, "rid3-garli-gtr-g-25-pct"); addAnalysisNode(element, "rid3-garli-gtr-g-50-pct"); //*************** RAxML **************************** // base method addAnalysisNode(element, "recidcm3_raxml"); // recidcm3 boosted version addAnalysisNode(element, "rid3-rax-12.5-pct"); addAnalysisNode(element, "rid3-rax-25-pct"); addAnalysisNode(element, "rid3-rax-50-pct"); } // end for } /** * convert NexusFilex.xml to a DOM object * * @return */ private static Document xml2doc() { LOGGER.debug("Parsing xml file to Document object: " + currentXmlFile); Document doc = null; // create a parser SAXBuilder parser = new SAXBuilder(); try { // get the DOM document of the xml file LOGGER.debug("Parsing xml: " + currentXmlFile); doc = parser.build(new File(currentXmlFile)); } catch (JDOMException ex) { ex.printStackTrace(); } catch (IOException ex) { ex.printStackTrace(); } return doc; } // end xml2doc /** * copy file from one to another * * @param src * @param dst * @throws IOException */ private static void copy(File src, File dst) throws IOException { InputStream in = new FileInputStream(src); OutputStream out = new FileOutputStream(dst); // Transfer bytes from in to out byte[] buf = new byte[1024]; int len; while ((len = in.read(buf)) > 0) { out.write(buf, 0, len); } in.close(); out.close(); } public void update(Observable o, Object arg) { LOGGER.debug("update is called by client program with " + arg.toString()); // update node ? } public static void processData(String fileRun, String analysisRun) { LOGGER.debug("Calling processData() for " + currentXmlFile); // get the xml file and convert to Document type doc = xml2doc(); Double bestML = null; String bestScore = ""; String iteration = ""; String bestIteration = null; Properties properties = new Properties(); // load input.xml try { String filename = _resultsDir + inputXml; properties.loadFromXML(new FileInputStream(filename)); } catch (Exception ex) { LOGGER.debug("Error in reading input.xml " + ex.toString()); } String fileName = properties.getProperty("file"); String analysisName = properties.getProperty("analysis"); String start = properties.getProperty("start"); if (!fileName.equals(fileRun) || !analysisName.equals(analysisRun)) { LOGGER.debug("Error - filename and analysis do not match " + fileName+"("+fileRun+");"+analysisName+"("+analysisRun+")"); System.exit(-1); } LOGGER.debug("File and analysis names matched.... proceed..."); // locate the element node we are dealing with Element element = getAnalysisNode(doc, fileName, analysisName); // read done file and add it to xml file if it's not empty try { String filename = _resultsDir + doneFile; String savedFilename = _resultsDir + analysisName + "_" + fileName + ".done"; // save a copy of the "done" file and rename it LOGGER.debug("Copy files from " + filename + " to " + savedFilename); copy(new File(filename), new File(savedFilename)); BufferedReader in = new BufferedReader(new FileReader(_resultsDir + doneFile)); String str; while ((str = in.readLine()) != null) { addAttribute(element, "message", str); } in.close(); } catch (IOException ex) { LOGGER.debug("Error in reading done file " + ex.toString()); } // record the stop time in xml file String stop = getCurrentTime(df); addAttribute(element, "stop", stop); // calculate time elapsed try { String elapsedHMS = getElapsedTime(df, start, stop); addAttribute(element, "time", elapsedHMS); } catch (Exception ex) { LOGGER.debug("Exception in calculating time elapsed: " + ex.toString()); } // try to locate ML score, and if found, add them to xml file BufferedReader in = null; try { String filename = _resultsDir + analysisName + "_" + fileName; LOGGER.debug("Getting result file in:" + filename); in = new BufferedReader(new FileReader(filename)); String line; while ( (line = in.readLine()) != null) { //convert the line to lower String l_line = line.toLowerCase(); int loc = 0; if ( (loc = l_line.indexOf("score=")) >= 0) { // record each of the score found as a child node Element mLElement = new Element("ML"); // do record everything in the line up to where score is found int endIndex = l_line.indexOf("]") +1; mLElement.setAttribute("score", line.substring(0, endIndex)); // add a child node and update element element.addContent(mLElement); updateXml(); // try to figure out the best ML score int beginIndex = new String("score=").length(); String score = line.substring(loc + beginIndex, endIndex -1); double score_d = 0D; // skip if no score is found if (score == null) continue; // skip if score doesn't contain any digit try { score_d = Double.valueOf(score).doubleValue(); } catch (NumberFormatException ex) { continue; } // skip if score is not a negative number if (score_d > 0) continue; // record the iteration where this occurs String iteration_pattern = "iteration"; int iteration_begin = l_line.indexOf(iteration_pattern); // there is no iteration for base method if (iteration_begin < 0) { iteration = null; } else { int iteration_end = l_line.indexOf("(", iteration_begin); iteration = line.substring(iteration_begin, iteration_end-1); } if (bestML == null) { bestML = new Double(score_d); } if (Math.abs(score_d) < Math.abs(bestML.doubleValue())) { bestML = new Double(score_d); bestIteration = iteration; } bestScore = (bestML == null) ? (""):(bestML.toString()); bestIteration = (bestIteration == null) ? (""):(bestIteration.toString()); } // end if } // end while } catch (Exception ex) { LOGGER.debug("Exception caught in reading result file " + ex.toString()); } finally { LOGGER.debug("closing file...."); if (in != null) try {in.close();} catch (Exception ex) {} } // record best score and iteration if (bestScore.equals("")) bestIteration = ""; addAttribute(element, "bestScore", bestScore); addAttribute(element, "bestIteration", bestIteration); runNextAnalysis(doc); } private static String getCurrentTime(String df) { SimpleDateFormat sdf = new SimpleDateFormat(df); Calendar calendar = Calendar.getInstance(TimeZone.getDefault()); sdf.setTimeZone(TimeZone.getDefault()); String time = sdf.format(calendar.getTime()); return time; } private static String getElapsedTime(String df, String start, String stop) throws Exception { LOGGER.debug("Start and Stop times = " + start + ";" + stop); SimpleDateFormat sdf = new SimpleDateFormat(df); String elapsed = new String(); long mstart = sdf.parse(start).getTime(); long mstop = sdf.parse(stop).getTime(); long timeInSeconds_l = (mstop - mstart) / 1000; int timeInSeconds = (int)timeInSeconds_l; int hours = timeInSeconds / 3600; timeInSeconds = timeInSeconds - (hours *3600); int minutes = timeInSeconds / 60; timeInSeconds = timeInSeconds - (minutes *60); int seconds = timeInSeconds; elapsed = Integer.toString(hours) +"h:"; elapsed += Integer.toString(minutes) + "m:"; elapsed += Integer.toString(seconds) + "s"; LOGGER.debug("Elapsed time is " + elapsed); return elapsed; } /** * * * @param request * @return */ private synchronized static int getWebCount() { String key = "WebCount"; String count = null; FileInputStream fi = null; FileOutputStream fo = null; Properties properties = new Properties(); try { // make sure file exits File file = new File(webcountDir); LOGGER.debug("webcountDir" + webcountDir); if (!file.exists()) { file.createNewFile(); } else { fi = new FileInputStream(webcountDir); properties.loadFromXML(fi); count = properties.getProperty(key); } count = (count == null) ? ("1") : (count); // update the count in the properties file now fo = new FileOutputStream(webcountDir); int newCount = Integer.valueOf(count).intValue(); // increment the value properties.setProperty(key, Integer.toString(++newCount)); properties.storeToXML(fo, null); } catch (Exception ex) { LOGGER.debug("Error in loading the web info properties file" + ex.toString()); } finally { if (fi != null) try {fi.close();} catch (Exception ex){} if (fo != null) try {fo.close();} catch (Exception ex){} } // return the count return Integer.parseInt(count); } }