package org.cipres.test;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Observable;
import java.util.Observer;
import java.util.Properties;
import java.util.TimeZone;
import org.apache.log4j.Logger;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;
/**
* define the *.xml format in the following ways:
* note that analysis name is not really used, replaced by toolInfer,
* but you must make sure the analysis name are unique
*
*
*
*
*
*
*
**/
public class TestCipresAction implements Observer {
private static final Logger LOGGER = Logger.getLogger(TestCipresAction.class);
public TestCipresAction() {}
private static String inputXml = "input.xml";
private static final String nexusinfoFile = "nexusinfo.txt";
private static final String doneFile = "done";
private static final String df = "MM/dd/yy HH:mm:ss";
private static String remoteRegistry, cipresBuildDir, cipreslib, commandToolDir, cipresCp, datadirectory;
private static String toNexus_datadirectory = "to_nexusfiles/";
private static String resultsDir = "results/";
private static String _resultsDir = null;
private static String xmlfile = "scripts/NexusFiles.xml";
private static final String webcountDir = "../webcount.properties";
private static final String configFile = System.getProperty("user.dir") + "/scripts/configuration.xml";
private static final String mylib = System.getenv("M2_REPO") + "/org/cipres/webapp/cipres-test/1.0-SNAPSHOT/cipres-test-1.0-SNAPSHOT.jar";
private static final String jdomlib = System.getenv("M2_REPO") + "/jdom/jdom/1.0/jdom-1.0.jar";
private static final String observer = "org.cipres.test.TestCipresAction";
private static final String cipresAppName = "org.cipres.portal.RunServiceApp";
private static Document doc = null;
private static String currentXmlFile = null;
/**
* main program to kick off the run
*
* @param args
* @throws Exception
*/
public static void main( String[] args ) throws Exception
{
currentXmlFile = xmlfile;
// read the configuration file
processConfigFile();
// print variable
printVars();
// get the xml file and convert to Document type
doc = xml2doc();
// update the xml file to get list of tools to run
//getTools(doc);
// convert all the file to nexus, determine the datatype, and ntaxa
// run this only if we are doing end-to-end testings
//if (remoteRegistry != null && remoteRegistry.length() > 0) preprocessData();
preprocessData();
// run the next analysis
runNextAnalysis(doc);
} // end main
private static void printVars()
{
System.out.println("*****************************************************");
System.out.println("remoteRegistry\t = " + remoteRegistry);
System.out.println("cipresBuildDir\t = " + cipresBuildDir);
System.out.println("commandToolDir\t = " + commandToolDir);
System.out.println("datadirectory\t = " + datadirectory);
System.out.println("resultsDir\t = " + resultsDir);
System.out.println("xmlfile\t = " + xmlfile);
System.out.println("webcountDir\t = " + webcountDir);
System.out.println("configFile\t = " + configFile);
System.out.println("cipreslib\t = " + cipreslib);
System.out.println("cipresCp\t = " + cipresCp);
System.out.println("observer\t = " + observer);
System.out.println("cipresAppName\t = " + cipresAppName);
System.out.println("*****************************************************");
}
/**
* get variables from configuration.xml
*
**/
private static void processConfigFile()
{
FileInputStream stream = null;
try
{
// load the configuration file
stream = new FileInputStream(configFile);
Properties properties = new Properties();
properties.loadFromXML(stream);
remoteRegistry = properties.getProperty("remoteRegistry");
cipresBuildDir= properties.getProperty("cipresBuildDir");
commandToolDir = properties.getProperty("commandToolDir");
datadirectory = properties.getProperty("nexusDir");
// update variables
cipreslib = cipresBuildDir + "lib/cipres/cipres-classpath.jar";
cipresCp = cipreslib + ":" + mylib + ":" + jdomlib;
}
catch (Exception ex)
{
LOGGER.debug("Error in process configuration.xml properties file:" + ex);
}
finally
{
if (stream != null) try {stream.close();} catch (Exception ex) {;}
}
}
/**
* get the next analysis to run, and record the starting time in NexusFiles.xml
*
* @param doc
*/
private static void runNextAnalysis(Document doc)
{
String fileRun = null;
String analysisRun = null;
String treeInfer = null; // name of the paup infer file
String toolFile= null; // name of the command tool file
String timeout = null; // timeout in mins
int exitValue = 0;
List list = doc.getRootElement().getChildren();
for (Element element: list) // list of
{
fileRun = element.getAttributeValue("name");
List analyses = element.getChildren(); // list of
for (Element analysis: analyses)
{
analysisRun = analysis.getAttributeValue("name");
String start = analysis.getAttributeValue("start");
String stop = analysis.getAttributeValue("stop");
treeInfer = analysis.getAttributeValue("treeInfer");
toolFile = analysis.getAttributeValue("toolFile");
timeout = analysis.getAttributeValue("timeout");
String RunServiceApp = analysis.getAttributeValue("RunServiceApp");
// find the next analysis to run
if (RunServiceApp == null)
{
start = getCurrentTime(df);
addAttribute(analysis, "start", start);
LOGGER.debug("==== Analysis to run " + analysisRun + " for " + fileRun + "=====");
// get the datatype and ndata
exitValue = startAnalysis(doc, fileRun, analysisRun, treeInfer, toolFile, timeout, start);
LOGGER.debug("Exit Value return = " + exitValue);
if (exitValue == 0)
{
addAttribute(analysis, "RunServiceApp", "0");
// analysis is done, process data received from cipres
processData(fileRun, analysisRun);
}
else
{
stop = getCurrentTime(df);
addAttribute(analysis, "stop", stop);
addAttribute(analysis, "RunServiceApp", "-1");
LOGGER.debug("Exit Value for startAnalysis returns error " + exitValue);
}
}
} // end for
} // end for
LOGGER.debug("All done. No more analysis to run");
System.exit(0);
}
private static void preprocessData()
{
String fileRun = null;
String analysisRun = null;
int exitValue = 0;
String cipresbin = cipresBuildDir + "bin" + "/";
List list = doc.getRootElement().getChildren();
for (Element element: list) // list of
{
// parse the file to nexus
fileRun = element.getAttributeValue("name");
// user paup as the parser
//String command = "python " + System.getenv("CIPRES_HOME") +"/CIPRES-and-deps/cipres/framework/etc/to_nexus.py";
// command += " --file=" + toNexus_datadirectory + fileRun + " " + datadirectory + fileRun;
String[] commandArray1 = {"python",
cipresbin+"to_nexus.py",
"--phylip-to-nexus="+cipresbin+"phylip_to_nexus",
"--file="+toNexus_datadirectory+fileRun,
datadirectory+fileRun};
exitValue = submitJob(commandArray1);
String value = (exitValue == 0) ? (fileRun) : ("");
addAttribute(element, "toNexus", value);
if (exitValue != 0) continue; // skip RunNexusReader
// determine the datatype, ntaxa of the file
String propertyFilePath = resultsDir + nexusinfoFile;
String[] commandArray2 = new String[] {
"java",
"-cp", cipreslib,
"-DoriginalNexusName=" + fileRun,
"-DnexusFile=" + toNexus_datadirectory + fileRun,
"-DpropertyFile=" + propertyFilePath,
"-DtreeInferTool=null",
"org.cipres.portal.RunNexusReader"
};
exitValue = submitJob(commandArray2);
String datatype = "";
String ntaxa = "";
if (exitValue == 0)
{
try
{
// locate the nexusinfo.txt properties file
InputStream stream = new FileInputStream(propertyFilePath);
Properties properties = new Properties();
properties.load(stream);
datatype = properties.getProperty("datatype");
ntaxa = properties.getProperty("ntaxa");
}
catch (Exception ex)
{
LOGGER.debug("Error in process nexusinfo properties file:" + ex);
}
finally
{
if (datatype == null) datatype="";
if (ntaxa == null) ntaxa="";
addAttribute(element, "datatype", datatype);
addAttribute(element, "ntaxa", ntaxa);
}
}
else
{
}
} // end for
}
private static int submitJob(String[] commandArray)
{
int exitValue = 0;
LOGGER.debug("submitJob with command " + Arrays.toString(commandArray));
try
{
Runtime rt = Runtime.getRuntime();
Process process = rt.exec(commandArray);
// any error message?
StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream(), "ERROR");
// any output?
StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream(), "OUTPUT");
// start reading
outputGobbler.start();
errorGobbler.start();
exitValue = process.waitFor();
LOGGER.debug("Process returns exitValue = " + exitValue);
}
catch(Exception ex)
{
ex.printStackTrace();
}
LOGGER.debug("Return Code is " + exitValue);
return exitValue;
}
/**
* update the xml file
*
*/
private static void updateXml()
{
FileOutputStream fo = null;
try
{
// update the xml file
Format format = Format.getPrettyFormat();
XMLOutputter output = new XMLOutputter(format);
fo = new FileOutputStream(currentXmlFile);
output.output(doc, fo);
}
catch (IOException ex)
{
ex.printStackTrace();
}
finally
{
if (fo!= null) try {fo.close();} catch (Exception ex) {}
}
}
/**
* return the analysis node in NexusFiles.xml given the file name and analysis name
*
* @param attrName
* @param attrValue
*/
private static Element getAnalysisNode(Document doc, String fileName, String analysisName)
{
String file = null;
String analysis = null;
boolean found = false;
Element foundElmenet = null;
// find the appropriate node, and add the attribute
List list = doc.getRootElement().getChildren();
for (Element element: list)
{
if (found) break;
file = element.getAttributeValue("name");
if (!file.equals(fileName)) continue;
List analyses = element.getChildren("analysis");
for (Element e: analyses)
{
analysis = e.getAttributeValue("name");
if (analysis.equals(analysisName))
{
found = true;
foundElmenet = e;
break;
} // end if
} // end for
} // end for
return foundElmenet;
}
/**
* add an attribute a specified element
*
* @param element
* @param attrName
* @param attrValue
*/
private static void addAttribute(Element element,String attrName, String attrValue)
{
element.setAttribute(attrName, attrValue);
updateXml();
}
/**
* start running the job here
*
* @param doc
* @param fileName
* @param analysisName
* @param start
*/
private static int startAnalysis(Document doc, String fileName, String analysisName, String treeInfer, String toolFile, String timeout, String start)
{
_resultsDir = resultsDir + toolFile + "_X_" + treeInfer + "/";
// make sure results directory is create
new File(_resultsDir).mkdir();
// read count and increment it
int count = getWebCount();
// get the current date and attach to groupLabel
Date now = new Date();
SimpleDateFormat sdf = new SimpleDateFormat("MM-dd-yyyy");
//String paupInferFile = (analysisName.indexOf("paup") >=0) ?
// "paup_infer.xml" : "quick_paup_infer.xml";
// if (treeInfer != null) paupInferFile = treeInfer;
// if preprocessData() is not executed, we need to get the
// nexus file from datadirectory
//if (remoteRegistry == null || remoteRegistry.length() <= 0)
//{
//toNexus_datadirectory = datadirectory;
//}
String[] commandArray = new String[] {
"java",
"-cp", cipresCp,
"-DoriginalNexusName=" + fileName,
"-DnexusFile=" + toNexus_datadirectory + fileName,
"-DtoolFile=" + commandToolDir + toolFile,
"-DtreeInferTool=" + commandToolDir + treeInfer,
"-DtoolName=" + analysisName,
"-DinputType=" + "nexus",
"-DresultsDir=" + _resultsDir,
"-DgroupLabel=" + "Job-" + count + "(lcchan@sdsc.edu)",
"-Dobserver=" + observer,
"-DremoteRegistry=" + remoteRegistry,
"-Dtimeout=" + timeout,
cipresAppName
};
LOGGER.debug("Command to run analysis: " + Arrays.toString(commandArray));
// save the inputs
saveInputs(fileName, analysisName, treeInfer, toolFile, start);
int exitValue = submitJob(commandArray);
return exitValue;
}
/**
* save the information so we can use it later to retrieve the starting time for a job
*
* @param fileName
* @param analysisName
* @param start
*/
private static void saveInputs(String fileName, String analysisName, String treeInfer, String toolFile, String start)
{
Properties properties = new Properties();
properties.setProperty("file", fileName);
properties.setProperty("analysis", analysisName);
properties.setProperty("start", start);
String filename = _resultsDir + inputXml;
try
{
LOGGER.debug("Storing input.xml in " + filename);
properties.storeToXML(new FileOutputStream(filename), null);
}
catch (IOException ex)
{
LOGGER.debug("Error in writing data to input.xml: " + ex.toString());
}
}
/**
* get list of the tools to run for this node
*
* @param nodeName
* @return
*/
private static void addAnalysisNode(Element element, String tool)
{
Element e = new Element("analysis");
e.setAttribute("name", tool);
element.addContent(e);
updateXml();
}
/**
* get the list of analysese to run using the nexus file information
*
* @param doc
*/
private static void getTools(Document doc)
{
String ntaxa_str = null;
List list = doc.getRootElement().getChildren();
// for each of element
for(Element element: list)
{
if (element.getChildren("analysis").size() > 0 ) continue;
String node = element.getAttributeValue("name");
//************** PAUP ***********************
// base method
addAnalysisNode(element, "paup");
// recidcm3 boosted version
addAnalysisNode(element, "rid3-paup-12.5-pct");
addAnalysisNode(element, "rid3-paup-25-pct");
addAnalysisNode(element, "rid3-paup-50-pct");
//************** GARLI ************************
// base method
addAnalysisNode(element, "garli-gtr-g");
// recidcm3 boosted versoin
addAnalysisNode(element, "rid3-garli-gtr-g-12.5-pct");
addAnalysisNode(element, "rid3-garli-gtr-g-25-pct");
addAnalysisNode(element, "rid3-garli-gtr-g-50-pct");
//*************** RAxML ****************************
// base method
addAnalysisNode(element, "recidcm3_raxml");
// recidcm3 boosted version
addAnalysisNode(element, "rid3-rax-12.5-pct");
addAnalysisNode(element, "rid3-rax-25-pct");
addAnalysisNode(element, "rid3-rax-50-pct");
} // end for
}
/**
* convert NexusFilex.xml to a DOM object
*
* @return
*/
private static Document xml2doc()
{
LOGGER.debug("Parsing xml file to Document object: " + currentXmlFile);
Document doc = null;
// create a parser
SAXBuilder parser = new SAXBuilder();
try
{
// get the DOM document of the xml file
LOGGER.debug("Parsing xml: " + currentXmlFile);
doc = parser.build(new File(currentXmlFile));
}
catch (JDOMException ex)
{
ex.printStackTrace();
}
catch (IOException ex)
{
ex.printStackTrace();
}
return doc;
} // end xml2doc
/**
* copy file from one to another
*
* @param src
* @param dst
* @throws IOException
*/
private static void copy(File src, File dst) throws IOException {
InputStream in = new FileInputStream(src);
OutputStream out = new FileOutputStream(dst);
// Transfer bytes from in to out
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) > 0) {
out.write(buf, 0, len);
}
in.close();
out.close();
}
public void update(Observable o, Object arg)
{
LOGGER.debug("update is called by client program with " + arg.toString());
// update node ?
}
public static void processData(String fileRun, String analysisRun)
{
LOGGER.debug("Calling processData() for " + currentXmlFile);
// get the xml file and convert to Document type
doc = xml2doc();
Double bestML = null;
String bestScore = "";
String iteration = "";
String bestIteration = null;
Properties properties = new Properties();
// load input.xml
try
{
String filename = _resultsDir + inputXml;
properties.loadFromXML(new FileInputStream(filename));
}
catch (Exception ex)
{
LOGGER.debug("Error in reading input.xml " + ex.toString());
}
String fileName = properties.getProperty("file");
String analysisName = properties.getProperty("analysis");
String start = properties.getProperty("start");
if (!fileName.equals(fileRun) || !analysisName.equals(analysisRun))
{
LOGGER.debug("Error - filename and analysis do not match " + fileName+"("+fileRun+");"+analysisName+"("+analysisRun+")");
System.exit(-1);
}
LOGGER.debug("File and analysis names matched.... proceed...");
// locate the element node we are dealing with
Element element = getAnalysisNode(doc, fileName, analysisName);
// read done file and add it to xml file if it's not empty
try {
String filename = _resultsDir + doneFile;
String savedFilename = _resultsDir + analysisName + "_" + fileName + ".done";
// save a copy of the "done" file and rename it
LOGGER.debug("Copy files from " + filename + " to " + savedFilename);
copy(new File(filename), new File(savedFilename));
BufferedReader in = new BufferedReader(new FileReader(_resultsDir + doneFile));
String str;
while ((str = in.readLine()) != null) {
addAttribute(element, "message", str);
}
in.close();
} catch (IOException ex) {
LOGGER.debug("Error in reading done file " + ex.toString());
}
// record the stop time in xml file
String stop = getCurrentTime(df);
addAttribute(element, "stop", stop);
// calculate time elapsed
try
{
String elapsedHMS = getElapsedTime(df, start, stop);
addAttribute(element, "time", elapsedHMS);
}
catch (Exception ex)
{
LOGGER.debug("Exception in calculating time elapsed: " + ex.toString());
}
// try to locate ML score, and if found, add them to xml file
BufferedReader in = null;
try
{
String filename = _resultsDir + analysisName + "_" + fileName;
LOGGER.debug("Getting result file in:" + filename);
in = new BufferedReader(new FileReader(filename));
String line;
while ( (line = in.readLine()) != null)
{
//convert the line to lower
String l_line = line.toLowerCase();
int loc = 0;
if ( (loc = l_line.indexOf("score=")) >= 0)
{
// record each of the score found as a child node
Element mLElement = new Element("ML");
// do record everything in the line up to where score is found
int endIndex = l_line.indexOf("]") +1;
mLElement.setAttribute("score", line.substring(0, endIndex));
// add a child node and update element
element.addContent(mLElement);
updateXml();
// try to figure out the best ML score
int beginIndex = new String("score=").length();
String score = line.substring(loc + beginIndex, endIndex -1);
double score_d = 0D;
// skip if no score is found
if (score == null) continue;
// skip if score doesn't contain any digit
try {
score_d = Double.valueOf(score).doubleValue();
}
catch (NumberFormatException ex)
{
continue;
}
// skip if score is not a negative number
if (score_d > 0) continue;
// record the iteration where this occurs
String iteration_pattern = "iteration";
int iteration_begin = l_line.indexOf(iteration_pattern);
// there is no iteration for base method
if (iteration_begin < 0) {
iteration = null;
}
else
{
int iteration_end = l_line.indexOf("(", iteration_begin);
iteration = line.substring(iteration_begin, iteration_end-1);
}
if (bestML == null)
{
bestML = new Double(score_d);
}
if (Math.abs(score_d) < Math.abs(bestML.doubleValue())) {
bestML = new Double(score_d);
bestIteration = iteration;
}
bestScore = (bestML == null) ? (""):(bestML.toString());
bestIteration = (bestIteration == null) ? (""):(bestIteration.toString());
} // end if
} // end while
}
catch (Exception ex)
{
LOGGER.debug("Exception caught in reading result file " + ex.toString());
}
finally
{
LOGGER.debug("closing file....");
if (in != null) try {in.close();} catch (Exception ex) {}
}
// record best score and iteration
if (bestScore.equals("")) bestIteration = "";
addAttribute(element, "bestScore", bestScore);
addAttribute(element, "bestIteration", bestIteration);
runNextAnalysis(doc);
}
private static String getCurrentTime(String df)
{
SimpleDateFormat sdf = new SimpleDateFormat(df);
Calendar calendar = Calendar.getInstance(TimeZone.getDefault());
sdf.setTimeZone(TimeZone.getDefault());
String time = sdf.format(calendar.getTime());
return time;
}
private static String getElapsedTime(String df, String start, String stop) throws Exception
{
LOGGER.debug("Start and Stop times = " + start + ";" + stop);
SimpleDateFormat sdf = new SimpleDateFormat(df);
String elapsed = new String();
long mstart = sdf.parse(start).getTime();
long mstop = sdf.parse(stop).getTime();
long timeInSeconds_l = (mstop - mstart) / 1000;
int timeInSeconds = (int)timeInSeconds_l;
int hours = timeInSeconds / 3600;
timeInSeconds = timeInSeconds - (hours *3600);
int minutes = timeInSeconds / 60;
timeInSeconds = timeInSeconds - (minutes *60);
int seconds = timeInSeconds;
elapsed = Integer.toString(hours) +"h:";
elapsed += Integer.toString(minutes) + "m:";
elapsed += Integer.toString(seconds) + "s";
LOGGER.debug("Elapsed time is " + elapsed);
return elapsed;
}
/**
*
*
* @param request
* @return
*/
private synchronized static int getWebCount()
{
String key = "WebCount";
String count = null;
FileInputStream fi = null;
FileOutputStream fo = null;
Properties properties = new Properties();
try
{
// make sure file exits
File file = new File(webcountDir);
LOGGER.debug("webcountDir" + webcountDir);
if (!file.exists())
{
file.createNewFile();
}
else
{
fi = new FileInputStream(webcountDir);
properties.loadFromXML(fi);
count = properties.getProperty(key);
}
count = (count == null) ? ("1") : (count);
// update the count in the properties file now
fo = new FileOutputStream(webcountDir);
int newCount = Integer.valueOf(count).intValue();
// increment the value
properties.setProperty(key, Integer.toString(++newCount));
properties.storeToXML(fo, null);
}
catch (Exception ex)
{
LOGGER.debug("Error in loading the web info properties file" + ex.toString());
}
finally
{
if (fi != null) try {fi.close();} catch (Exception ex){}
if (fo != null) try {fo.close();} catch (Exception ex){}
}
// return the count
return Integer.parseInt(count);
}
}