package org.cipres.test;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Observable;
import java.util.Observer;
import java.util.Properties;
import java.util.TimeZone;

import org.apache.log4j.Logger;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;


/**
 * You can also run this outside of Eclipse using the following script:
 * 
 * 	java -cp \
 *	/users/u3/lcchan/.m2/repository/org/cipres/webapp/cipres-test/1.0-SNAPSHOT/cipres-test-1.0-SNAPSHOT.jar:\
 *  /users/u3/lcchan/.m2/repository/jdom/jdom/1.0/jdom-1.0.jar \
 *  org.cipres.webapp.helpers.RunPaupTestAction
 * 
 * RunPaupTestAction.java
 * 
 * Created on Apr 11, 2007
 * @author lcchan
 *
 */
public class RunPaupTestAction implements Observer {

	private static final Logger LOGGER = Logger.getLogger(RunPaupTestAction.class);
	
	public RunPaupTestAction() {}
	
	private static final String inputXml = "input.xml";
	private static final String doneFile = "done";
	private static final String df = "MM/dd/yy HH:mm:ss";

	//	************ replace the following for testing... (not to run the real jobs)
	//private static final String cipres = "/users/u3/lcchan/.m2/repository/org/cipres/helpers/cipres-framework-stub/1.0/cipres-framework-stub-1.0.jar";

	private static final String cipres = "/projects/cipres/lcchan_cipres_test/CIPRES-and-deps/cipres/build/lib/cipres/cipres-classpath.jar";
	private static final String version = "Version1.0";
	//private static final String version = "Test";

	private static final String cipresperf = "/users/u3/lcchan/projects/cipresdev/trunk/cipres-benchmark-tests/" + version;

	private static final String datadirectory  = cipresperf + "/nexusfiles_with_startingtree/";
	private static final String datadirectory_notree = cipresperf + "/nexusfiles_without_startingtree/";
	private static final String commanddirectory = cipresperf + "/commandfiles/" + System.getenv("SHOST") +"/";
	private static String resultsDirectory = cipresperf + "/results/" + System.getenv("SHOST") + "/";
	private static String xmlfile = cipresperf  + "/scripts/" + System.getenv("SHOST") + "/NexusFiles.xml";
	private static final String webcountdirectory = cipresperf + "/scripts/" + "webcount.properties";
	
	private static final String myCipresCp = "/users/u3/lcchan" + "/.m2/repository/org/cipres/webapp/cipres-test/1.0-SNAPSHOT/cipres-test-1.0-SNAPSHOT.jar"; //$HOME
	private static final String jdomCp = "/users/u3/lcchan/.m2/repository/jdom/jdom/1.0/jdom-1.0.jar";
	private static final String cipresCp = cipres + ":" + myCipresCp + ":" + jdomCp;
										
	private static final String observer = "org.cipres.test.RunPaupTestAction";
	private static final String cipresAppName = "org.cipres.portal.RunServiceApp";
	
	private static Document doc = null;

	private static String currentXmlFile = null;
	
	/**
	 * main program to kick off the run
	 * 
	 * @param args
	 * @throws Exception
	 */
    public static void main( String[] args ) throws Exception
    { 
		currentXmlFile = xmlfile + "." + System.getenv("SHOST");
    	
    	// get the xml file and convert to Document type
    	doc = xml2doc(); 
    	
    	// update the xml file to get list of tools to run
    	//getTools(doc);
    	
    	// run the next analysis
    	runNextAnalysis(doc);
    	
    } // end main
   /**
    * get the next analysis to run, and record the starting time in NexusFiles.xml
    * 
    * @param doc
    */
    private static void runNextAnalysis(Document doc)
    {
	String fileRun = null;
	String analysisRun = null;
    
    	List<Element> list = doc.getRootElement().getChildren();
    	
    	for (Element element: list) // list of <file>
    	{
    		fileRun = element.getAttributeValue("name");
    		List<Element> analyses = element.getChildren(); // list of <analysis>
			int counter = 0;
    		for (Element analysis: analyses)
    		{
				counter++;
    			analysisRun = analysis.getAttributeValue("name");
    			String start = analysis.getAttributeValue("start");
    			String stop = analysis.getAttributeValue("stop");
				String treeInfer = analysis.getAttributeValue("treeInfer");
    			// find the next analysis to run
    			if (start == null || stop == null)
    			{	
    				LOGGER.debug("==== Analysis to run " + analysisRun + " for " + fileRun + "=====");
    				// record the start time in xml file and add it to current analysis node
    				start = getCurrentTime(df);
    				addAttribute(analysis, "start", start);
					int exitValue = startAnalysis(doc, fileRun, analysisRun, treeInfer, start);
					LOGGER.debug("Exit Value return = " + exitValue);
					if (exitValue == 0)
					{
						// analysis is done, process data received from cipres
						processData(fileRun, analysisRun, counter);
					}
					else
					{
						LOGGER.debug("Exit Value for startAnalysis returns error " + exitValue);
						System.exit(-1);
					}
    			} // end if
    		} // end for
    	} // end for
    	LOGGER.debug("All done. No more analysis to run");
    	System.exit(0);
    }
    /**
     * update the xml file
     *
     */
    private static void updateXml()
    {
		FileOutputStream fos = null;
    	try
		{
			// update the xml file
    		Format format = Format.getPrettyFormat();
			XMLOutputter output = new XMLOutputter(format);
			fos = new FileOutputStream(currentXmlFile);
			output.output(doc, fos);
		}
		catch (IOException ex)
		{
			ex.printStackTrace();
		}
		finally
		{
			if (fos != null) try {fos.close();} catch (Exception ex){}
		}

    }
    /**
     * return the analysis node in NexusFiles.xml given the file name and analysis name
     * 
     * @param attrName
     * @param attrValue
     */
    private static Element getAnalysisNode(Document doc, String fileName, String analysisName, int counter)
    {
    	String file = null;
    	String analysis = null;
    	boolean found = false;
    	Element foundElmenet = null;
    	
    	// find the appropriate node, and add the attribute
    	List<Element> list = doc.getRootElement().getChildren();
    	
    	for (Element element: list)
    	{
    		if (found) break;
    		file = element.getAttributeValue("name");
    		if (!file.equals(fileName)) continue;
    		
    		List<Element> analyses = element.getChildren("analysis");
    		
			int lcounter = 0;
    		for (Element e: analyses)
    		{
			lcounter++;
    			analysis = e.getAttributeValue("name");
    			if (analysis.equals(analysisName) && (lcounter==counter))
    			{
    				found = true;
    				foundElmenet = e;
    				break;
    			} // end if
    		} // end for
    	} // end for
    	return foundElmenet;
    }
    /**
     * add an attribute a specified element
     * 
     * @param element
     * @param attrName
     * @param attrValue
     */
    private static void addAttribute(Element element,String attrName, String attrValue)
    {
    	element.setAttribute(attrName, attrValue);
    	updateXml();
    }
    /**
     * start running the job here
     * 
     * @param doc
     * @param fileName
     * @param analysisName
     * @param start
     */
    private static int startAnalysis(Document doc, String fileName, String analysisName, String treeInfer, String start)
    {	
    	int exitValue = 0;
    	
    	// make sure results directory is create
    	new File(resultsDirectory).mkdir();
    	
    	// if we are running paup, make sure to use datasets without starting trees
    	String datadir = (analysisName.indexOf("paup") >=0 ) ? (datadirectory_notree) : (datadirectory);
    	// user paup infer file accordingly
		String paupInferFile = (analysisName.indexOf("paup") >=0) ?  "paup_infer.xml" : "quick_paup_infer.xml";
		if (treeInfer != null) paupInferFile = treeInfer;


		// read count and increment it
		int count = getWebCount();

		// get the current date and attach to groupLabel
		Date now = new Date();
		SimpleDateFormat sdf = new SimpleDateFormat("MM-dd-yyyy");
    	
		String[] commandArray = new String[] {
			"java",
			"-cp", cipresCp,
	         "-DoriginalNexusName=" + fileName + ".nex",
	         "-DnexusFile=" + datadir + fileName + ".nex",
	         "-DtoolFile=" + commanddirectory + analysisName + ".xml",
	         "-DtreeInferTool=" + commanddirectory + paupInferFile,
	         "-DtoolName=" + analysisName,
	         "-DresultsDir=" + resultsDirectory,
	         "-DgroupLabel=" + "Job-" + count + "(lcchan@sdsc.edu)",
	         "-Dtimeout=144000", // set it to run 100 days
	         "-Dobserver=" + observer,
	         cipresAppName
		};
		LOGGER.debug("Command to run analysis: " + Arrays.toString(commandArray));
		// save the inputs
		saveInputs(fileName, analysisName, start);
		
		try
	   	{
			Runtime rt = Runtime.getRuntime();
			Process process  = rt.exec(commandArray);
			
			// any error message?
			StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream(), "ERROR");
			// any output?
			StreamGobbler outputGobbler = new StreamGobbler(process.getInputStream(), "OUTPUT");

	    		// start reading
	        outputGobbler.start();
	        errorGobbler.start();
	        exitValue = process.waitFor();
	        LOGGER.debug("Process returns exitValue = " + exitValue);
	   	}
	   	catch(Exception ex)
	   	{
	   		ex.printStackTrace();
	   	}
	   	return exitValue;
   }
    /**
     * save the information so we can use it later to retrieve the starting time for a job
     * 
     * @param fileName
     * @param analysisName
     * @param start
     */
    private static void saveInputs(String fileName, String analysisName, String start)
    {
    	Properties properties = new Properties();
    	properties.setProperty("file", fileName);
    	properties.setProperty("analysis", analysisName);
    	properties.setProperty("start", start);
    	FileOutputStream fos = null;


    	try
    	{
		String filename = resultsDirectory + inputXml + "." + System.getenv("SHOST");
    	fos = new FileOutputStream(filename);
		LOGGER.debug("Storing input.xml in " + filename);
    		properties.storeToXML(fos, null);
    	}
    	catch (IOException ex)
    	{
    		LOGGER.debug("Error in writing data to input.xml: " + ex.toString());
    	}
		finally
		{
			if (fos != null) try {fos.close();} catch (Exception ex) {}
		}
    }
    /**
     * get list of the tools to run for this node
     * 
     * @param nodeName
     * @return
     */
    private static void addAnalysisNode(Element element, String tool)
    {
    	Element e = new Element("analysis");
    	e.setAttribute("name", tool);
    	element.addContent(e);
    	updateXml();
    }
    /**
     * get the list of analysese to run using the nexus file information
     * 
     * @param doc
     */
    private static void getTools(Document doc)
    {
    	String ntaxa_str = null;
    	
    	List<Element> list = doc.getRootElement().getChildren();
    	// for each of <file> element
    	for(Element element: list)
    	{
    		if (element.getChildren("analysis").size() > 0 ) continue;
    		
    		String node = element.getAttributeValue("name");
    		
    		//************** PAUP ***********************
    		// base method
    		addAnalysisNode(element, "paup");
    		// recidcm3 boosted version
    		addAnalysisNode(element, "rid3-paup-12.5-pct");
    		addAnalysisNode(element, "rid3-paup-25-pct");
    		addAnalysisNode(element, "rid3-paup-50-pct");
    		
    		//************** GARLI ************************
    		// base method
    		addAnalysisNode(element, "garli-gtr-g");
    		// recidcm3 boosted versoin
    		addAnalysisNode(element, "rid3-garli-gtr-g-12.5-pct");
    		addAnalysisNode(element, "rid3-garli-gtr-g-25-pct");
    		addAnalysisNode(element, "rid3-garli-gtr-g-50-pct");
    		
    		//*************** RAxML ****************************
    		// base method
    		addAnalysisNode(element, "recidcm3_raxml");
    		// recidcm3 boosted version
    		addAnalysisNode(element, "rid3-rax-12.5-pct");
    		addAnalysisNode(element, "rid3-rax-25-pct");
    		addAnalysisNode(element, "rid3-rax-50-pct");
    	} // end for
    }
    /**
     * convert NexusFilex.xml to a DOM object
     * 
     * @return
     */
    private static Document xml2doc()
    {
		LOGGER.debug("Parsing xml file to Document object: " + currentXmlFile);

    	Document doc = null;
    	 // create a parser
    	SAXBuilder parser = new SAXBuilder();
    	try
    	{
    		// get the DOM document of the xml file
    		LOGGER.debug("Parsing xml: " + currentXmlFile);
    		doc = parser.build(new File(currentXmlFile));
    	}
    	catch (JDOMException ex)
    	{
    		ex.printStackTrace();
    	}
    	catch (IOException ex)
    	{
    		ex.printStackTrace();
    	}
    	return doc;
    } // end xml2doc
    /**
     * copy file from one to another
     * 
     * @param src
     * @param dst
     * @throws IOException
     */
    private static void copy(File src, File dst) throws IOException {
        InputStream in = new FileInputStream(src);
        OutputStream out = new FileOutputStream(dst);
    
        // Transfer bytes from in to out
        byte[] buf = new byte[1024];
        int len;
        while ((len = in.read(buf)) > 0) {
            out.write(buf, 0, len);
        }
        in.close();
        out.close();
    }
    
    public void update(Observable o, Object arg)
    {
    	LOGGER.debug("Calling updating.. nothing to do...");
    	// do nothing
    }
    public static void processData(String fileRun, String analysisRun, int counter)
    {
    	LOGGER.debug("Calling processData() for " + currentXmlFile);
    	
    	//  get the xml file and convert to Document type
    	doc = xml2doc(); 
    	Double bestML = null;
    	String bestScore = "";
    	String iteration = "";
    	String bestIteration = null;
    	
    	Properties properties =  new Properties();
    	FileInputStream fis = null;

    	// load input.xml
    	try 
    	{
			String filename = resultsDirectory + inputXml + "." + System.getenv("SHOST");
    		fis = new FileInputStream(filename);
    		properties.loadFromXML(fis);
    	} 
    	catch (Exception ex)
    	{
    		LOGGER.debug("Error in reading input.xml " + ex.toString());
    	}
		finally
		{
			if (fis != null) try {fis.close();} catch (Exception ex) {}
		}
    	String fileName = properties.getProperty("file");
    	String analysisName = properties.getProperty("analysis");
    	String start = properties.getProperty("start");
    	
    	if (!fileName.equals(fileRun) || !analysisName.equals(analysisRun))
    	{
    		LOGGER.debug("Error - filename and analysis do not match " + fileName+"("+fileRun+");"+analysisName+"("+analysisRun+")");
    		System.exit(-1);
    	}
    	LOGGER.debug("File and analysis names matched.... proceed...");

		//locate the Element node we are dealing with
        Element element = getAnalysisNode(doc, fileName, analysisName, counter);
    	// read done file and add it to xml file if it's not empty
        try {
        	// save a copy of the "done" file and rename it
        	String filename = resultsDirectory + doneFile;
        	String savedFilename = resultsDirectory+analysisName+"_"+fileName+"_"+counter+".done";
        	LOGGER.debug("Copy files from " + filename + " to " + savedFilename);
        	copy(new File(filename), new File(savedFilename));
        	
            BufferedReader in = new BufferedReader(new FileReader(resultsDirectory + doneFile));
            String str;
            while ((str = in.readLine()) != null) {
                addAttribute(element, "message", str);
            }
            in.close();
        } catch (IOException ex) {
        	LOGGER.debug("Error in reading done file " + ex.toString());
        }

        // try to locate ML score, and if found, add them to xml file
        // Email from terri where to locate ML score
        //There's no nexus standard that covers it but if you look at some of the results files, 
        //you can see where I'm writing it.  Each "Tree" line in the file contains the 
        //text "[Score=n]" where n is either a number or the word "UNSCORED". 
        //If you're running a method that gives ML scores, n is where you can find the ML score.
        try
        {
        	String filename = resultsDirectory + analysisName + "_" + fileName + ".nex";
		//========================== debug =======================================================
        	//  filename="/users/u3/lcchan/projects/cipresdev/trunk/cipresweb-test-files/rid3-paup-12.5-pct_dataset3.nex";
		//========================== debug =======================================================
        	LOGGER.debug("Getting result file in: " + filename);

        	// save a copy of the ".nex" file and rename it
        	String savedFilename = resultsDirectory+analysisName+"_"+fileName+"_"+counter+".nex";
			// in case if we are running the same analysis more than once
        	copy(new File(filename), new File(savedFilename));
        	LOGGER.debug("Copy files from " + filename + " to " + savedFilename);

        	BufferedReader in = new BufferedReader(new FileReader(filename));
        	String line;
        	
        	while ( (line = in.readLine()) != null)
        	{
        		//convert the line to lower 
        		String l_line = line.toLowerCase();
        		int loc = 0;
        		if ( (loc = l_line.indexOf("score=")) >= 0)
        		{
        			// record each of the score found as a child node
        			Element mLElement = new Element("ML");
        			// do record everything in the line up to where score is found
        			int endIndex = l_line.indexOf("]") +1;
        			mLElement.setAttribute("score", line.substring(0, endIndex));
        			// add a child node and update element
        			element.addContent(mLElement);
        			updateXml();
        			
        			// try to figure out the best ML score
        			int beginIndex = new String("score=").length();
        			String score = line.substring(loc + beginIndex, endIndex -1);
					double score_d = 0D;

        			// skip if no score is found
        			if (score == null) continue;
					// skip if score doesn't contain any digit
					try {
        				score_d = Double.valueOf(score).doubleValue();
					} 
					catch (NumberFormatException ex)
					{
						continue;
					}
        			// skip if score is not a negative number
        			if (score_d > 0) continue;

        			// record the iteration where this occurs
        			String iteration_pattern = "iteration";
        			int iteration_begin = l_line.indexOf(iteration_pattern);
					// there is no iteration for base method
        			if (iteration_begin < 0) {
						iteration = null;
					}
					else
					{
        				int iteration_end = l_line.indexOf("(", iteration_begin);
        				iteration = line.substring(iteration_begin, iteration_end-1);
					}
        			
        			if (bestML == null)
        			{
        				bestML = new Double(score_d);
        			}
        			if (Math.abs(score_d) < Math.abs(bestML.doubleValue())) {
					bestML = new Double(score_d);
					bestIteration = iteration;
				}
        			bestScore = (bestML == null) ? (""):(bestML.toString());
        			bestIteration = (bestIteration == null) ? (""):(bestIteration.toString());
        		} // end if
        	} // end while
        } // end try
        catch (Exception ex)
        {
        	LOGGER.debug("Exception caught in reading result file " + ex.toString());
        }
                
    	// record the stop time in xml file  
        String stop = getCurrentTime(df);
    	addAttribute(element, "stop", stop);
    
    	// calculate time elapsed
    	try
    	{
    		String elapsedHMS = getElapsedTime(df, start, stop);
    		addAttribute(element, "time", elapsedHMS);
    		//   record best score the iteration
    		if (bestScore.equals("")) bestIteration = "";
    		addAttribute(element, "bestScore", bestScore);
    		addAttribute(element, "iteration", bestIteration);
    	}
    	catch (Exception ex)
    	{
    		LOGGER.debug("Exception in calculating time elapsed: " + ex.toString());
    	}
    	runNextAnalysis(doc);
    }
    
	private static String getCurrentTime(String df)
	{
		SimpleDateFormat sdf = new SimpleDateFormat(df);
		Calendar calendar = Calendar.getInstance(TimeZone.getDefault());
		sdf.setTimeZone(TimeZone.getDefault());
		String time = sdf.format(calendar.getTime());
		return time;
	}
	/**
	 * 
	 * 
	 * @return
	 */
	private static String getElapsedTime(String df, String start, String stop) throws Exception
	{
		LOGGER.debug("Start and Stop times = " + start + ";" + stop);
		
		SimpleDateFormat sdf = new SimpleDateFormat(df);
		String elapsed = new String();
		
		long mstart = sdf.parse(start).getTime();
		long mstop = sdf.parse(stop).getTime();
		long timeInSeconds_l = (mstop - mstart) / 1000;
		
		int timeInSeconds = (int)timeInSeconds_l;

		int hours = timeInSeconds / 3600;
		timeInSeconds = timeInSeconds - (hours *3600);
		int minutes = timeInSeconds / 60;
		timeInSeconds = timeInSeconds - (minutes *60);
		int seconds = timeInSeconds;

		elapsed = Integer.toString(hours) +"h:";
		elapsed += Integer.toString(minutes) + "m:";
		elapsed += Integer.toString(seconds) + "s";
		
		LOGGER.debug("Elapsed time is " + elapsed);
		return elapsed;
	}
	/**
	 * 
	 * 
	 * @param request
	 * @return
	 */
	private synchronized static int getWebCount()
	{
		String key = "WebCount";
		
		String count = null;
		FileInputStream fi = null;
		FileOutputStream fo = null;
		
		Properties properties = new Properties();
		
		try
		{
			// make sure file exits
			File file = new File(webcountdirectory);
			//LOGGER.debug("webcountdirectory" + webcountdirectory); 
			if (!file.exists())
			{
				file.createNewFile();
			}
			else
			{
				fi = new FileInputStream(webcountdirectory);
				properties.loadFromXML(fi);
				count = properties.getProperty(key);
			}
			count = (count == null) ? ("1") : (count);
			
			// update the count in the properties file now
			fo = new FileOutputStream(webcountdirectory);
			int newCount = Integer.valueOf(count).intValue();
			// increment the value
			properties.setProperty(key, Integer.toString(++newCount));
			properties.storeToXML(fo, null);
		}
		catch (Exception ex)
		{
			LOGGER.debug("Error in loading the web info properties file" + ex.toString());
		}
		finally
		{
			if (fi != null) try {fi.close();} catch (Exception ex){}
			if (fo != null) try {fo.close();} catch (Exception ex){}
		}
		// return the count
		return Integer.parseInt(count);
	}
}