package org.ngbw.pise.commandrenderer;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import org.ngbw.sdk.ValidationException;
import org.ngbw.sdk.api.tool.CommandRenderer;
import org.ngbw.sdk.api.tool.FieldError;
import org.ngbw.sdk.common.util.StringUtils;
import org.ngbw.sdk.common.util.SuperString;
import org.ngbw.sdk.tool.RenderedCommand;


/**
 * This Class implements CommandRenderer this implementation takes care of PiseXML and Perl 
 * contents.
 *
 * @author Rami
 * @author R. Hannes Niedner 
 */
public class PiseCommandRenderer implements CommandRenderer
{
    private static Logger logger = Logger.getLogger(PiseCommandRenderer.class);
    
    private static final String SCHEDULER_CONF = "scheduler.conf";
    
    private static final Pattern shellEscape = Pattern.compile("(\\||&|;|<|>|\\(|\\)|\\$|`|\\\\|\"|'|\\*|\\?|\\[|\\]|#|~|=|%)");

    private final Map<URL, PiseMarshaller> cfgMap = new HashMap<URL, PiseMarshaller>();

    //parameter name -> parameter value
    private Map<String, List<String>> parameters;

    // private variable for the toUnixCmd() method
    private Map<Integer, List<String>> unixCmdGroups = null;
    
    private Map<String, Map<Integer, List<String>>> paramFiles = null;

    private PiseMarshaller piseMarshaller;
    
    private RenderedCommand renderedCommand = null;
    
    private PerlEval perlEval = null;

    List<FieldError> parameterErrors = new ArrayList<FieldError>();


    public PiseCommandRenderer ()
    {
        super();
        init();
    }


    private void init ()
    {
        parameters = null;
        piseMarshaller = null;
        unixCmdGroups = new TreeMap<Integer, List<String>>();
        paramFiles = new TreeMap<String, Map<Integer, List<String>>>();
        renderedCommand = new RenderedCommand();
    }


    @Override
    public RenderedCommand validate ( URL url, Map<String, List<String>> parameters )
    {
        logger.debug("\n********** PiseCommmandRenderer VALIDATE submission ***********");
        return render(url, parameters, true);
    }


    @Override
    public RenderedCommand render ( URL url, Map<String, List<String>> parameters )
    {
        logger.debug("\n********** PiseCommmandRenderer RENDER submission ***********");
        return render(url, parameters, false);
    }
    
    
    /** 
     * This method evaluates Perls precond and ctrls elements for the parameters. Throws 
     * JobValidationException if a ctrl is violated.  If precond for a parameter is not met, the 
     * parameter is skipped, in the sense that if it has a format element, the code in
     * the format element will not be generated. However, the value of the parameter may be 
     * referenced in another parameter's code, so all parameters that have preconds and are 
     * referenced elsewhere MUST supply a default value with vdef element. If a precond isn't met
     * this method replaces the value of the parameter in the parameter map with the param's default
     * value.
     * 
     * <p>
     * For InFile and Sequence types, the value of the parameter is the source document ID, as a
     * string. 
     * 
     * <p>
     * Caller must verify that entries in parameters map meet certain criteria BEFORE
     * calling this method:
     * 
     * <p>
     * - keys must correspond to parameters in the pise xml file
     * - values of the correct type as specified by the pise file (for example, integer, string,
     * double)
     * - values are in min/max range specified by the pise xml
     * 
     * @param url           URL of PISE XML file 
     * @param parameters    a map of parameter name to value, where parameter is one of the visible
     *                      parameters in the pise xml (including InFile and Sequence parameters, 
     *                      even the hidden "isinput" primary input). These are just the parameters 
     *                      that the user has supplied a value for, and that aren't disabled. It 
     *                      also includes ones for which the gui (or the GuiSimulator for the REST 
     *                      API) supplied a default value 
     * @param validateOnly  
     * @return 
     */
    private RenderedCommand 
    render ( URL url, 
             Map<String, List<String>> parameters, 
             boolean validateOnly )
    {
        try
        {
            logger.debug("##########################################################################################");
            logger.debug("Render command from " + parameters.size() + " visible parameters using config:" + url
                    + ", validateOnly = " + validateOnly);

            /*
             * Keep a pointer to the passed in parameters map. This method will modify the map -
             * upon return, caller will see a modified map. For example, this method adds dummy
             * parameters that correspond to the parameter files that are generated by the pisexml.
             */
            this.parameters = parameters;
            piseMarshaller = initPiseMarshaller(url);

            perlEval = new PerlEval();
            perlEval.initialize();
            List<String> commandList = toUnixCmd(validateOnly);
            perlEval.terminate();

            //log.debug("toUnixCmd returns a list with " + commandList.size() + " elements");
            String[] commandArray = new String[commandList.size()];
            renderedCommand.setCommand(commandList.toArray(commandArray));
            
            logger.debug("Returned command: " + StringUtils.join(renderedCommand.getCommand(), " "));
            
            Map<String, List<String>> input = renderedCommand.getInputFileMap();
            
            Map<String, String> output = renderedCommand.getOutputFileMap();
            
            for (String parameter : input.keySet())
            {
                logger.debug("Input: " + parameter + ": " + input.get(parameter));
            }
            
            for (String parameter : output.keySet())
            {
                logger.debug("Output: " + parameter + ": " + output.get(parameter));
            }
            
            setSchedulerProperties();
            
            return renderedCommand;
        }
        catch ( ValidationException jve )
        {
            throw jve;
        }
        catch ( Exception err )
        {
            throw new RuntimeException(err);
        }
        finally
        {
            if (perlEval != null)
            {
                perlEval.cleanup();
            }
        }
    }


    /**
     * If the xml specified a param file with a specific name (given by SCHEDULER_CONF)
     * we expect that file to contain properties for scheduling the job. We load the contents
     * of the file into the renderedCommand.schedulerProperties.
     */
    private void setSchedulerProperties ()
    {
        Map<String, List<String>> inputFileNames = renderedCommand.getInputFileMap();

        for (Map.Entry<String, List<String>> entry : inputFileNames.entrySet())
        {
            String parameter = entry.getKey();

            for (String fileName : entry.getValue())
            {
                if (fileName.equals(SCHEDULER_CONF))
                {
                    logger.debug("Found " + SCHEDULER_CONF + " in renderedCommand.inputFileMap, for parameter: " + parameter);

                    List<String> valueList = parameters.get(parameter);

                    if (valueList == null)
                    {
                        logger.debug("Parameter " + parameter + " value is null.");
                        return;
                    }

                    String value = valueList.get(0);

                    logger.debug("Parameter " + parameter + " value is:\n" + value);

                    try
                    {
                        renderedCommand.getSchedulerProperties().load(new ByteArrayInputStream(value.getBytes()));
                    }
                    catch ( Throwable t )
                    {
                        logger.warn("Error loading scheduler properties", t);
                    }

                    return;
                }
            }
        }
    }


    //initialize the JAXB Marshaller
    private PiseMarshaller initPiseMarshaller ( URL url )
    {
        if (url == null)
        {
            throw new NullPointerException("Tool config file URL is null!");
        }
        if (cfgMap.containsKey(url) == false)
        {
            try
            {
                PiseMarshaller pm = new PiseMarshaller(url.openStream());
                cfgMap.put(url, pm);
            }
            catch ( IOException e )
            {
                throw new RuntimeException("Cannot initialize PiseMarshaller.", e);
            }
        }
        return cfgMap.get(url);
    }


    /*
     * all parameter names in the submitted maps have an underscore appended. This returns the names
     * without the trailing underscore.
     */
    private Set<String> getParameterSet ()
    {
        Set<String> keySet = new HashSet<String>();
        for (String key : parameters.keySet())
        {
            String paramName = key.replaceFirst("_$", "");
            keySet.add(paramName);
        }
        return keySet;
    }


    private String getParameterKey ( String parameter )
    {
        return parameter + "_";
    }


    private List<String> getParameterValue ( String parameter )
    {
        String key = getParameterKey(parameter);
        return parameters.get(key);
    }


    private void setParameterValue ( String parameter, String value )
    {
        String key = getParameterKey(parameter);
        List<String> newList = new ArrayList<String>();

        newList.add(value);

        parameters.put(key, newList);
    }


    private void setParameterValue ( String parameter, String value, int index )
    {
        String key = getParameterKey(parameter);
        List<String> valueList = parameters.get(key);

        valueList.set(index, value);
    }


    private void removeParameter ( String parameter )
    {
        String key = getParameterKey(parameter);
        parameters.remove(key);
    }


    private void addInputFileName ( String parameter, String fileName )
    {
        String key = getParameterKey(parameter);
        Map<String, List<String>> inputs = renderedCommand.getInputFileMap();
        List<String> names = inputs.get(key);

        if (names == null)
        {
            names = new ArrayList<String>();

            inputs.put(key, names);
        }

        names.add(fileName);
    }


    private void setOutputFileName ( String parameter, String fileName )
    {
        renderedCommand.getOutputFileMap().put(parameter, fileName);
    }


    private String evaluatePerlStatement ( String perlStatement ) throws Exception
    {
        return perlEval.evaluateStatement(perlStatement);
    }


    /*
     * If no precond or precond is true, and ismandatory=true, then make sure we've got a value for
     * it.
     * If not, add an error to parameterErrors.
     */
    public void validateRequiredParameters () throws Exception
    {
        for (String param : piseMarshaller.getRequiredSet())
        {
            if (processPrecond(param) && (getParameterValue(param) == null))
            {
                parameterErrors.add(new FieldError(param, param + " is required."));
                logger.debug("\tERR " + param + " is required.");
            }
        }
    }


    private List<String> toUnixCmd ( boolean validateOnly ) throws IOException, InterruptedException, ExecutionException, Exception
    {

        List<String> commandList = new ArrayList<String>();

        logger.debug("##########################################################################################");
        logger.debug("Validating required parameters");
        validateRequiredParameters();

        logger.debug("##########################################################################################");
        logger.debug("Evaluate visible parameters");
        for (String paramName : getParameterSet())
        {
            processParameter(paramName, false);
        }

        //If we've found errors, bail out.  Errors that occur after this are NOT expected and are most likely a pisexml error.
        if (parameterErrors.size() > 0)
        {
            throw new ValidationException(parameterErrors);
        }

        // These are the parameters hidden from the GUI, but necessary for generating the command line correctly
        logger.debug("##########################################################################################");
        logger.debug("toUnixCmd: processing " + piseMarshaller.getHiddenSet().size() + " HIDDEN parameters");
        logger.debug("Evaluate hidden parameters");
        for (String paramName : piseMarshaller.getHiddenSet())
        {
            processParameter(paramName, true);
        }

        /*
         * These are the parameters that generate an outfile, they are hidden from the GUI since the
         * GUI dosen't
         * give the user the possibility to specify the names of the output files.
         */
        logger.debug("##########################################################################################");
        logger.debug("toUnixCmd: processing " + piseMarshaller.getOutfileSet().size() + " OUTFILESET parameters");
        for (String paramName : piseMarshaller.getOutfileSet())
        {
            processParameter(paramName, true);
        }

        // Not sure of the difference bewteen Pise ResultFile and piseOutfile, but here we handle ResultFiles.
        logger.debug("toUnixCmd: processing " + piseMarshaller.getResultSet().size() + " RESULTSET parameters");
        for (String paramName : piseMarshaller.getResultSet())
        {
            processParameter(paramName, true);
        }

        /*
         * All kinds of parameters, visible, hidden, etc can specify text to be placed in a
         * parameter
         * file instead of on the commandline. The calls to processParameter above put entries in
         * paramFiles each time they find a parameter that puts text in a parameter file. The map is
         * keyed by
         * parameter file name. Here we append all the lines that go into each parameter file into a
         * single
         * string and we add the param file name/ param file contents to our list of input files.
         */
        for (Map.Entry<String, Map<Integer, List<String>>> entry : paramFiles.entrySet())
        {
            // Append the lines that go in the parameter file.
            String key = entry.getKey();
            StringBuilder builder = new StringBuilder();

            for (List<String> lines : entry.getValue().values())
            {
                for (String line : lines)
                {
                    builder.append(line);
                }
            }

            String value = builder.toString();

            /*
             * This adds an entry in the parameter map, with key = parameter file name, value =
             * contents.
             * See NEWLINE comment at end of file.
             */
            setParameterValue(key, value.replace("\\n", "\n"));

            /*
             * This adds an entry in the input file map with key = parameter file name, value =
             * parameter file name.
             */
            addInputFileName(key, key);
        }

        // 3- Creating the command line by ordering them with respect to their group
        for (Map.Entry<Integer, List<String>> entry : unixCmdGroups.entrySet())
        {
            StringBuilder command = new StringBuilder();
            Iterator<String> iter = entry.getValue().iterator();

            command.append(iter.next());

            while (iter.hasNext())
            {
                command.append(" ");
                command.append(iter.next());
            }

            commandList.add(command.toString());
            logger.debug("for unixCmdGroups[" + entry.getKey() + "] commandList.add: " + command.toString());
        }

        return commandList;
    }


    private boolean processPrecond ( String paramName ) throws Exception
    {
        String precond = piseMarshaller.getPrecond(paramName);
        String vdef = piseMarshaller.getVdef(paramName);
        if (precond != null)
        {
            logger.debug("\tEVALUATE Precondition for " + paramName);
            precond = preparePerlExpression(precond, paramName, vdef);
            String perlPrecond = evaluatePerlStatement(precond);
            if (!Boolean.valueOf(perlPrecond))
            {
                logger.debug("\tprecond is false");
                return false;
            }
        }
        logger.debug("\tprecond is true");
        return true;
    }


    private void processControls ( String paramName ) throws Exception
    {
        List<PiseMarshaller.Control> controls = piseMarshaller.getCtrl(paramName);
        String vdef = piseMarshaller.getVdef(paramName);

        logger.debug("EVALUATE Controls for InFile " + paramName);

        List<String> values = getParameterValue(paramName);

        for (String paramValue : values)
        {
            evaluateControls(controls, paramName, paramValue, vdef);
        }
    }


    private void processParameter ( String paramName, boolean hidden )
            throws IOException, InterruptedException, ExecutionException, Exception
    {
        logger.debug("START: processParameter: " + paramName + " (hidden: " + hidden + ")");

        @SuppressWarnings("unused")
        String isCommand = piseMarshaller.getIsCommand(paramName);
        @SuppressWarnings("unused")
        String ishidden = piseMarshaller.getIsHidden(paramName);

        String type = piseMarshaller.getType(paramName);
        String vdef = piseMarshaller.getVdef(paramName);
        String separator = piseMarshaller.getSeparator(paramName);
        String paramfile = piseMarshaller.getParamFile(paramName);
        String filenames = piseMarshaller.getFileNames(paramName);
        List<PiseMarshaller.Control> controls = piseMarshaller.getCtrl(paramName);
        List<String> values = getParameterValue(paramName);

        if (type == null)
        {
            logger.debug("\t" + paramName + " not found in Pise XML\n");

            return;
        }

        if (values == null)
        {
            values = new ArrayList<String>();

            values.add(null);
        }
        else if (hidden)
        {
            List<String> newList = new ArrayList<String>();

            for (int index = 0; index < values.size(); index += 1)
            {
                newList.add(null);
            }

            values = newList;
        }

        for (int index = 0; index < values.size(); index += 1)
        {
            String parameterValue = values.get(index);

            if (parameterValue == null || (parameterValue.length() < 100))
            {
                logger.debug("\t" + paramName + "=" + parameterValue + ", type=" + type);
            }
            else
            {
                logger.debug("\t" + paramName + "=" + parameterValue.substring(0, Math.min(100, parameterValue.length()))
                        + "...(truncated), type=" + type);
            }

            // With flist, user chooses a label that pise maps to a string of code.
            String flist = piseMarshaller.getflistValue(paramName, parameterValue);
            String perlFormat = null;
            String perlPrecond = null;

            String group = piseMarshaller.getGroup(paramName);
            if (type.equals("Results") && group == null)
            {
                group = "-99"; // this group is not going to be used at all
            }
            logger.debug("Group: " + group);
            int groupValue = 1;
            if (group != null)
            {
                groupValue = Integer.parseInt(group);
            }
            /*
             * here we assume that flist and format are mutually exclusive however the case where we
             * have a format and a flist,
             * the flist will override format I don't see the sense of having both in the same
             * parameter
             */
            String format = piseMarshaller.getFormat(paramName);
            if (flist != null)
            {
                format = flist;
            }
            /*
             * The front end code is responsible for inserting the separator character, if any,
             * and concatenating the list elements. However, if the pise doesn't specify a
             * separator,
             * the front end uses "@" as a separator because it must maintain the distinct elements
             * in order
             * to repopulate a form from them. THIS MEANS YOU CAN'T USE AN '@' in a LIST ELEMENT.
             */
            if (type != null && type.equals("List"))
            {
                if (separator == null)
                {
                    // Remove the dummy '@' that the front end inserted and concatenate the elements since there
                    // is not separator specified in the pise xml.
                    parameterValue = SuperString.valueOf(parameterValue, '@').concatenate();

                    setParameterValue(paramName, parameterValue);
                }
                else
                {
                    ;
                }
            }
            else if (type.equals("InFile") || type.equals("Sequence"))
            {
                // filenames not null for infile and sequence means we have assigned standard names for these input files
                if (filenames != null)
                {
                    int offset;
                    String label;

                    if (index > 0)
                    {
                        String basename;
                        String extension;

                        offset = filenames.lastIndexOf(".");

                        if (offset >= 0)
                        {
                            basename = filenames.substring(0, offset);
                            extension = filenames.substring(offset);
                        }
                        else
                        {
                            basename = filenames;
                            extension = "";
                        }

                        label = basename + "_" + index + extension;
                    }
                    else
                    {
                        label = filenames;
                    }

                    offset = parameterValue.indexOf(" ");
                    offset = parameterValue.indexOf(" ", offset + 1);
                    parameterValue = parameterValue.substring(0, offset + 1) + label;

                    setParameterValue(paramName, parameterValue, index);
                }
            }
            String precond = piseMarshaller.getPrecond(paramName);
            if (precond != null)
            {
                logger.debug("Evaluate Precondition for Hidden Parameter");
                precond = preparePerlExpression(precond, parameterValue, vdef);
                perlPrecond = evaluatePerlStatement(precond);
                logger.debug("Precond evaluation: " + perlPrecond + ", boolean value is " + Boolean.valueOf(perlPrecond));
                if (!Boolean.valueOf(perlPrecond))
                {
                    logger.debug("\tPrecondition not satisfied, not generating code for " + paramName);
                    return;
                }
            }
            if (evaluateControls(controls, paramName, parameterValue, vdef) == false)
            {
                // If controls are violated quit processing this parameter
                logger.debug("Return early because evaluateControls returned false");
                return;
            }

            logger.debug("format: " + (format == null ? "null" : format));
            if (format != null)
            {
                format = restitutionFormat(format, parameterValue, type, vdef);
                perlFormat = evaluatePerlStatement(format);
                logger.debug("Format evaluation: " + perlFormat);
            }
            if (perlFormat != null)
            {
                logger.debug("paramfile: " + (paramfile == null ? "null" : paramfile));
                if (paramfile != null)
                {
                    Map<Integer, List<String>> files = paramFiles.get(paramfile);

                    if (files == null)
                    {
                        files = new TreeMap<Integer, List<String>>();

                        paramFiles.put(paramfile, files);
                    }

                    List<String> lines = files.get(groupValue);

                    if (lines == null)
                    {
                        lines = new ArrayList<String>();

                        files.put(groupValue, lines);
                    }

                    lines.add(perlFormat);

                    if (logger.isDebugEnabled())
                    {
                        StringBuilder message = new StringBuilder();

                        for (String line : lines)
                        {
                            message.append(line);
                        }

                        logger.debug("parameter: " + paramName + " put in inputDataMap: '" + paramfile + "' -> '" + message.toString() + "'");
                    }
                }
                else
                {
                    List<String> commands = unixCmdGroups.get(groupValue);

                    if (commands == null)
                    {
                        commands = new ArrayList<String>();

                        unixCmdGroups.put(groupValue, commands);
                    }

                    commands.add(perlFormat);

                    if (logger.isDebugEnabled())
                    {
                        StringBuilder message = new StringBuilder();
                        Iterator<String> iter = commands.iterator();

                        message.append(iter.next());

                        while (iter.hasNext())
                        {
                            message.append(" ");
                            message.append(iter.next());
                        }

                        logger.debug("unixCmdGroups[" + groupValue + "] = " + message.toString());
                    }
                }
            }

            if (type.equals("InFile") || type.equals("Sequence"))
            {
                String fileName = getInputFileLabel(parameterValue);

                addInputFileName(paramName, fileName);
                logger.debug("addInputFileName: '" + paramName + "' -> '" + fileName + "'");
            }
            else if (filenames != null)
            {
                // Most output filenames are given in Results but can also be defined in Switch, List, OutFile, etc.
                setOutputFileName(paramName, filenames);
                logger.debug("setOutputFileName: '" + paramName + "' -> '" + filenames + "'");
            }
        }

        logger.debug("END : processParameter: " + paramName + " (hidden: " + hidden + ")\n");
    }


    /*
     * - replaces "defined $value", and "defined $param_name" in "str" argument.
     * - "parameterValue" argument is the value of the current parameter, i.e the one that has the
     * code we're evaluating.
     *
     * This is frequently used to check whether a data item has been supplied for
     * a parameter of type InFile or Sequence. If the user hasn't supplied a data item,
     * there is no default, so the parameter will *not* be in our map.
     *
     * This version is different than replaceDefined() in GuiSimulator. In GuiSimulator
     * we put all the parameters from the pise file into our map with a value of empty
     * string (if there is no vdef and user hasn't entered anything) so we don't test
     * for null there, we test for "". However, once GuiSimulator is finished we
     * remove all the empty string parameters and only pass along those that have
     * non-empty values, so here we need to test for the parameter not being in
     * our map at all instead of testing whether it is the empty string.
     */
    private String replaceDefined ( String str, String parameterValue )
    {
        StringBuffer buf = new StringBuffer();

        // Find the text "defined", followed by whitespace, followed by "$", followed by one or more
        // "word" characters (i.e letter, number or underscore).  Capture the "word" into group 1.
        Pattern p = Pattern.compile("defined\\s\\$(\\w+)");

        Matcher matcher = p.matcher(str);
        while (matcher.find())
        {
            String var = matcher.group(1);
            if (var.equals("value"))
            {
                if (parameterValue == null)
                {
                    matcher.appendReplacement(buf, "0");
                }
                else
                {
                    matcher.appendReplacement(buf, "1");
                    //matcher.appendReplacement(buf, (parameterValue == "") ? "0" : "1");
                }
            }
            else
            {
                List<String> value = getParameterValue(var);
                matcher.appendReplacement(buf, (value != null) ? "1" : "0");
            }
        }
        matcher.appendTail(buf);
        return buf.toString();
    }


    /*
     * Prepare a perl precond, control or warn expression, to be executed by our perlEval class.
     *
     * Replaces $var, $value, defined $var, etc with values from the parameter map, then
     * builds a perl expression of the form: (expr) ? print "true" : print "false
     * and returns the expression.
     *
     * Calling code will pass this expression to "perl -e". The stdout from running perl -e on the
     * expression will produce the text "true" or "false" and we pass the stdout to java's
     * Boolean.valueOf() to get
     * a boolean evaluation of the precond.
     *
     */
    private String preparePerlExpression ( String precond, String paramValue, String vdef )
    {
        logger.debug("\tOriginal Expression:      " + precond);

        precond = replaceDefined(precond, paramValue);

        //log.debug("Expression after replaceDefined:      " + precond);
        // look for $value, $vdef, $<var>
        Pattern p = Pattern.compile("\\$\\w*");
        Matcher m = p.matcher("" + precond);
        StringBuffer sb = new StringBuffer();

        while (m.find())
        {
            if (m.group().contains("$value"))
            {
                if (paramValue != null)
                {
                    m.appendReplacement(sb, "\"" + paramValue + "\"");
                }
                else
                {
                    // If paramValue is null, just leaves "$value" in the perl expression.
                    logger.warn("\tpreparePerlExpression '$value  NOTHING TO REPLACE IT WITH");
                }
            }
            else if (m.group().contains("$vdef"))
            {
                if (vdef == null)
                {
                    logger.warn("\tpreparePerlExpression '$value' NOTHING TO REPLACE IT WITH");
                }
                else
                {
                    m.appendReplacement(sb, "\"" + vdef + "\"");
                }
            }
            else if (m.group().equalsIgnoreCase("$") == false)
            {
                String myKey = m.group().substring(1);
                List<String> theValue = getParameterValue(myKey);
                if (theValue == null)
                {
                    // leave the undefined variable reference in the expression and let perl handle it.
                    logger.warn("\tINVALID PRECONDITION - uses value of undefined parameter " + myKey + ".  Use 'defined'");
                }
                else
                {
                    m.appendReplacement(sb, "\"" + theValue.get(0) + "\"");
                }
            }
        }
        m.appendTail(sb);
        precond = sb.toString();

        precond = precond.replaceAll("false", "0");
        precond = precond.replaceAll("true", "1");

        // precond returns a True or False depends on the condition being verified
        precond = "(" + precond + ")? print \"true\" : print \"false\";";

        logger.debug("\tFinal Perl Expression: " + precond);
        return precond;
    }


    private String restitutionFormat ( String format, String paramValue, String type, String vdef )
    {
        logger.debug("Original Format:      " + format);

        format = replaceDefined(format, paramValue);

        logger.debug("Format after replaceDefined:      " + format);

        Pattern p = Pattern.compile("\\$\\w*");
        Matcher m = p.matcher(format);

        while (m.find())
        {
            if (m.group().contains("$value"))
            {
                if (paramValue == null)
                {
                    logger.warn("'$value' is NULL");
                }
                else
                {
                    if (type.equals("InFile") || type.equals("Sequence"))
                    {
                        String label = getInputFileLabel(paramValue);

                        if (label != null)
                        {
                            paramValue = label;
                        }
                    }
                    else if (type.equals("String"))
                    {
                        paramValue = shellEscape.matcher(paramValue).replaceAll("\\\\\\\\$1");
                    }

                    format = format.replace("$value", paramValue);
                }
            }
            else if (m.group().contains("$vdef"))
            {
                if (vdef == null)
                {
                    logger.warn("'$vdef' is NULL");
                }
                else
                {
                    format = format.replace("$vdef", vdef);
                }
            }
            else
            {
                String myKey = m.group().substring(1);
                String myValue = null;

                if (getParameterSet().contains(myKey) == false)
                {
                    /*
                     * TL: modified for raxmlhpc.xml "model" parameter. Need to get default value of
                     * parameters that
                     * haven't been set in the gui but are referenced in this parameter's code
                     * element.
                     *
                     * 2/23/2015 TL: I'm not sure this is still true. Parameters like 'dna_grtcat'
                     * do reference others
                     * like 'invariable' which is not mandatory. However, if it has a vdef and it's
                     * precond was met,
                     * it should be in our map. If that isn't guaranteed, based on the precond or
                     * lack of a vdef,
                     * I think we can rewrite the format statements to be conditional on, say
                     * 'defined $invariable'.
                     */
                    logger.warn("INVALID FORMAT statement references undefined parameter " + myKey + ", will use vdef or empty string.");
                    myValue = piseMarshaller.getVdef(myKey);
                    if (myValue == null) // no vDef element in the pise xml
                    {
                        logger.warn("INVALID FORMAT statement references undefined parameter " + myKey + ", will use empty string.");
                        myValue = "";
                    }
                    else
                    {
                        logger.warn("INVALID FORMAT statement references undefined parameter " + myKey + ", will use vdef '" + myValue + "'");
                    }
                }
                else
                {
                    List<String> values = getParameterValue(myKey);

                    if (values != null)
                    {
                        myValue = values.get(0);
                    }
                    else
                    {
                        myValue = "";
                    }
                }
                format = format.replace(m.group(), myValue);
            }
        }
        /*
         * A format can be a simple "..." or a condition (...)? "...":"..."
         * If it's ? expression we stick a "print" in front of both components.
         * But if it starts with "<<" we're going to assume the whole thing is a here document
         * and not process the ? operator. This isn't exactly what a real perl interpreter
         * would do but I think it's close enough for the pise and ngbw xml files we deal with these
         * days.
         */
        if (!format.trim().startsWith("<<") && format.contains("?"))
        {
            format = format.substring(0, format.indexOf("?") + 1) + " print " + format.substring(format.indexOf("?") + 1);
            format = format.substring(0, format.indexOf(":") + 1) + " print " + format.substring(format.indexOf(":") + 1);
        }
        else
        {
            format = "print " + format;
        }
        format = format.replaceAll("false", "0");
        format = format.replaceAll("true", "1");

        logger.debug("Parsed Format: " + format);

        return format;
    }


    private String getInputFileLabel ( String identifier )
    {
        if (identifier.startsWith("TaskInputSourceDocument "))
        {
            int offset = identifier.indexOf(" ");

            offset = identifier.indexOf(" ", offset + 1);

            return identifier.substring(offset + 1);
        }

        return null;
    }


    /*
     * Controls are written to express an error condition when true.
     * For example to require runtime to be <= 168, you write
     * "$runtime > 168.0"
     *
     * Returns true (ie. all's well) if there are no controls or all controls evaluate to false.
     * For each control that is true, sets an error message in ??? TODO
     */
    private boolean evaluateControls ( List<PiseMarshaller.Control> controls, String paramName, String paramValue, String vdef )
            throws Exception
    {
        int errorCount = 0;
        String perl;
        String evaluatedPerl;
        if (controls == null)
        {
            return true;
        }
        for (PiseMarshaller.Control c : controls)
        {
            perl = preparePerlExpression(c.perl, paramValue, vdef);
            evaluatedPerl = evaluatePerlStatement(perl);
            logger.debug("ctrl: '" + perl + "' EVAL TO '" + evaluatedPerl + "'");
            if (Boolean.valueOf(evaluatedPerl) == true)
            {
                parameterErrors.add(new FieldError(paramName, c.message));
                errorCount += 1;
            }
        }
        return errorCount == 0;
    }


}

/*
 * NEWLINE replacement: (search for "NEWLINE" to see what this comment is referring to).
 *
 * This code replaces the two character sequence of <backslash, n> with a single newline character.
 * Since evalutePerl() returns the stdtout trimmed of leading and trailing whitespace, if we
 * want to have a parameter be followed by a newline, we need the result of the perl evaluation
 * to end not with a newline character but with a backslash followed by an n, which we'll replace
 * with a newline right here.
 *
 * note that when you print a java string, as the logging messages in this file do, if
 * the string contains an actual newline, it will print on multiple lines. If you see
 * "\n" in the value displayed, it means the string contains two characters: a backslash
 * followed by an n. This is mostly what we see because if you put "\n" in an xml file
 * element, the xml unmarshaller delivers this as a java string containing a backslash
 * followed by an n.
 *
 * Both perl (if using double quoted strings) and java interpret "\\n" as the two character
 * sequence <backslash, newline>. The first quote escapes the second, yielding a literal
 * backslash and the n is unquoted. In a perl single quoted string, '\\n' is three characters:
 * two backslashes followed by an n. If you have a format like '$value\\n', with single quotes,
 * perl leaves the \\n alone and when we get here we replace the \n with a newline, so we're
 * left with a single backslash followed by a newline ... probably not what you want.
 * On the other hand, "$value\\n" works because when ask perl to print this it gives us
 * the value followed by a backslash follwed by an n.
 */