#!/usr/bin/python '''Utility classes with implementation details that are common to wrapping NEXUS-based programs''' import os import tempfile from PIPRes.util.server_import import * from PIPRes.util.io import tryRemove, tryRmdir from PIPRes.basic import writeTrees, MatrixFormat, writeDataMatrix, readTreeFromPhylipFile from PIPRes.cipres_types import * _LOG = cipresGetLogger('pipres.wrap.nexus_program_server.py') def mapTreeToLeafSubset(tree): '''This function is used when you have a data structure (e.g. a matrix) for all of the taxa up to ntax, but you only want to write out data pertaining to the taxa that appear on "tree". The function maps the tree to a new tree with no gaps in the leaf set. Returns a tuple of: * tree with mapped leaves, * list of new "local" tax names, * list of indices of a full matrix that should correspond to the tree, * toExternalTranslation mapping function see WrapExternalProgram._createTempDataFile() for an example of how this is used.''' toExternalList = map(lambda x: x - 1, tree.getLeafSet()) toLocalTranslation, toExternalTranslation = createMappingFuncs(toExternalList) mappedTree = CipresTree(tree) mappedTree.mapLeaves(toLocalTranslation) mappedTree.taxaManager = None rows = copy.copy(toExternalList) rows.sort() return mappedTree, [('tax%d' % (1 + toExternalTranslation(i))) for i in xrange(mappedTree.nTaxa)], rows, toExternalTranslation def addScoreAndMapLeaves(treeToReturn, score, criterion, toExternalTranslation): if toExternalTranslation is not None: treeToReturn = CipresTree(treeToReturn) treeToReturn.mapLeaves(toExternalTranslation) addScore(treeToReturn, score, criterion) #_LOG.debug('Returning ' + str(treeToReturn)) return treeToReturn class InvocationStyleEnum: kUseSystem, kUseSpawn, kUseSubprocess = range(3) class WrapExternalProgram: _invocationStyle = InvocationStyleEnum.kUseSpawn def killChildProcesses(subprocList): #assumes that we are on UNIX and using subprocess import signal for s in subprocList: pid = isinstance(s, int) and s or int(s.pid) _LOG.debug('Killing process %d' % pid) if sys.platform.upper().startswith('WIN'): # note the "pid" will actually be a handle on windows # got this code from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/347462 win32api.TerminateProcess(pid, -1) else: os.kill(pid, signal.SIGTERM) killChildProcesses = staticmethod(killChildProcesses) def _callProcess(self, dir, cmdAsList, invocationStyle=None, **kwargs): '''launches the cmdAsList from dir and then returns to the curdir at entry.''' prevDir = os.path.abspath(os.curdir) os.chdir(dir) invocationStyleCode = len(kwargs) > 0 and InvocationStyleEnum.kUseSubprocess or invocationStyle if invocationStyleCode is None: invocationStyleCode = WrapExternalProgram._invocationStyle #invocationStyleCode = InvocationStyleEnum.kUseSpawn subP = None try: for k, v in os.environ.iteritems(): _LOG.debug('%-20s=%s' % (k,v)) rc = None if invocationStyleCode == InvocationStyleEnum.kUseSystem: cmd = ' '.join(cmdAsList) _LOG.debug('system(%s)' % cmd) os.system(cmd) else: if invocationStyleCode == InvocationStyleEnum.kUseSpawn: _LOG.debug('spawnv(os.P_WAIT, %s, %s)' % (cmdAsList[0], str(cmdAsList))) subP = os.spawnv(os.P_NOWAIT, cmdAsList[0], cmdAsList) self.runningProcesses.append(subP) rc = os.waitpid(subP, 0)[1] else: _LOG.debug('subprocess.call(%s, %s)' % (str(cmdAsList), str(kwargs))) subP = subprocess.Popen(cmdAsList, **kwargs) self.runningProcesses.append(subP) rc = subP.wait() _LOG.debug('returned %d' % rc) _LOG.debug('process ran') finally: os.chdir(prevDir) if subP is not None and subP in self.runningProcesses: self.runningProcesses.remove(subP) def cleanupTemporaries(self, tempDirectory, extraPaths, numberedPaths = []): '''tries to remove "tempDirectory" after removing the files within it. "extraPaths" is a list of file paths (relative to tempDirectory) to be removed. "numberedPaths" is like but each entry should contain one %d (and no other printf style directives. Integers from 0, 1... will be substituted in (and then the file will be removed).''' for p in extraPaths: _LOG.debug('try remove ' + p) tryRemove(os.path.join(tempDirectory, p)) for pTemplate in numberedPaths: p = pTemplate % 0 tryRemove(os.path.join(tempDirectory, p)) i = 1 while tryRemove(os.path.join(tempDirectory, pTemplate % i)): i += 1 tryRmdir(tempDirectory) def _createTempDataFile(self, **kwargs): '''returns (tempDirectory, (matrixFilename, matrixFile), (treeFilename, treeFile), toExternalTranslation, treeTaxLabels) with the matrix and tree already written to the fileStream. kwargs can contain: tree matrix matrixFilename treeFilename dirPref matrixFormat (default = MatrixFormat.PHYLIP The field self.mappedTree will be overwritten. ''' tree = kwargs.get('tree') matrix = kwargs.get('matrix') dirPref = kwargs.get('dirPref', 'wrap') if tree is not None: self.mappedTree, treeTaxLabels, rows, toExternalTranslation = mapTreeToLeafSubset(tree) else: self.mappedTree, treeTaxLabels, rows, toExternalTranslation = None, [], None, None tempDirectory = tempfile.mkdtemp('', dirPref, dir = self.registry.properties.tmpDir) _LOG.debug('created directory %s' % tempDirectory) matrixFilename = kwargs.get('matrixFilename', 'infile') matrixFile = None if matrix is not None: filePath = os.path.join(tempDirectory, matrixFilename) matrixFile = open(filePath, 'w') _LOG.debug('writing matrix to ' + filePath) writeDataMatrix(matrix, matrixFile, rowSlice=rows, format=kwargs.get('matrixFormat', MatrixFormat.PHYLIP)) treeFilename = kwargs.get('treeFilename', 'treefile') treeFile = None if tree is not None and not kwargs.get('treeForTaxaOnly', False): treeToWrite = self.mappedTree filePath = os.path.join(tempDirectory, treeFilename) treeFile = open(filePath, 'w') writeTrees(treeFile, [treeToWrite], treeTaxLabels) return tempDirectory, [matrixFilename, matrixFile], [treeFilename, treeFile], toExternalTranslation, treeTaxLabels # use subprocess in wrappers (which are usually simple servers) iff we can import it. try: import subprocess WrapExternalProgram._invocationStyle = InvocationStyleEnum.kUseSubprocess except ImportError: if sys.platform.upper().startswith('WIN'): import win32all # we only need this if we can't use subprocess # sys.exit('subprocess module (version of python 2.4 and greater) is required.') class RNGSeeder(random.Random): '''Very simple wrapper around that random.Random that adds a version of randint called "externalSeed" which can be used to seed other program's random number generators.''' def __init__(self, x=None): '''x can be a seed.''' random.Random.__init__(self, x) if x is not None: self.seed(x) else: self.nextExternalSeed = self.randint(2,2147483647) self.lastExternalSeed = None def seed(self, x): random.Random.seed(self, x) self.nextExternalSeed = x def externalSeed(self): '''returns a random int in the range [2,2147483647]. The last returned value is stored in the field lastExternalSeed.''' self.lastExternalSeed = self.nextExternalSeed self.nextExternalSeed = self.randint(2,2147483647) return self.lastExternalSeed class SimpleExternalWrapperServer(SimpleServer, WrapExternalProgram): def __init__(self, registry): SimpleServer.__init__(self, registry) self.userConfigCmd = '' self.runningProcesses = [] self.exiting = False def getUIXml(self): return '' def execute(self, command): _LOG.debug('execute called with "%s"' % command) self.userConfigCmd = command # for NEXUS programs (in nexus_program_server) we concatenate commands. Here we replace return (True, '') def __del__(self): self.exiting = True _LOG.debug('__del__() called') self.removeServants() def remove(self): self.exiting = True _LOG.debug('remove() called') self.removeServants() SimpleServer.remove(self) def removeServants(self): _LOG.debug('removeServants()') WrapExternalProgram.killChildProcesses(self.runningProcesses)