#!/usr/bin/python # Copyright (c) 2005 by Mark T. Holder, Florida State University. (see end of file) """Parses the cipres_config.properties file""" import os import sys import getpass import re import time import subprocess from socket import gethostname from PIPRes.util.io import expandPath, getCipresUserDir, getCipresFile from PIPRes.util.cipres import cipresGetLogger _LOG = cipresGetLogger('pipres.util._cipres_properties') def isWindows(): return sys.platform == 'win32' class CipresConfigurationError(AssertionError): def __init__(self, message, file = None): self.file = file if file: message = 'In file %s: %s' % (file, message) AssertionError.__init__(self, message) def getIORFromFilePath(regIORFilePath, svc): """Reads the filepath expecting just an IOR (used for getting the properties and registry services svc is a string (e.g. "registry" or "properties service") to display in error messages""" if not os.path.exists(regIORFilePath): msg = regIORFilePath + ' does not exist' _LOG.debug("Connecting to " + svc + " service: " + msg) raise CipresConfigurationError(msg) regIORFileObj = open(regIORFilePath, 'rU') try: regIORContents = regIORFileObj.readlines() regIOR = regIORContents[0].strip() if not regIOR: msg = regIORFilePath + ' is empty.' _LOG.debug("Connecting to " + svc + " service: " + msg) raise CipresConfigurationError(msg) return regIOR finally: regIORFileObj.close() propsAsAService = True if propsAsAService: def getCipresRootDirNoProperties(): """Checks for $CIPRES_ROOT (used when the trying to find the Properties or Registry service so we cannot use the Properties service to get this info.""" cipresRoot = os.environ.get('CIPRES_ROOT') if not cipresRoot: m = 'CIPRES_ROOT environmental variable must be set' raise CipresConfigurationError(m) if not os.path.exists(cipresRoot): raise CipresConfigurationError(cipresRoot + ' does not exist') return cipresRoot def getCipresResouresDirNoProperties(): """Checks for $CIPRES_ROOT/share/cipres/resources (used when the trying to find the Properties service so we cannot use the Properties service to get this info.""" cipresRoot = getCipresRootDirNoProperties() cipresResourcesDir = os.path.join(cipresRoot, 'share', 'cipres', 'resources') if not os.path.exists(cipresResourcesDir): raise CipresConfigurationError(cipresResourcesDir + ' does not exist') return cipresResourcesDir def getCipresPortsDirNoProperties(): pd = os.path.join(getCipresResouresDirNoProperties(), 'ports') if not os.path.exists(pd): raise CipresConfigurationError(pd + ' does not exist') return pd def launch_registry(): cip_root = getCipresRootDirNoProperties() suffix = isWindows() and ".bat" or ".sh" reg_launcher = "user_registry" + suffix path_to_launcher = os.path.join(cip_root, "bin", reg_launcher) _LOG.debug("Launching registry using %s" % path_to_launcher) if not os.path.exists(path_to_launcher): m = path_to_launcher + ' does not exist' _LOG.debug(m) raise CipresConfigurationError(m) return subprocess.Popen(path_to_launcher, shell=True) def getCipresRegistryIOR(launch_reg=True): cipresPortsDir = getCipresPortsDirNoProperties() username = getpass.getuser() host_frag = "_" + gethostname() mids = ["", host_frag] for middle in mids: try: regIORFile = "%s%s_registry.txt" % (username, middle) regIORFilePath = os.path.join(cipresPortsDir, regIORFile) return getIORFromFilePath(regIORFilePath, "registry") except CipresConfigurationError: pass if launch_reg: reg_proc = launch_registry() for i in range(50): try: return getCipresRegistryIOR(launch_reg=False) except: if reg_proc.poll() is not None: m = "Registry could not be launched" raise CipresConfigurationError(m) time.sleep(.5) raise CipresConfigurationError("No registry ports file found does not exist") def getCipresPropertiesIOR(): cipresPortsDir = getCipresPortsDirNoProperties() username = getpass.getuser() propIORFile = username + "_propertyservice.txt" propIORFilePath = os.path.join(cipresPortsDir, propIORFile) return getIORFromFilePath(propIORFilePath, "property service") def launchPropertiesService(): import subprocess cipresRoot = os.environ.get('CIPRES_ROOT') if not cipresRoot: raise CipresConfigurationError('CIPRES_ROOT environmental variable must be set') script = isWindows() and 'user_propsvc.bat' or 'user_propsvc.sh' scriptPath = os.path.join(cipresRoot, 'bin', script) if not os.path.exists(scriptPath): raise CipresConfigurationError(scriptPath + ' does not exist') # temp we should record the pid and redirect output to a file if isWindows(): subprocess.Popen([scriptPath]) # spawns process that is never killed else: subprocess.Popen(['/bin/sh', scriptPath]) # spawns process that is never killed return def testPropertiesService(ior): orb = getCipresORB() sleepSeconds = 0.5 nRetries = 10 for i in range(nRetries): try: propServiceObjRef = orb.string_to_object(ior) isSet, value = propServiceObjRef.getProperty('cipres.root') return propServiceObjRef except: launchPropertiesService() time.sleep(sleepSeconds) raise CipresConfigurationError('Could not contact (or launch) the CipresProperties service on port %d' % propertiesPort) def getCipresPropertiesObjectRef(): # code to find propService ior pIOR = getCipresPropertiesIOR() return testPropertiesService(pIOR) # def getPortFromPortsFileObj(userPortsFileObj, desiredService=''): # desiredService = desiredService.upper() # allPorts = [] # for line in userPortsFileObj: # try: # service, portStr = line.split() # port = int(portStr) # if desiredService: # if service.upper() == desiredService: # return port # else: # allPorts.append(port) # except: # raise CipresConfigurationError('Expecting\nService port-number\nformat. Encountered:\n' + line, userPortsFilePath) # if desiredService: # return None # return allPorts # # # def launchPropertiesServiceOnFreePort(): # '''Launches the CipresProperties on a port that appears to be free. # Updates the ports-.txt file. # Returns the port number.''' # _portFileNamePattern = re.compile(r'ports-[a-zA-Z]+\.txt') # cipresResourcesDir = getCipresResouresDirNoProperties() # propertiesPort = None # userName = getpass.getuser() # userPortsFileName = 'ports-%s.txt' % userName # userPortsFilePath = os.path.join(cipresResourcesDir, userPortsFileName) # userPortsLockFileName = 'ports-%s-lock.txt' % userName # userPortsLockFilePath = os.path.join(cipresResourcesDir, userPortsLockFileName) # sleepSeconds = 0.5 # maxNumRetries = 10 # numRetries = 0 # #CHANGE this to launching the service and waiting for it to write the ports-.txt file # # # #while os.path.exists(userPortsLockFilePath): # # time.sleep(sleepSeconds) # # numRetries += 1 # # if numRetries > maxNumRetries: # # raise CipresConfigurationError('Lock file present (indicating that another process is editing %s) for too long. Remove the file manually if you are confident that %s is not being edited.' % (userPortsFileName, userPortsFileName), userPortsLockFilePath) # #else: # # open(userPortsLockFilePath, 'w').close() # # try: # # #now that we have the lock, check # # propertiesPort = parseFileForPropertiesServicePort(userPortsFileObj) # # finally: # # os.remove(userPortsLockFilePath) # # def parseFileForPropertiesServicePort(userPortsFilePath): # '''Reads userPortsFilePath for the CipresProperties port (and returns the port or None). # Creates userPortsFilePath if it does not exist.''' # propertiesPort = None # if os.path.exists(userPortsFilePath): # userPortsFileObj = open(userPortsFilePath, 'rU') # try: # propertiesPort = getPortFromPortsFileObj(userPortsFileObj, 'CipresProperties') # finally: # userPortsFileObj.close() # #else: # # try: # # open(userPortsFilePath, 'w').close() # # os.chmod(userPortsFilePath, 0664) # # except: # # raise CipresConfigurationError('Could not create the file ' + userPortsFilePath) # return propertiesPort # # def testForPropertiesServiceOnPort(propertiesPort): # '''Check that the CipresProperties Service is running on "propertiesPort". # Relaunches the process if it is not responing. # Returns an object reference to the service if successful (or raises a CipresConfigurationError). # ''' # ior = 'corbaloc::127.0.0.1:%d/CipresProperties' % propertiesPort # return testPropertiesService(ior) class CipresProperties: def __init__(self): self.propService = getCipresPropertiesObjectRef() def getPlatform(self): return sys.platform platform = property(getPlatform) else: #not propsAsAService class CipresProperties: def __init__(self): pass def getPlatform(self): return sys.platform platform = property(getPlatform) class FakeProps(CipresProperties): def __init__(self, **kwargs): self.registryRetryCount = 12 self.registryRetryInterval = 0.5 self.applicationSendNexusAsString = False self.__dict__.update(kwargs) def set(self, k, v): self.__dict__[k] = v return v def get(self, k): v = self.__dict__.get(k) if v is None: return False, '' return True, v if False: #old code class ConfigurationError(ValueError): def __init__(self, message, file = '', line = -1): self.message = message self.file = file self.line = line def __str__(self): if self.file: if self.line >= 0: return '%s (from line %d of %s)' %(self.message, self.line, self.file) return '%s (file = %s)' %(self.message, self.file) if self.line >= 0: return '%s (line %d of configuration file)' %(self.message, self.line) return self.message ############################################################################################# # functions used to validate data types (or raise ConfigurationError exceptions). ############################################################################################# def _propertyToBool(s): '''Interprets string {0|1}, {Yes|No}, or {True,False} as a python bool (or raises ConfigurationError''' uc = s.upper() if uc in ['0', 'N', 'NO', 'F', 'FALSE', '']: return False if uc in ['1', 'Y', 'YES', 'T', 'TRUE']: return True raise ConfigurationError('Invalid value (%s). Expecting {1, Yes, True, 0, No, or False}' % s) def _verifyPath(rawVal, shouldBeDir = False): rawVal = expandPath(rawVal) if os.path.exists(rawVal): if not (shouldBeDir ^ os.path.isdir(rawVal)): return rawVal raise ConfigurationError('Expecting the path (%s) to an existing %s' % (rawVal, (shouldBeDir and 'directory' or 'file'))) def _propertyToFilePath(s): return _verifyPath(s, False) def _propertyToDirPath(s): return _verifyPath(s, True) def _propertyToNewOrExistingDir(rawVal): rawVal = expandPath(rawVal) try: if not os.path.exists(rawVal): os.makedirs(rawVal) if os.path.isdir(rawVal): return rawVal except: pass if os.path.exists(rawVal): errorMsg = 'The path (%s) exists, but is not a directory' % rawVal else: errorMsg = 'The path (%s) does not exist and could not be created' % rawVal raise ConfigurationError(errorMsg) def _propertyToFloat(rawVal): try: return float(rawVal) except ValueError: pass except TypeError: pass raise ConfigurationError('Invalid value (%s). Expecting a real number' % rawVal) def _propertyToInt(rawVal): try: return int(rawVal) except ValueError: pass except TypeError: pass raise ConfigurationError('Invalid value (%s). Expecting an integer' % rawVal) ############################################################################################# # Classes that encapsulate each type of property that we read (these process values and yield # Encapsulating the properties allows us to substitute PIPRes names for the properties that # map to the value in the properties file (this is mainly used so that PIPRes code does not # need to be changed much if we alter a property name) and to make the properties follow # PIPRes capitalization rules. ############################################################################################# class CipresConfigProperty(object): def __init__(self, pipresName, serializedName, **kwargs): self.pipresName = pipresName self.serializedName = serializedName self.processFunc = kwargs.get('processFunc') self.transformFunc = kwargs.get('transformFunc') self.serializeFunc = kwargs.get('serializeFunc') self.default = kwargs.get('default') self.reset() def matchesKey(self, serializedKey): return serializedKey == self.serializedName def processValue(self, rawValue): if self.processFunc is not None: try: self.value = self.processFunc(rawValue) except ConfigurationError, x: self.errMessage = x.message + ' (error while processing the %s property)' % self.serializedName x.message = self.errMessage raise x else: self.value = rawValue if self.transformFunc is not None: self.value = self.transformFunc(self.value) def reset(self): self.value = None try: if self.default is not None: self.processValue(str(self.default)) except ConfigurationError: pass self.errMessage = '' def validatePipresFormIfSet(self, d): v = d.get(self.pipresName) if v is not None: self.processValue(str(v)) class DirCipresConfigProperty(CipresConfigProperty): def __init__(self, pipresName, serializedName, **kwargs): kwargs['processFunc'] = kwargs.get('processFunc', _propertyToDirPath) super(DirCipresConfigProperty, self).__init__(pipresName, serializedName, **kwargs) class FileCipresConfigProperty(CipresConfigProperty): def __init__(self, pipresName, serializedName, **kwargs): kwargs['processFunc'] = kwargs.get('processFunc', _propertyToFilePath) super(FileCipresConfigProperty, self).__init__(pipresName, serializedName, **kwargs) class FloatCipresConfigProperty(CipresConfigProperty): def __init__(self, pipresName, serializedName, **kwargs): kwargs['processFunc'] = kwargs.get('processFunc', _propertyToFloat) super(FloatCipresConfigProperty, self).__init__(pipresName, serializedName, **kwargs) class IntCipresConfigProperty(CipresConfigProperty): def __init__(self, pipresName, serializedName, **kwargs): kwargs['processFunc'] = kwargs.get('processFunc', _propertyToInt) super(IntCipresConfigProperty, self).__init__(pipresName, serializedName, **kwargs) class BoundedIntCipresConfigProperty(IntCipresConfigProperty): def __init__(self, pipresName, serializedName, **kwargs): self.min = kwargs.get('min') self.max = kwargs.get('max') super(BoundedIntCipresConfigProperty, self).__init__(pipresName, serializedName, **kwargs) def processValue(self, rawValue): super(BoundedIntCipresConfigProperty, self).processValue(rawValue) if self.min is not None: if self.max is not None: if self.value < self.min or self.value > self.max: self.value = None raise ConfigurationError('Expecting an integer in the range [%d, %d] for the %s property.' % (self.min, self.max, self.serializedName)) else: if self.value < self.min: self.value = None raise ConfigurationError('Expecting an integer greater than %d for the %s property.' % (self.min - 1, self.serializedName)) else: if self.max is not None and self.value > self.max: self.value = None raise ConfigurationError('Expecting an integer less than %d for the %s property.' % (self.max + 1, self.serializedName)) class BoundedFloatCipresConfigProperty(FloatCipresConfigProperty): def __init__(self, pipresName, serializedName, **kwargs): self.min = kwargs.get('min') self.max = kwargs.get('max') super(BoundedFloatCipresConfigProperty, self).__init__(pipresName, serializedName, **kwargs) def processValue(self, rawValue): super(BoundedFloatCipresConfigProperty, self).processValue(rawValue) if self.min is not None: if self.max is not None: if self.value < self.min or self.value > self.max: self.value = None raise ConfigurationError('Expecting an number in the range [%d, %d] for the %s property.' % (self.min, self.max, self.serializedName)) else: if self.value < self.min: self.value = None raise ConfigurationError('Expecting an number greater than %d for the %s property.' % (self.min - 1, self.serializedName)) else: if self.max is not None and self.value > self.max: self.value = None raise ConfigurationError('Expecting an number less than %d for the %s property.' % (self.max + 1, self.serializedName)) class BoolCipresConfigProperty(CipresConfigProperty): def __init__(self, pipresName, serializedName, **kwargs): kwargs['processFunc'] = kwargs.get('processFunc', _propertyToBool) super(BoolCipresConfigProperty, self).__init__(pipresName, serializedName, **kwargs) def _interpretCipresTagVal(propertyList, tag, rawVal): '''Uses the CipresConfigProperty objects in propertyList to validate rawVal as a value for the tag. The tag is assumed to have had the cipres. prefix removed!''' for prop in propertyList: if prop.matchesKey(tag): prop.processValue(rawVal) return prop.pipresName, prop.value # cipres. properties that are not in the properties list are returned as camelCase keys and string values tagSplit = tag.split('.') return tagSplit[0] + ''.join([(i[0].upper() + i[1:]) for i in tagSplit[1:] if len(i) > 0]), rawVal class CipresProperties(object): '''A class that eads the configuration file, validates the subset of the content that is used by PIPRes. Instances should generally be accessed using getCipresProperty() from PIPRes.util.cipres this function guarantees that the CipresProperties instance is a singletong''' _requiredProperties = [ #'applicationRoot', #'registryFile', #'tmpDir', 'registryRetryInterval', 'registryRetryCount', ] ############################################################################################# # Instances of specific properties that are validated. ############################################################################################# expectedProperties = [ #DirCipresConfigProperty( 'applicationRoot', 'application.root' ), #FileCipresConfigProperty( 'registryFile', 'registry.file', default = os.path.join(getCipresFile('cipres_registry.xml')) ), DirCipresConfigProperty( 'tmpDir', 'tmpDir', default = os.path.join(getCipresUserDir(True), 'tmp'), processFunc = _propertyToNewOrExistingDir ), BoundedFloatCipresConfigProperty('registryRetryInterval', 'registry.retryInterval', default = 2.0, min = 0, transformFunc = lambda x : float(x)/1000.0 , serializeFunc = lambda x : '%d' % x), BoundedIntCipresConfigProperty( 'registryRetryCount', 'registry.retryCount', default = 12, min = 1), BoolCipresConfigProperty( 'useColocatedRegistry', 'pipres.use_colocated_registry', default = False), BoolCipresConfigProperty( 'applicationSendNexusAsString', 'application.sendNexusAsString', default = False), CipresConfigProperty( 'registryCommandPrefix' , 'registry.commandPrefix'), ] _pipresPropertyMap = {} _serialzedPropertyMap = {} def __init__(self, **kwargs): self.registryRetryCount = 12 self.registryRetryInterval = 0.5 self.useColocatedRegistry = False self.registryIor = 'corbaloc::127.0.0.1:1070/CipresRegistry' self.registryCommandPrefix = None self.platform = sys.platform self.applicationSendNexusAsString = False propFile = kwargs.get('fileObject') propFileDefaults = kwargs.get('fileObjectDefaults') fileName = kwargs.get('fileName') fileNameDefaults = kwargs.get('fileNameDefaults') if propFileDefaults is None and fileNameDefaults is not None: propFileDefaults = open(fileNameDefaults, 'rU') if propFile is None: if not fileName: raise TypeError, 'Expecting fileObject or fileName argument to CipresProperties.__init__' propFile = open(fileName, 'rU') if propFileDefaults is not None: self._readCipresProperties(propFileDefaults, fileNameDefaults) self._readCipresProperties(propFile, fileName, False) else: self._readCipresProperties(propFile, fileName) def reset(self): self.__dict__.clear() for p in CipresProperties.expectedProperties: p.reset() self.__dict__[p.pipresName] = p.value def _readCipresProperties(self, propFile, fileName, reset=True): _LOG.debug('Pipres begin loading cipres properties from %s' % fileName) if reset is True: self.reset() try: for n, line in enumerate(propFile): line = line.lstrip() if line.startswith('cipres.'): brokenLine= line.split('=') if len(brokenLine) < 2: errorMsg = 'Expecting "=" in line starting with "cipres."' raise ConfigurationError(errorMsg, line = n) tag = brokenLine[0].rstrip() value = '='.join(brokenLine[1:]).strip() try: key, translatedVal = _interpretCipresTagVal(CipresProperties.expectedProperties, tag[len('cipres.'):], value) except ConfigurationError, e: e.file = fileName or '', n raise e self.__dict__[key] = translatedVal self._validateProperties() except ConfigurationError, e: e.file = fileName or '' _LOG.error('Error: %s\n\tin properties file (%s) at line %d' %(e.message, e.file, e.line)) raise e #self._propertiesFileName = fileName _LOG.debug('Pipres loaded cipres properties from %s' % fileName) return self def _validateProperties(self): for p in CipresProperties._requiredProperties: if self.__dict__.get(p) is None: propObj = CipresProperties._pipresPropertyMap.get(p) assert(p is not None) # required properties should all be in the _pipresPropertyMap dictionary). errorMsg = 'The required property "cipres.%s" was not found' % propObj.serializedName _LOG.error(errorMsg) raise ConfigurationError(errorMsg) for prop in CipresProperties.expectedProperties: prop.validatePipresFormIfSet(self.__dict__) import sys self.platform = sys.platform if 'registryCommandPrefix' not in self.__dict__: self.registryCommandPrefix = None if 'PIPRES_ROOT' not in os.environ and 'CIPRES_ROOT' in os.environ: os.environ['PIPRES_ROOT'] = os.path.join(os.environ.get('CIPRES_ROOT'), 'lib', 'python', 'site-packages') return self def __eq__(self, other): try: od = isinstance(other, dict) and other or other.__dict__ except AttributeError: raise NotImplementedError, 'CipresProperties __eq__ expecting dict or class with __dict__' #for k in self.__dict__.keys() + od.keys(): # if self.__dict__[k] != od[k]: # print k, 'is', self.__dict__[k], 'not', od[k] return self.__dict__ == od def __ne__(self, other): return not self == other for prop in CipresProperties.expectedProperties: assert(prop.pipresName not in CipresProperties._pipresPropertyMap) assert(prop.serializedName not in CipresProperties._serialzedPropertyMap) CipresProperties._pipresPropertyMap[prop.pipresName] = prop CipresProperties._serialzedPropertyMap[prop.serializedName] = prop def getFullSharedLibName(libName, platform): if platform.startswith('darwin'): return 'lib%s.dylib' % libName elif platform == 'win32': return 'lib%s.dll' % libName elif platform.startswith('linux'): return 'lib%s.so' % libName else: # TEMP just guessing linux style return 'lib%s.so' % libName def getPathToCipresDistributedLib(libName, cipresRoot, platform): '''Looks of dynamic libraries distributed with CIPRES and returns the full path or None if not found''' fullLibName = getFullSharedLibName(libName, platform) if platform.startswith('darwin'): dirName = 'osx-ppc' elif platform.startswith('linux'): dirName = 'linux' else: return None fullPath = os.path.join(expandPath(cipresRoot), 'lib', dirName, fullLibName) if os.path.exists(fullPath): return os.path.abspath(fullPath) return None # This file is part of the PIPRes library # The PIPRes library is free software; you can redistribute it # and/or modify it under the terms of the GNU Lesser General # Public License as published by the Free Software Foundation; # either version 2.1 of the License, or (at your option) any later # version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free # Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, # MA 02111-1307, USA