refactored some of the cdb python classes (utility, base api and cli classes,...

refactored some of the cdb python classes (utility, base api and cli classes, exceptions, constants)

src/python/dm/common/utility/ldapUtility.py

+#!/usr/bin/env python
+
+import ldap
+
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+
+class LdapUtility:
+
+    def __init__(self, serverUrl, dnFormat):
+        self.serverUrl = serverUrl
+        self.dnFormat = dnFormat
+         
+    def checkCredentials(self, username, password):
+        """Verifies credentials for username and password. """
+        ldapUsername = self.dnFormat % username
+        try:
+            # build client
+            ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+            ldapClient = ldap.initialize(self.serverUrl)
+            ldapClient.set_option(ldap.OPT_REFERRALS,0)
+            ldapClient.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
+
+            #ldapClient.ststart_tls_s()
+            #ldapClient.set_option(ldap.OPT_X_TLS,ldap.OPT_X_TLS_DEMAND)
+            #ldapClient.set_option( ldap.OPT_X_TLS_DEMAND, True)
+            #ldapClient.set_option( ldap.OPT_DEBUG_LEVEL, 255)
+
+            # perform a synchronous bind
+            ldapClient.simple_bind_s(ldapUsername, password)
+            #ldapClient.whoami_s()
+        except ldap.INVALID_CREDENTIALS, ex:
+            ldapClient.unbind()
+            raise AuthenticationError('Invalid LDAP credentials for user %s' % username)
+        except ldap.SERVER_DOWN, ex:
+            raise CommunicationError('Cannot reach LDAP server %s' % self.serverUrl)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    ldapUtility = LdapUtility(serverUrl='ldaps://ldap.anl.gov:636', dnFormat='uid=%s,dc=anl,dc=gov')
+    ldapUtility.checkCredentials('sveseli', 'sv')

src/python/dm/common/utility/loggingManager.py

+#!/usr/bin/env python
+
+#
+# Logging manager singleton.
+#
+
+#######################################################################
+
+import re
+import sys
+import os.path
+import logging
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+
+#######################################################################
+
+
+class LoggingManager:
+    """ 
+    The log manager class is initialized via a configuration file 
+    that may have the following sections:
+
+    ConsoleLogging      # Used for output on the screen
+    FileLogging         # Used for logging into a file
+
+    Each section in the configuration file should have the following 
+    keys:
+        handler     # Indicates which handler class to use 
+        level       # Indicates logging level
+        format      # Indicates format for log messages
+        dateFormat  # Indicates date format used for log messages
+
+    Given below is an example of a valid configuration file:
+
+    [ConsoleLogging]
+        handler=ConsoleLoggingHandler(sys.stdout,)
+        level=info
+        format=[%(levelname)s] %(message)s 
+        dateFormat=%m/%d/%y %H:%M:%S
+    [FileLogging]
+        handler=TimedRotatingFileLoggingHandler('/tmp/dm.log')
+        level=debug
+        format=%(asctime)s,%(msecs)d [%(levelname)s] %(module)s:%(lineno)d %(user)s@%(host)s %(name)s (%(process)d): %(message)s
+        dateFormat=%m/%d/%y %H:%M:%S
+    """
+
+    # Get singleton instance.
+    @classmethod
+    def getInstance(cls):
+        from dm.common.utility.loggingManager import LoggingManager
+        try:
+            lm = LoggingManager()
+        except LoggingManager, ex:
+            lm = ex
+        return lm
+
+    # Singleton.
+    __instance = None
+
+    def __init__(self):
+        if LoggingManager.__instance:
+             raise LoggingManager.__instance
+        LoggingManager.__instance = self
+        self.consoleHandler = None
+        self.fileHandlerList = []
+        self.maxIntLevel = logging.CRITICAL
+        self.minIntLevel = logging.NOTSET
+        self.levelRegExList = []
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.initFlag = False
+
+    def setMinLogLevel(self, minLogLevel=logging.INFO):
+        self.minIntLevel = minLogLevel
+
+    def parseLevelRegEx(self, levelRegExList):
+        """ Parse expressions of the form <regex>=<log level>. """
+        lines = levelRegExList.split('\n')
+        for line in lines:
+            try:
+                 (regex, level) = line.rsplit('=', 1)
+                 pattern = re.compile(regex)
+                 tuple = (pattern, logging.getLevelName(level.upper()))
+                 self.levelRegExList.append(tuple)
+            except Exception, ex:
+                 self.logger.error('Parser error in log configuration file: %s' % line)
+                 self.logger.exception(ex)
+
+    # Get Log Level based on a string representation
+    def getIntLogLevel(self, levelStr):
+        level = logging.getLevelName(levelStr)
+        # Level should be an integer
+        try:
+            return int(level) 
+        except ValueError, ex:
+            raise ConfigurationError('"%s" is not valid log level' % levelStr)
+
+    # Configure log handlers.
+    def configureHandlers(self):
+        """ Configure log handlers from the config file. """
+        cm = ConfigurationManager.getInstance()
+        configFile = cm.getConfigFile()
+        configParser = cm.getConfigParserFromConfigFile(configFile)
+        configSections = cm.getConfigSectionsFromConfigParser(configParser)
+
+        # Console handler.
+        defaults = { 
+            'level' : cm.getConsoleLogLevel(),
+            'format' : cm.getLogRecordFormat(),
+            'dateFormat' : cm.getLogDateFormat(),
+            'handler' : 'ConsoleLoggingHandler(sys.stdout,)'
+        }
+        consoleHandler = self.__configureHandler(configParser, 'ConsoleLogging', defaults)
+
+        if consoleHandler != None:
+            self.consoleHandler = consoleHandler
+
+        # File logging. 
+        # Do not configure if log directory does not exist.
+        defaults['handler'] = None
+        defaults['level'] = cm.getFileLogLevel() 
+        if not os.path.exists(configFile):
+            # No config file, we'll configure default.
+            defaultLogFile = cm.getLogFile()
+            defaultLogDir = os.path.dirname(defaultLogFile)
+            if os.path.exists(defaultLogDir):
+                handler = 'TimedRotatingFileLoggingHandler("%s")' % defaultLogFile 
+                defaults['handler'] = handler
+                fileHandler = self.__configureHandler(configParser, 'FileLogging', defaults)
+                if fileHandler != None:
+                    self.fileHandlerList.append(fileHandler)
+
+        else:
+            # Parse all file loggers present in the config file
+            for configSection in configSections:
+                if configSection.startswith('FileLogging'):
+                     fileHandler = self.__configureHandler(configParser, configSection, defaults)
+                     if fileHandler != None:
+                         self.fileHandlerList.append(fileHandler)
+
+        # Add handlers to the root logger.  Use logging class here
+        # to make sure we can have a logger when we parse the 
+        # logger expressions
+        rootLogger = logging.getLogger('')
+        for handler in [self.consoleHandler] + self.fileHandlerList:
+            rootLogger.addHandler(handler)
+
+        # Get a logger factory based on our current config 
+        self.configureLoggers(configParser, defaultLevel='debug')
+
+    def configureLoggers(self, configParser, defaultLevel='error'):
+        rootLogLevel = 'error'
+        levelRegEx = '^.*$=%s' % (defaultLevel)
+        if configParser is not None and configParser.has_section('LoggerLevels'):
+            rootLogLevel = ConfigurationManager.getOptionFromConfigParser(configParser, 'LoggerLevels', 'root', rootLogLevel)
+            levelRegEx = ConfigurationManager.getOptionFromConfigParser(configParser, 'LoggerLevels', 'levelregex', levelRegEx)
+
+        rootLevelInt = logging.getLevelName(rootLogLevel.upper())
+        logging.getLogger('').root.setLevel(rootLevelInt)
+        logging.getLogger('').debug('Set root logger to %s' % rootLevelInt)
+
+        if not levelRegEx:
+            return
+
+        # Parse expressions of the form <regex>=<log-level>. """
+        lines = levelRegEx.split('\n')
+        for line in lines:
+            try:
+                # Use the right split so we can have '='s in the regex
+                (regex, level) = line.rsplit('=', 1)
+                pattern = re.compile(regex)
+                tuple = (pattern, logging.getLevelName(level.upper()))
+                self.levelRegExList.append(tuple)
+            except Exception, ex:
+                # Do not fail
+                self.logger.error('Parser error in log configuration file: %s' % line)
+                self.logger.exception(ex)
+
+    # Configure particular handler with given defaults.
+    def __configureHandler(self, configParser, configSection, defaults):
+        """ Configure specified handler with a given defaults. """
+        handlerOption = defaults['handler']
+        try:
+            if configParser is not None:
+                handlerOption = configParser.get(configSection, 'handler')
+        except Exception, ex:
+            pass
+
+        # If handlerOption is empty, handler cannot be instantiated.
+        handler = None
+        if handlerOption != None:
+            # Handler argument format: MyHandler(arg1, arg2, ...)
+            # Module will be in lowercase letters, but the class
+            # should be capitalized.
+            handlerName = re.sub('\(.*', '', handlerOption)
+            moduleName = handlerName[0].lower() + handlerName[1:]
+            try:
+                exec 'from dm.common.utility import %s' % (moduleName)
+                exec 'handler = %s.%s' % (moduleName, handlerOption)
+            except IOError, ex:
+                errNo, errMsg = ex
+                import errno
+
+                # If the exception raised is an I/O permissions error, ignore
+                # it and disable this log handler.  This allows non-root users
+                # to use the (system-wide) default log configuration
+                if errNo != errno.EACCES:
+                    raise
+                handler = None 
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+        # Only request setting from the config file if it was
+        # not set via environment variable, or programmatically.
+        cm = ConfigurationManager.getInstance()
+        if handler != None:
+            try:
+                level = cm.getOptionFromConfigParser(configParser, configSection, 'level', defaults['level'])
+                intLevel = self.getIntLogLevel(level.upper())
+                handler.setLevel(intLevel)
+
+                format = cm.getOptionFromConfigParser(configParser, configSection, 'format', defaults['format'])
+                dateFormat = cm.getOptionFromConfigParser(configParser, configSection, 'dateFormat', defaults['dateFormat'])
+
+                handler.setFormatter(logging.Formatter(format, dateFormat))
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+            # Apply filters to handler
+            filter = None
+            try:
+                filter = configParser.get(configSection, 'filter')
+                if filter:
+                    handler.addFilter(logging.Filter(filter))
+            except Exception, ex:
+                pass
+        return handler
+
+    def getLogger(self, name='defaultLogger'):
+        if not self.initFlag:
+            self.initFlag = True
+            self.configureHandlers()
+        logger = logging.getLogger(name)
+        logger.setLevel(self.getLevel(name))
+        return logger
+
+    def getLevel(self, name):
+        # Match from the known regex list.
+        level = logging.NOTSET
+
+        # The last regex is most important.
+        for e in reversed(self.levelRegExList):
+            (pattern, level) = e
+
+            # If we return not None it is a match
+            if not None == pattern.match(name):
+                break
+
+        if level > self.maxIntLevel:
+            level = self.maxIntLevel
+        if level < self.minIntLevel:
+            level = self.minIntLevel
+        return level
+
+    def setConsoleLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            self.consoleHandler.setLevel(intLevel)
+            self.maxIntLevel = intLevel
+            self.logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+    def setFileLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            for handler in self.fileHandlerList:
+                handler.setLevel(intLevel)
+            self.maxIntLevel = intLevel
+            self.logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    lm = LoggingManager.getInstance()
+    logger = lm.getLogger('Main')
+    logger.error('Error In Main')
+    logger.debug('Debug In Main')
+    logger.warn('Warn In Main')
+    logger.info('Info In Main')
+    logger = lm.getLogger('Main')
+    logger.info('Info In Main')
+    logger = lm.getLogger('')
+    logger.info('Info using root logger')
+    logger = lm.getLogger('Main.2')
+    logger.info('Info in Main.2')
+    lm.setConsoleLogLevel('info')
+    logger.debug('You should not see this message')
+    lm.setConsoleLogLevel('debug')
+    logger.debug('Debug in Main.2')

src/python/dm/common/utility/objectCache.py

+#!/usr/bin/env python
+
+import threading
+import time
+from collections import deque
+
+class ObjectCache:
+    """
+    Cache objects identified by id. Objects are removed from cache
+    based on the last accessed algorithm.
+    """
+
+    # How much larger than object cache should time stamp deq be
+    # allowed to grow.
+    DEFAULT_TIME_STAMP_DEQ_SIZE_FACTOR = 2
+    # Cache info expiration time.
+    DEFAULT_OBJECT_LIFETIME = 60 # seconds
+
+    def __init__(self, cacheSize, objectLifetime=DEFAULT_OBJECT_LIFETIME):
+        self.lock = threading.RLock()
+        self.objectMap = {} # id/object map
+        self.timeStampDeq = deque() # timeStamp deq
+        self.cacheSize = cacheSize
+        self.objectLifetime = objectLifetime
+        self.deqSize = ObjectCache.DEFAULT_TIME_STAMP_DEQ_SIZE_FACTOR*cacheSize
+
+    def setCacheSize(self, cacheSize):
+        self.cacheSize = cacheSize
+
+    def setObjectLifetime(self, objectLifetime):
+        self.objectLifetime = objectLifetime
+
+    def __purgeOne(self):
+        # Get rid of one cached item based on the last accessed algorithm.
+        while True:
+            deqEntry = self.timeStampDeq.popleft()
+            oldId = deqEntry[0]
+            cachedEntry = self.objectMap.get(oldId)
+            if cachedEntry is not None:
+                # Timestamp entry is valid.
+                if cachedEntry == deqEntry:
+                    # Found an old item, get rid of it from the cache.
+                    del self.objectMap[oldId]
+                    break
+        # Done.
+        return
+
+    def __purgeTimeStampDeq(self):
+        # Get rid of stale entries.
+        timeStampDeq = deque()
+        while len(self.timeStampDeq):
+            deqEntry = self.timeStampDeq.popleft()
+            id = deqEntry[0]
+            cachedEntry = self.objectMap.get(id)
+            if cachedEntry is not None:
+                # Timestamp entry is valid.
+                if cachedEntry == deqEntry:
+                    # Found current item, keep it.
+                    timeStampDeq.append(deqEntry)
+        # Done.
+        self.timeStampDeq = timeStampDeq 
+        return
+
+    def put(self, id, item, objectLifetime=None):
+        updateTime = time.time()
+        expirationTime = updateTime + self.objectLifetime
+        if objectLifetime is not None:
+            expirationTime = updateTime + objectLifetime
+        entry = (id, item, updateTime, expirationTime)
+        self.lock.acquire()
+        try:
+            self.objectMap[id] = entry
+            self.timeStampDeq.append(entry)
+            if len(self.objectMap) > self.cacheSize:
+                self.__purgeOne()
+            if len(self.timeStampDeq) > self.deqSize:
+                self.__purgeTimeStampDeq()
+        
+        finally:
+            self.lock.release()
+
+    def get(self, id):
+        return self.objectMap.get(id)
+
+    def remove(self, id):
+        self.lock.acquire()
+        try:
+            item = self.objectMap.get(id)
+            if item is not None:
+                del self.objectMap[id]
+            return item
+        finally:
+            self.lock.release()
+
+    def isEmpty(self):
+        return len(self.objectMap) == 0
+
+    def size(self):
+        return len(self.objectMap) 
+
+    def __str__(self):
+        return '%s' % self.timeStampDeq
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    c = ObjectCache(3)
+
+    class Item:
+        def __init__(self, id):
+            self.id = id
+        def getId(self):
+            return self.id
+        def __str__(self):
+            return '%s' % self.id
+
+    class Item2:
+        def __init__(self, name):
+            self.name = name 
+        def getName(self):
+            return self.name
+        def __str__(self):
+            return '%s' % self.name
+
+    for i in range(0,5):
+        item = Item(i)
+        c.put(i, item)
+        print 'Added item: ', item
+        print 'Cache: ', c
+        time.sleep(1)
+
+    for j in range(0,3):
+        item = Item(2)
+        c.put(2, item)
+        print 'Updated item: ', item
+        print 'Cache: ', c
+        time.sleep(1)
+
+    item = c.remove(2)
+    print 'Deleted item 2: ', item
+    print 'Cache: ', c
+    time.sleep(1)
+    item = c.get(2)
+    print 'Got item 2: ', item
+    print 'Cache: ', c
+    print
+    time.sleep(1)
+
+    print
+    c = ObjectCache(3)
+    c.put('sv', Item2('sv')) 
+    print c
+    i = c.get('sv')
+    print i
+    print 'Done'
+

src/python/dm/common/utility/osUtility.py

+#!/usr/bin/env python
+
+import os
+
+class OsUtility:
+
+    @classmethod
+    def createDir(cls, path, mode=None):
+        """ Create directory if it does not exist already. """
+        if not os.path.isdir(path):
+            os.makedirs(path)
+            if mode is not None:
+                os.chmod(path, mode)
+
+    @classmethod
+    def removeLink(cls, path):
+        """ Remove link on a given path. """
+        if not os.path.islink(path):
+            return
+        os.remove(path)
+
+    @classmethod
+    def removeFile(cls, path):
+        """ Remove file on a given path. """
+        if not os.path.isfile(path):
+            return
+        os.remove(path)
+
+    @classmethod
+    def removeAndIgnoreErrors(cls, path):
+        """ Remove file on a given path and ignore any errors. """
+        try:
+            os.remove(path)
+        except Exception, ex:
+            pass
+
+    @classmethod
+    def removeDir(cls, path):
+        """ Remove dir on a given path, even if it is not empty. """
+        if not os.path.isdir(path):
+            return
+        files=os.listdir(path)
+        for f in files:
+            fullpath=os.path.join(path, f)
+            if os.path.islink(fullpath) or not os.path.isdir(fullpath):
+                os.remove(fullpath)
+            else:
+                removeDir(fullpath)
+        os.rmdir(path)
+        
+    @classmethod
+    def chownPath(cls, path, uid, gid):
+        """ Change owner on a given path recursively.  """
+        if os.path.isfile(path):
+            os.chown(path, uid, gid)
+            return 
+        elif os.path.islink(path):
+            os.lchown(path, uid, gid)
+            return 
+        elif os.path.isdir(path):
+            files=os.listdir(path)
+            for f in files:
+                fullpath=os.path.join(path, f)
+                chownPath(fullpath, uid, gid)
+            os.chown(path, uid, gid)
+
+    @classmethod
+    def chownPathByUserName(cls, path, userName):
+        """ Change owner on a given path recursively.  """
+        import pwd
+        user = pwd.getpwnam(userName)
+        chownPath(path, user.pw_uid, user.pw_gid)
+
+    @classmethod
+    def findFiles(cls, dirPath, fileList=None):
+        """ List files in a given directory. Return list of absolute paths.
+            Do not follow symbolic links.
+        """
+        fList = fileList
+        if not fList:
+            fList = []
+        if os.path.isdir(dirPath):
+            files = os.listdir(dirPath)
+            for f in files:
+                fullPath = os.path.join(dirPath, f)
+                if os.path.isfile(fullPath):
+                    fList.append(fullPath)
+                elif os.path.isdir(fullPath):
+                    fList = findFiles(fullPath, fList)
+        return fList
+
+    @classmethod
+    def importNameFromFile(cls, name, filePath):
+        """ Import specified name from file. """
+        import sys
+        import os.path
+        dirName = os.path.dirname(filePath)
+        moduleName = os.path.basename(filePath).replace('.py', '')
+        sys.path = [dirName] + sys.path
+        cmd = 'from %s import %s as tmpObject' % (moduleName, name)
+        exec cmd
+        del sys.path[0]
+        return tmpObject
+
+    @classmethod
+    def getUserHomeDir(cls):
+        """ Get current user home directory. """
+        from os.path import expanduser
+        home = expanduser('~')
+        return home
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/utility/sslUtility.py

+#!/usr/bin/env python
+
+import ssl
+
+class SslUtility:
+
+    DEFAULT_SSL_CONTEXT = ssl._create_default_https_context
+
+    @classmethod
+    def useUnverifiedSslContext(cls, func):
+
+        def wrapper(*args, **kwargs):
+            # Disable SSL checking
+            ssl._create_default_https_context = ssl._create_unverified_context
+
+            # Perform function call 
+            result = func(*args, **kwargs)
+
+            # Revert back to original SSL settings
+            ssl._create_default_https_context = SslUtility.DEFAULT_SSL_CONTEXT 
+            return result
+
+        return wrapper
+

src/python/dm/common/utility/timeUtility.py

+#!/usr/bin/env python
+
+import time
+class TimeUtility:
+
+    @classmethod
+    def getCurrentGMTimeStamp(cls):
+        """ Formats GMT timestamp. """
+        return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time()))
+
+    @classmethod
+    def formatGMTimeStamp(cls, t):
+        """ Format GMT timestamp. """
+        return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(t))
+
+    @classmethod
+    def getCurrentLocalTimeStamp(cls):
+        """ Formats local timestamp. """
+        return time.strftime('%Y/%m/%d %H:%M:%S %Z', time.localtime(time.time()))
+
+    @classmethod
+    def formatLocalTimeStamp(cls, t):
+        """ Formats local timestamp. """
+        return time.strftime('%Y/%m/%d %H:%M:%S %Z', time.localtime(t))
+
+

src/python/dm/common/utility/timedRotatingFileLoggingHandler.py

+#!/usr/bin/env python
+
+#######################################################################
+
+import socket
+import pwd
+import os
+from logging.handlers import TimedRotatingFileHandler
+
+#######################################################################
+
+class TimedRotatingFileLoggingHandler(TimedRotatingFileHandler):
+    """ Class that enables logging into files. """
+    def __init__(self, filename, when='D', interval=1, backupCount=0, encoding=None):
+        TimedRotatingFileHandler.__init__(self, filename, when, interval, backupCount, encoding)
+        self.user = pwd.getpwuid(os.getuid())[0]
+        self.host = socket.gethostname()
+
+    def emit(self, record):
+        record.__dict__['user'] = self.user
+        record.__dict__['host'] = self.host
+        return TimedRotatingFileHandler.emit(self, record)
+                                                                                
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass