src/python/dm/common/service/dmController.py

+#!/usr/bin/env python
+
+#
+# Base DM controller class.
+#
+
+#######################################################################
+
+import cherrypy
+import httplib
+from sys import exc_info
+from dm.common.utility import loggingManager
+from dm.common.constants import dmStatus
+from dm.common.constants import dmHttpHeaders
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions import dmExceptionMap
+from dm.common.exceptions.internalError import InternalError
+
+#######################################################################
+
+class DmController(object):
+    """ Base controller class. """
+    def __init__(self):
+        self._logger = loggingManager.getLogger(self.__class__.__name__)
+
+    def getLogger(self):
+        return self._logger
+
+    @classmethod
+    def addDmResponseHeaders(cls, status=dmStatus.DM_OK, msg='Success', exceptionType=None):
+        cherrypy.response.headers[dmHttpHeaders.DM_STATUS_CODE_HTTP_HEADER] = status
+        cherrypy.response.headers[dmHttpHeaders.DM_STATUS_MESSAGE_HTTP_HEADER] = msg
+        if exceptionType is not None:
+            cherrypy.response.headers[dmHttpHeaders.DM_EXCEPTION_TYPE_HTTP_HEADER] = exceptionType
+
+    @classmethod
+    def addDmSessionRoleHeaders(cls, role):
+        cherrypy.response.headers[dmHttpHeaders.DM_SESSION_ROLE_HTTP_HEADER] = role
+
+    @classmethod
+    def addDmExceptionHeaders(cls, ex):
+        cls.handleException(ex)
+
+    @classmethod
+    def handleCpException(cls):
+        cherrypy.response.status = httplib.OK
+        ex = exc_info()[1]
+        if ex == None:
+            ex = InternalError('Internal Webservice Error')
+        cls.handleException(ex)
+
+    @classmethod
+    def handleException(cls, ex):
+        exClass = ex.__class__.__name__.split('.')[-1]
+        status = None
+        msg = '%s' % ex
+        msg = msg.split('\n')[0]
+        for code in dmExceptionMap.exceptionMap.keys():
+            exStr = dmExceptionMap.exceptionMap.get(code).split('.')[-1]
+            if exStr == exClass:
+                status = code
+        if not status:
+            status = dmStatus.DM_ERROR
+        cls.addDmResponseHeaders(status, msg, exClass)
+
+    def formatJsonResponse(self, response):
+        cherrypy.response.headers['Content-Type'] = 'application/json'
+        return '%s' % (response)
+

src/python/dm/common/service/dmRestWebService.py

+#!/usr/bin/env python
+
+#
+# Base web service class.
+#
+
+####################################################################
+import sys
+import cherrypy
+from cherrypy.process import plugins
+from cherrypy import server
+
+from dm.common.constants import dmStatus
+from dm.common.utility import configurationManager
+from dm.common.utility import loggingManager
+from dm.common.utility import dmModuleManager
+
+####################################################################
+
+class DmRestWebService:
+
+    DEFAULT_N_SERVER_REQUEST_THREADS = 10
+    DEFAULT_SERVER_SOCKET_TIMEOUT = 30
+
+    class SignalHandler:
+        def __init__(self, signal, oldSignalHandler):
+            self._signal = signal
+            self._oldSignalHandler = oldSignalHandler
+            self._logger = loggingManager.getInstance().getLogger(self.__class__.__name__)
+
+        def signalHandler(self):
+            self._logger.debug('%s signal handler called' % self._signal)
+            dmModuleManager.getInstance().stopModules()
+            self._oldSignalHandler()
+
+    def __init__(self, routeMapper):
+        self._configurationManager = configurationManager.getInstance()
+        self._routeMapper = routeMapper
+        self._options = None
+        self._args = None
+        self._logger = loggingManager.getInstance().getLogger(self.__class__.__name__)
+        self._logger.info('Initializing service')
+
+    def prepareOptions(self):
+        from optparse import OptionParser
+        p = OptionParser()
+        p.add_option('-d', '--daemon', action="store_true",
+            dest='daemonFlag', default=False, 
+            help="Run as a daemon.")
+        p.add_option('-p', '--pidfile',
+            dest='pidfile', default=None,
+            help="Store the process id in the given file.")
+        p.add_option('-P', '--port',
+            dest='serverPort', default=self.getDefaultServerPort(),
+            help="Server port.")
+        p.add_option('-H', '--host',
+            dest='serverHost', default=self.getDefaultServerHost(),
+            help="Server host.")
+        p.add_option('-C', '--ssl-ca-cert', 
+            dest='sslCaCert', default=None,
+            help='SSL CA certificate path (used for client SSL certificate verification). Requires --ssl-key and --ssl-cert.')
+        p.add_option('-c', '--ssl-cert', 
+            dest='sslCert', default=None,
+            help='SSL certificate path. SSL operation requires both --ssl-key and --ssl-cert. Client SSL certificate verification also requires --ssl-ca-cert.')
+        p.add_option('-k', '--ssl-key', 
+            dest='sslKey', default=None, 
+            help='SSL key path. SSL operation requires both --ssl-key and --ssl-cert. Client SSL certificate verification also requires --ssl-ca-cert.')
+        p.add_option('', '--n-server-threads', 
+        dest='nServerThreads', default=DmRestWebService.DEFAULT_N_SERVER_REQUEST_THREADS,
+            help='Number of service request handler threads (defaut: %s).' % DmRestWebService.DEFAULT_N_SERVER_REQUEST_THREADS)
+        return p
+
+    def initDmModules(self):
+        pass 
+
+    def getDefaultServerHost(self):
+        return None
+
+    def getDefaultServerPort(self):
+        return None
+
+    # Instantiate modified signal handler that stops dm modules first, 
+    # and then does the default action.
+    def modifySignalHandlers(self, engine):
+        pluginsSignalHandler = plugins.SignalHandler(engine)
+        handlers = pluginsSignalHandler.handlers
+
+        # Add handler for interrupt
+        handlers['SIGINT'] = engine.exit
+
+        # Modify all signal handlers
+        for signal in handlers.keys():
+            self._logger.debug('Modifying signal: %s' % signal)
+            oldSignalHandler = handlers[signal]
+            self._logger.debug('Old signal handler: %s' % oldSignalHandler)
+            signalHandler = DmRestWebService.SignalHandler(signal, oldSignalHandler)
+            self._logger.debug('Setting signal handler to: %s' % signalHandler.signalHandler)
+            handlers[signal] = signalHandler.signalHandler
+            pluginsSignalHandler.subscribe()
+
+    def initServerLog(self):
+        cherrypyLogLevel = self._configurationManager.getCherrypyLogLevel()
+        cherrypy.log.error_log.setLevel(cherrypyLogLevel)
+        cherrypy.log.error_file = self._configurationManager.getCherrypyLogFile()
+        cherrypy.log.error_log.propagate = False
+        cherrypy.log.access_log.setLevel(cherrypyLogLevel)
+        cherrypy.log.access_file = self._configurationManager.getCherrypyAccessFile()
+        cherrypy.log.access_log.propagate = False
+
+    def updateServerConfig(self):
+        configDict = {
+            'server.socket_host' : self._options.serverHost,
+            'server.socket_port' : int(self._options.serverPort),
+            'server.thread_pool' : int(self._options.nServerThreads),
+            'log.screen' : (self._options.daemonFlag != True),
+        }
+        cherrypy.config.update(configDict)
+
+    def prepareServer(self):
+        self._logger.debug('Preparing service configuration')
+        try:
+            optionParser = self.prepareOptions()
+            (self._options, self._args) = optionParser.parse_args()
+            dispatch = self._routeMapper.setupRoutes()
+            self._logger.debug('Using route dispatch: %s' % dispatch)
+
+            config = {
+                '/' : {
+                    'request.dispatch' : dispatch,
+                },
+            }
+                   
+            # No root controller as we provided our own.
+            cherrypy.tree.mount(root=None, config=config)
+            self.initServerLog()
+            self.updateServerConfig()
+
+            self._logger.info('Using host %s' % self._options.serverHost)
+            self._logger.info('Using port %s' % self._options.serverPort)
+            self._logger.debug('Using %s request handler threads' % self._options.nServerThreads)
+        except Exception, ex:
+            self._logger.exception(ex)
+            sys.exit(dmStatus.DM_ERROR)
+
+    # Run server.
+    def __runServer(self):
+        self._logger.info('Starting service')
+        engine = cherrypy.engine
+
+        # Set up Deamonization
+        if self._options.daemonFlag:
+            plugins.Daemonizer(engine).subscribe()
+        self._logger.debug('Daemon mode: %s' % self._options.daemonFlag)
+
+        if self._options.pidfile != None:
+            plugins.PIDFile(engine, self._options.pidfile).subscribe()
+        self._logger.debug('Using PID file: %s' % self._options.pidfile)
+
+        if self._options.sslCert != None and self._options.sslKey != None:
+            server.ssl_ca_certificate = None
+            if self._options.sslCaCert != None:
+                server.ssl_ca_certificate = self._options.sslCaCert
+                self._configurationManager.setSslCaCertFile(self._options.sslCaCert)
+                self._logger.info('Using SSL CA cert file: %s' % self._options.sslCaCert)
+                server.ssl_certificate = self._options.sslCert
+                self._configurationManager.setSslCertFile(self._options.sslCert)
+                self._logger.info('Using SSL cert file: %s' % self._options.sslCert)
+
+                server.ssl_private_key = self._options.sslKey
+                self._configurationManager.setSslKeyFile(self._options.sslKey)
+                self._logger.info('Using SSL key file: %s' % self._options.sslKey)
+
+        server.ssl_module = 'builtin'
+        #server.ssl_module = 'pyopenssl'
+
+        # Increase timeout to prevent early SSL connection terminations
+        server.socket_timeout = DmRestWebService.DEFAULT_SERVER_SOCKET_TIMEOUT
+        
+        # Setup the signal handler to stop the application while running.
+        if hasattr(engine, 'signal_handler'):
+            engine.signal_handler.subscribe()
+        if hasattr(engine, 'console_control_handler'):
+            engine.console_control_handler.subscribe()
+            self.modifySignalHandlers(engine)
+
+        # Turn off autoreloader.
+        self._logger.debug('Turning off autoreloader')
+        engine.autoreload.unsubscribe()
+
+        # Start the engine.
+        try:
+            self._logger.debug('Starting engine')
+            engine.start()
+
+            # Prepare dm services.
+            # Doing this before engine starts may cause issues with existing timers.
+            self._logger.debug('Starting modules')
+            self.initDmModules()
+            dmModuleManager.getInstance().startModules()
+        except Exception, ex:
+            self._logger.exception('Service exiting: %s' % ex)
+            dmModuleManager.getInstance().stopModules()
+            return dmStatus.DM_ERROR
+        self._logger.info('Service ready')
+        engine.block()
+        dmModuleManager.getInstance().stopModules()
+        self._logger.info('Service done')
+        return dmStatus.DM_OK
+
+    # Run server instance.
+    def run(self):
+        self.prepareOptions()
+        self.prepareServer()
+        sys.exit(self.__runServer())
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/utility/configurationManager.py

+#!/usr/bin/env python
+
+# 
+# Configuration manager singleton.
+#
+
+#######################################################################
+
+import os
+import socket
+import UserDict
+import pwd
+
+from dm.common.constants import dmServiceConstants
+from dm.common.exceptions.configurationError import ConfigurationError
+
+#######################################################################
+
+# Defaults.
+
+DEFAULT_DM_ROOT_DIR = '/opt/dm'
+DEFAULT_DM_CONFIG_FILE = '%s/etc/dm.conf'    # requires root dir
+
+DEFAULT_DM_LOG_FILE = '%s/var/log/dm.log'       # requires root dir
+DEFAULT_DM_LOG_CONFIG_FILE = '%s/etc/dm.log.conf'     # requires root dir
+DEFAULT_DM_CONSOLE_LOG_LEVEL = 'critical'
+DEFAULT_DM_FILE_LOG_LEVEL = 'info'
+#DEFAULT_DM_LOG_RECORD_FORMAT = '%(asctime)s,%(msecs)03d [%(levelname)s] %(module)s:%(lineno)d %(user)s@%(host)s %(name)s (%(process)d): %(message)s'
+#DEFAULT_DM_LOG_RECORD_FORMAT = '%(asctime)s,%(msecs)03d %(levelname)s %(module)s:%(lineno)d %(process)d:  %(message)s'
+DEFAULT_DM_LOG_RECORD_FORMAT = '%(asctime)s,%(msecs)03d %(levelname)s %(process)d:  %(message)s'
+DEFAULT_DM_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
+
+DEFAULT_DM_CHERRYPY_LOG_LEVEL = 'ERROR'
+DEFAULT_DM_CHERRYPY_LOG_FILE = '%s/var/log/cherrypy.error' # requires root dir
+DEFAULT_DM_CHERRYPY_ACCESS_FILE = '%s/var/log/cherrypy.access' # requires root dir
+
+DEFAULT_DM_SERVICE_PORT = 10036           # 100DM
+DEFAULT_DM_SERVICE_HOST = '0.0.0.0'
+DEFAULT_DM_SERVICE_PROTOCOL = dmServiceConstants.DM_SERVICE_PROTOCOL_HTTP
+DEFAULT_DM_ADMIN_USERNAME = ''
+DEFAULT_DM_ADMIN_PASSWORD = ''
+
+DEFAULT_DM_DB = 'postgresql'
+DEFAULT_DM_DB_HOST = 'localhost'
+DEFAULT_DM_DB_PORT = 20036                  # 200DM
+DEFAULT_DM_DB_PASSWORD = ''
+DEFAULT_DM_DB_USER = ''
+DEFAULT_DM_DB_SCHEMA = 'dm'
+DEFAULT_DM_DB_PASSWORD_FILE = '%s/lib/postgresql/etc/db.passwd' # requires root dir
+
+# Session cache file
+DEFAULT_DM_SESSION_CACHE_FILE = None
+
+# Enforce session credentials.
+DEFAULT_DM_REQUIRE_SESSION_CREDENTIALS = False
+
+# SSL variables
+DEFAULT_DM_SSL_CA_CERT_FILE = None
+DEFAULT_DM_SSL_CERT_FILE = None
+DEFAULT_DM_SSL_KEY_FILE = None
+
+# Get singleton instance.
+def getInstance():
+    """ Get configuration manager singleton instance. """
+    from dm.common.utility.configurationManager import ConfigurationManager
+    try:
+        cm = ConfigurationManager()
+    except ConfigurationManager, ex:
+        cm = ex
+    return cm
+
+class ConfigurationManager(UserDict.UserDict):
+    """ 
+    Singleton class used for keeping system configuration data. The class
+    initializes its data using predefined defaults, or from certain 
+    environment variables.
+
+    Usage:
+        from dm.common.utility import configurationManager
+        cm = configurationManager.getInstance()
+        cm.setConsoleLogLevel('info')
+        level = cm.getConsoleLogLevel()
+        cm['myKey'] = 'myValue'
+        value = cm.get('myKey')
+    """
+
+    # Singleton.
+    __instance = None
+
+    def __init__(self):
+        if ConfigurationManager.__instance:
+            raise ConfigurationManager.__instance
+        ConfigurationManager.__instance = self
+        UserDict.UserDict.__init__(self)
+        self['user'] = pwd.getpwuid(os.getuid())[0]
+        self['host'] = socket.gethostname()
+
+        self['defaultRootDir'] = DEFAULT_DM_ROOT_DIR
+        self.__setFromEnvVar('rootDir', 'DM_ROOT_DIR')
+
+        self['defaultConfigFile'] = DEFAULT_DM_CONFIG_FILE % self.getRootDir()
+        self['defaultLogFile'] = DEFAULT_DM_LOG_FILE % self.getRootDir()
+        self['defaultLogConfigFile'] = DEFAULT_DM_LOG_CONFIG_FILE % self.getRootDir()
+
+        self['defaultConsoleLogLevel'] = DEFAULT_DM_CONSOLE_LOG_LEVEL
+        self['defaultFileLogLevel'] = DEFAULT_DM_FILE_LOG_LEVEL
+        self['defaultLogRecordFormat'] = DEFAULT_DM_LOG_RECORD_FORMAT
+        self['defaultLogDateFormat'] = DEFAULT_DM_LOG_DATE_FORMAT
+
+        self['defaultCherrypyLogLevel'] = DEFAULT_DM_CHERRYPY_LOG_LEVEL
+        self['defaultCherrypyLogFile'] = DEFAULT_DM_CHERRYPY_LOG_FILE % self.getRootDir()
+        self['defaultCherrypyAccessFile'] = DEFAULT_DM_CHERRYPY_ACCESS_FILE % self.getRootDir()
+
+        self['defaultServicePort'] = DEFAULT_DM_SERVICE_PORT
+        self['defaultServiceHost'] = DEFAULT_DM_SERVICE_HOST
+        self['defaultServiceProtocol'] = DEFAULT_DM_SERVICE_PROTOCOL
+        self['defaultAdminUsername'] = DEFAULT_DM_ADMIN_USERNAME  
+        self['defaultAdminPassword'] = DEFAULT_DM_ADMIN_PASSWORD
+        self['defaultDb'] = DEFAULT_DM_DB
+        self['defaultDbHost'] = DEFAULT_DM_DB_HOST
+        self['defaultDbPort'] = DEFAULT_DM_DB_PORT
+        self['defaultDbPassword'] = DEFAULT_DM_DB_PASSWORD
+        self['defaultDbPasswordFile'] = DEFAULT_DM_DB_PASSWORD_FILE % self.getRootDir()
+        self['defaultDbUser'] = DEFAULT_DM_DB_USER
+        self['defaultDbSchema'] = DEFAULT_DM_DB_SCHEMA
+
+        self['defaultSessionCacheFile'] = DEFAULT_DM_SESSION_CACHE_FILE
+        self['defaultRequireSessionCredentials'] = DEFAULT_DM_REQUIRE_SESSION_CREDENTIALS
+
+        self['defaultSslCaCertFile'] = DEFAULT_DM_SSL_CA_CERT_FILE
+        self['defaultSslCertFile'] = DEFAULT_DM_SSL_CERT_FILE
+        self['defaultSslKeyFile'] = DEFAULT_DM_SSL_KEY_FILE
+                                                                                        # Settings that might come from environment variables.
+        self.__setFromEnvVar('logFile', 'DM_LOG_FILE')
+        self.__setFromEnvVar('logConfigFile', 'DM_LOG_CONFIG_FILE')
+        self.__setFromEnvVar('consoleLogLevel', 'DM_CONSOLE_LOG_LEVEL')
+        self.__setFromEnvVar('fileLogLevel', 'DM_FILE_LOG_LEVEL')
+        self.__setFromEnvVar('logRecordFormat', 'DM_LOG_RECORD_FORMAT')
+        self.__setFromEnvVar('logDateFormat', 'DM_LOG_DATE_FORMAT')
+
+        self.__setFromEnvVar('cherrypyLogLevel', 'DM_CHERRYPY_LOG_LEVEL')
+        self.__setFromEnvVar('cherrypyLogFile', 'DM_CHERRYPY_LOG_FILE')
+        self.__setFromEnvVar('cherrypyAccessFile', 'DM_CHERRYPY_ACCESS_FILE')
+
+        self.__setFromEnvVar('serviceProtocol', 'DM_SERVICE_PROTOCOL')
+        self.__setFromEnvVar('serviceHost', 'DM_SERVICE_HOST')
+        self.__setFromEnvVar('servicePort', 'DM_SERVICE_PORT')
+        self.__setFromEnvVar('adminUsername', 'DM_ADMIN_USERNAME')
+        self.__setFromEnvVar('adminPassword', 'DM_ADMIN_PASSWORD')
+
+        self.__setFromEnvVar('sessionCacheFile', 'DM_SESSION_CACHE_FILE')
+
+        self.__setFromEnvVar('sslCaCertFile', 'DM_SSL_CA_CERT_FILE')
+        self.__setFromEnvVar('sslCertFile', 'DM_SSL_CERT_FILE')
+        self.__setFromEnvVar('sslKeyFile', 'DM_SSL_KEY_FILE')
+
+        self.__setFromEnvVar('configFile', 'DM_CONFIG_FILE')
+        self.__setFromEnvVar('dbPasswordFile', 'DM_DB_PASSWORD_FILE')
+
+        # Settings that might come from file.
+        self.__setFromVarFile('dbPassword', self.getDbPasswordFile())
+
+    # This function will ignore errors if environment variable is not set.
+    def __setFromEnvVar(self, key, envVar):
+        """ 
+        Set value for the specified key from a given environment variable.
+        This function ignores errors for env. variables that are not set.
+        """
+        try:
+            self[key] = os.environ[envVar]
+        except:
+            pass
+
+    # This function will ignore errors if variable file is not present.
+    def __setFromVarFile(self, key, varFile):
+        """ 
+        Set value for the specified key from a given file. The first line  
+        in the file is variable value.
+        This function ignores errors.
+        """
+        try:
+            v = open(varFile, 'r').readline()
+            self[key] = v.lstrip().rstrip()
+        except Exception, ex:
+            pass
+
+    def __getKeyValue(self, key, default='__internal__'):
+        """
+        Get value for a given key.
+        Keys will be of the form 'logFile', and the default keys have
+        the form 'defaultLogFile'.
+        """
+        defaultKey = "default" + key[0].upper() + key[1:]
+        defaultValue = self.get(defaultKey, None)
+        if default != '__internal__':
+            defaultValue = default
+        return self.get(key, defaultValue)
+
+    def getHost(self):
+        return self['host']
+    
+    def getUser(self):
+        return self['user']
+
+    def getDefaultRootDir(self):
+        return self['defaultRootDir']
+
+    def setRootDir(self, rootDir):
+        self['rootDir'] = rootDir
+
+    def getRootDir(self, default='__internal__'):
+        return self.__getKeyValue('rootDir', default)
+
+    def getDefaultLogFile(self):
+        return self['defaultLogFile']
+
+    def setLogFile(self, logFile):
+        self['logFile'] = logFile
+
+    def getLogFile(self, default='__internal__'):
+        return self.__getKeyValue('logFile', default)
+
+    def hasLogFile(self):
+        return self.has_key('logFile')
+
+    def getDefaultLogConfigFile(self):
+        return self['defaultLogConfigFile']
+
+    def setLogConfigFile(self, logConfigFile):
+        self['logConfigFile'] = logConfigFile
+
+    def getLogConfigFile(self, default='__internal__'):
+        return self.__getKeyValue('logConfigFile', default)
+
+    def hasLogConfigFile(self):
+        return self.has_key('logConfigFile')
+
+    def getDefaultConsoleLogLevel(self):
+        return self['defaultConsoleLogLevel']
+
+    def setConsoleLogLevel(self, level):
+        self['consoleLogLevel'] = level 
+
+    def getConsoleLogLevel(self, default='__internal__'):
+        return self.__getKeyValue('consoleLogLevel', default)
+
+    def hasConsoleLogLevel(self):
+        return self.has_key('consoleLogLevel')
+
+    def getDefaultFileLogLevel(self):
+        return self['defaultFileLogLevel']
+
+    def setFileLogLevel(self, level):
+        self['fileLogLevel'] = level 
+
+    def getFileLogLevel(self, default='__internal__'):
+        return self.__getKeyValue('fileLogLevel', default)
+
+    def hasFileLogLevel(self):
+        return self.has_key('fileLogLevel')
+
+    def getDefaultLogRecordFormat(self):
+        return self['defaultLogRecordFormat']
+
+    def setLogRecordFormat(self, format):
+        self['logRecordFormat'] = format
+
+    def getLogRecordFormat(self, default='__internal__'):
+        return self.__getKeyValue('logRecordFormat', default)
+
+    def hasLogRecordFormat(self):
+        return self.has_key('logRecordFormat')
+
+    def getDefaultLogDateFormat(self):
+        return self['defaultLogDateFormat']
+
+    def setLogDateFormat(self, format):
+        self['logDateFormat'] = format
+
+    def getLogDateFormat(self, default='__internal__'):
+        return self.__getKeyValue('logDateFormat', default)
+
+    def hasLogDateFormat(self):
+        return self.has_key('logDateFormat')
+
+    def getDefaultCherrypyLogLevel(self):
+        return self['defaultCherrypyLogLevel']
+
+    def setCherrypyLogLevel(self, level):
+        self['cherrypyLogLevel'] = level 
+
+    def getCherrypyLogLevel(self, default='__internal__'):
+        return self.__getKeyValue('cherrypyLogLevel', default)
+
+    def hasCherrypyLogLevel(self):
+        return self.has_key('cherrypyLogLevel')
+
+    def getDefaultCherrypyLogCherrypy(self):
+        return self['defaultCherrypyLogFile']
+
+    def setCherrypyLogFile(self, cherrypyLogFile):
+        self['cherrypyLogFile'] = cherrypyLogFile
+
+    def getCherrypyLogFile(self, default='__internal__'):
+        return self.__getKeyValue('cherrypyLogFile', default)
+
+    def hasCherrypyLogFile(self):
+        return self.has_key('cherrypyLogFile')
+
+    def getDefaultCherrypyAccessFile(self):
+        return self['defaultCherrypyAccessFile']
+
+    def setCherrypyAccessFile(self, cherrypyAccessFile):
+        self['cherrypyAccessFile'] = cherrypyAccessFile
+
+    def getCherrypyAccessFile(self, default='__internal__'):
+        return self.__getKeyValue('cherrypyAccessFile', default)
+
+    def hasCherrypyAccessFile(self):
+        return self.has_key('cherrypyAccessFile')
+
+    def isDbAvailable(self):
+        if os.access(self.getDbPasswordFile(), os.R_OK):
+            return True
+        return False
+
+    def getDefaultServiceProtocol(self):
+        return self['defaultServiceProtocol']
+
+    def setServiceProtocol(self, serviceProtocol):
+        self['serviceProtocol'] = serviceProtocol
+
+    def getServiceProtocol(self, default='__internal__'):
+        return self.__getKeyValue('serviceProtocol', default)
+
+    def hasServiceProtocol(self):
+        return self.has_key('serviceProtocol')
+
+    def getDefaultServicePort(self):
+        return self['defaultServicePort']
+
+    def setServicePort(self, servicePort):
+        self['servicePort'] = servicePort 
+
+    def getServicePort(self, default='__internal__'):
+        return int(self.__getKeyValue('servicePort', default))
+
+    def hasServicePort(self):
+        return self.has_key('servicePort')
+
+    def getDefaultServiceHost(self):
+        return self['defaultServiceHost']
+
+    def setServiceHost(self, serviceHost):
+        self['serviceHost'] = serviceHost
+
+    def getServiceHost(self, default='__internal__'):
+        return self.__getKeyValue('serviceHost', default)
+
+    def hasServiceHost(self):
+        return self.has_key('serviceHost')
+
+    def getDefaultAdminUsername(self):
+        return self['defaultAdminUsername']
+
+    def setAdminUsername(self, adminUsername):
+        self['adminUsername'] = adminUsername 
+
+    def getAdminUsername(self, default='__internal__'):
+        return self.__getKeyValue('adminUsername', default)
+
+    def hasAdminUsername(self):
+        return self.has_key('adminUsername')
+
+    def getDefaultAdminPassword(self):
+        return self['defaultAdminPassword']
+
+    def setAdminPassword(self, adminPassword):
+        self['adminPassword'] = adminPassword 
+
+    def getAdminPassword(self, default='__internal__'):
+        return self.__getKeyValue('adminPassword', default)
+
+    def hasAdminPassword(self):
+        return self.has_key('adminPassword')
+
+    def getDefaultDb(self):
+        return self['defaultDb']
+
+    def setDb(self, db):
+        self['db'] = db
+
+    def getDb(self, default='__internal__'):
+        return self.__getKeyValue('db', default) 
+
+    def hasDb(self):
+        return self.has_key('db')
+
+    def getDefaultDbHost(self):
+        return self['defaultDbHost']
+
+    def setDbHost(self, dbHost):
+        self['dbHost'] = dbHost
+
+    def getDbHost(self, default='__internal__'):
+        return self.__getKeyValue('dbHost', default) 
+
+    def hasDbHost(self):
+        return self.has_key('dbHost')
+
+    def getDefaultDbPort(self):
+        return self['defaultDbPort']
+
+    def setDbPort(self, dbPort):
+        self['dbPort'] = dbPort
+
+    def getDbPort(self, default='__internal__'):
+        return self.__getKeyValue('dbPort', default) 
+
+    def hasDbPort(self):
+        return self.has_key('dbPort')
+
+    def getDefaultDbPassword(self):
+        return self['defaultDbPassword']
+
+    def setDbPassword(self, dbPassword):
+        self['dbPassword'] = dbPassword
+
+    def getDbPassword(self, default='__internal__'):
+        return self.__getKeyValue('dbPassword', default) 
+
+    def hasDbPassword(self):
+        return self.has_key('dbPassword')
+
+    def getDefaultDbPasswordFile(self):
+        return self['defaultDbPasswordFile']
+
+    def getDbPasswordFile(self, default='__internal__'):
+        return self.__getKeyValue('dbPasswordFile', default) 
+
+    def setDbPasswordFile(self, f):
+        self['dbPasswordFile'] = f
+
+    def hasDbPasswordFile(self):
+        return self.has_key('dbPasswordFile')
+
+    def getDefaultDbUser(self):
+        return self['defaultDbUser']
+
+    def getDbUser(self, default='__internal__'):
+        return self.__getKeyValue('dbUser', default) 
+
+    def setDbUser(self, u):
+        self['dbUser'] = u
+
+    def hasDbUser(self):
+        return self.has_key('dbUser')
+
+    def getDbSchema(self):
+        return self['dbSchema']
+
+    def getDefaultConfigFile(self):
+        return self['defaultConfigFile']
+
+    def setConfigFile(self, configFile):
+        self['configFile'] = configFile
+
+    def getConfigFile(self, default='__internal__'):
+        return self.__getKeyValue('configFile', default)
+
+    def hasConfigFile(self):
+        return self.has_key('configFile')
+
+    def getDefaultSessionCacheFile(self):
+        return self['defaultSessionCacheFile']
+
+    def setSessionCacheFile(self, sessionCacheFile):
+        self['sessionCacheFile'] = sessionCacheFile
+
+    def getSessionCacheFile(self, default='__internal__'):
+        return self.__getKeyValue('sessionCacheFile', default)
+
+    def hasSessionCacheFile(self):
+        return self.has_key('sessionCacheFile')
+
+    def getDefaultRequireSessionCredentials(self):
+        return self['defaultRequireSessionCredentials']
+
+    def setRequireSessionCredentials(self, requireSessionCredentials):
+        self['requireSessionCredentials'] = requireSessionCredentials
+
+    def getRequireSessionCredentials(self, default='__internal__'):
+        return self.__getKeyValue('requireSessionCredentials', default)
+
+    def hasRequireSessionCredentials(self):
+        return self.has_key('requireSessionCredentials')
+
+    def getDefaultSslCaCertFile(self):
+        return self['defaultSslCaCertFile']
+
+    def setSslCaCertFile(self, sslCaCertFile):
+        self['sslCaCertFile'] = sslCaCertFile
+
+    def getSslCaCertFile(self, default='__internal__'):
+        return self.__getKeyValue('sslCaCertFile', default)
+
+    def hasSslCaCertFile(self):
+        return self.has_key('sslCaCertFile')
+
+    def getDefaultSslCertFile(self):
+        return self['defaultSslCertFile']
+
+    def setSslCertFile(self, sslCertFile):
+        self['sslCertFile'] = sslCertFile
+
+    def getSslCertFile(self, default='__internal__'):
+        return self.__getKeyValue('sslCertFile', default)
+
+    def hasSslCertFile(self):
+        return self.has_key('sslCertFile')
+
+    def getDefaultSslKeyFile(self):
+        return self['defaultSslKeyFile']
+
+    def setSslKeyFile(self, sslKeyFile):
+        self['sslKeyFile'] = sslKeyFile
+
+    def getSslKeyFile(self, default='__internal__'):
+        return self.__getKeyValue('sslKeyFile', default)
+
+    def hasSslKeyFile(self):
+        return self.has_key('sslKeyFile')
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    cm = getInstance()
+    print cm

src/python/dm/common/utility/consoleLoggingHandler.py

+#!/usr/bin/env python
+
+#
+# Console logging handler class
+#
+
+#######################################################################
+
+import socket
+import pwd
+import os
+from logging import StreamHandler
+
+#######################################################################
+
+class ConsoleLoggingHandler(StreamHandler):
+    """ Class that enables console logging. """
+    def __init__(self, *args):
+        StreamHandler.__init__(self, *args)
+        self._user = pwd.getpwuid(os.getuid())[0]
+        self._host = socket.gethostname()
+
+    def emit(self, record):
+        record.__dict__['user'] = self._user
+        record.__dict__['host'] = self._host
+        return StreamHandler.emit(self, record)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    import sys
+    import logging
+    exec 'sh = ConsoleLoggingHandler(sys.stdout,)'
+    sh.setLevel(logging.INFO)
+    rootLogger = logging.getLogger('')
+    logging.basicConfig(level=logging.DEBUG)
+
+    mainLogger = logging.getLogger('main')
+    mainLogger.debug("main debug")
+    mainLogger.info("main info")
+                                                                                                                        

src/python/dm/common/utility/dmModuleManager.py

+#!/usr/bin/env python
+
+#
+# Module manager class.
+#
+
+#######################################################################
+
+import threading
+
+#######################################################################
+
+# Get singleton instance.
+def getInstance():
+    from dm.common.utility.dmModuleManager import DmModuleManager
+    try:
+        mgr = DmModuleManager()
+    except DmModuleManager, ex:
+        mgr = ex
+    return mgr
+
+class DmModuleManager:
+    """ Singleton class used for managing dm modules. """
+
+    # Singleton.
+    __instanceLock = threading.RLock()
+    __instance = None
+
+    def __init__(self):
+        DmModuleManager.__instanceLock.acquire()
+        try:
+            if DmModuleManager.__instance:
+                raise DmModuleManager.__instance
+            DmModuleManager.__instance = self
+            from dm.common.utility import loggingManager
+            self._logger = loggingManager.getLogger(self.__class__.__name__)
+            self._lock = threading.RLock()
+            self._moduleList = []
+            self._modulesRunning = False
+        finally:
+            DmModuleManager.__instanceLock.release()
+
+    def addModule(self, m):
+        self._lock.acquire()
+        try:
+            self._logger.debug('Adding dm module: %s' % m.__class__.__name__)
+            self._moduleList.append(m)
+        finally:
+            self._lock.release()
+
+    def startModules(self):
+        self._lock.acquire()
+        try:
+            if self._modulesRunning:
+                return
+            for m in self._moduleList:
+                self._logger.debug('Starting dm module: %s' % m.__class__.__name__)
+                m.start()
+            self._modulesRunning = True
+        finally:
+            self._lock.release()
+
+    def stopModules(self):
+        self._lock.acquire()
+        try:
+            if not self._modulesRunning:
+                return
+            n = len(self._moduleList)
+            for i in range(0, n):
+                m = self._moduleList[n-1-i]
+                self._logger.debug('Stopping dm module: %s' % m.__class__.__name__)
+                m.stop()
+            self._modulesRunning = False
+        finally:
+            self._lock.release()
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/utility/dmSubprocess.py

+#!/usr/bin/env python
+
+# 
+# Subprocess class
+#
+
+#######################################################################
+
+import os
+import subprocess
+import platform
+
+from dm.common.utility import loggingManager
+from dm.common.exceptions.commandFailed import CommandFailed
+
+#######################################################################
+
+class DmSubprocess(subprocess.Popen):
+
+    def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=None, close_fds=False, shell=True, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, useExceptions=True, quietMode=False):
+        """ Overrides Popen constructor with defaults more appropriate DM. """
+        subprocess.Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags)
+        self._logger = loggingManager.getLogger(self.__class__.__name__)
+        self._stdout = None
+        self._stderr = None
+        self._args = args
+        self._useExceptions = useExceptions
+        self._quietMode = quietMode
+
+    def _commandLog(self):
+        # Not very useful to show the name of this file.
+        # Walk up the stack to find the caller.
+        import traceback
+        stack =  traceback.extract_stack()
+        for i in range(2, len(stack)):
+            if stack[-i][0] != stack[-1][0]:
+                fileName, lineNumber, functionName, text = stack[-i]
+                break
+            else:
+                fileName = lineNumber = functionName = text = '?'
+
+        self._logger.debug('From [%s:%s] Invoking: [%s]' % (os.path.basename(fileName), lineNumber, self._args))
+
+    def run(self, input=None):
+        """ Run subprocess. """
+        if not self._quietMode:
+            self._commandLog()
+        (self._stdout, self._stderr) = subprocess.Popen.communicate(self, input)
+        if not self._quietMode:
+            self._logger.debug('Exit status: %s' % self.returncode)
+        if self.returncode != 0 and self._useExceptions:
+            if not self._quietMode:
+                self._logger.debug('StdOut: %s' % self._stdout)
+                self._logger.debug('StdErr: %s' % self._stderr)
+            error = self._stderr.strip()
+            if error == '':
+                error = self._stdout.strip()
+            raise CommandFailed('%s' % (error))
+        return (self._stdout, self._stderr) 
+
+    def getLogger(self):
+        return self._logger
+
+    def getArgs(self):
+        return self._args
+
+    def getStdOut(self):
+        return self._stdout
+
+    def getStdErr(self):
+        return self._stderr
+
+    def getExitStatus(self):
+        return self.returncode
+
+# Convenience function for getting subprocess.
+def getSubprocess(command):
+    if platform.system() != 'Windows':
+        close_fds = True
+    else:
+        close_fds = False
+    p = DmSubprocess(command, close_fds=close_fds)
+    return p
+
+# Convenience function for executing command.
+def executeCommand(command):
+    """ Create subprocess and run it, return subprocess object. """
+    p = getSubprocess(command)
+    p.run()
+    return p
+
+# Convenience function for executing command that may fail, and we do not
+# care about the failure.
+def executeCommandAndIgnoreFailure(command):
+    """ Create subprocess, run it, igore any failures, and return subprocess object. """
+    p = getSubprocess(command)
+    try:
+        p.run()
+    except CommandFailed, ex:
+        p.getLogger().debug('Command failed, stdout: %s, stderr: %s' % (p.getStdOut(), p.getStdErr()))
+    return p
+
+def executeCommandAndLogToStdOut(command):
+    """ Execute command, display output to stdout, maintain log file and return subprocess object. """
+    p = getSubprocess(command)
+    p._commandLog()
+
+    while True:
+        outp = p.stdout.readline()
+        if not outp:
+            break
+        print outp,
+
+    retval = p.wait()
+
+    p._logger.debug('Exit status: %s' % retval)
+
+    if retval != 0:
+        error = ''
+        while True:
+            err = p.stderr.readline()
+            if not err:
+                break
+            error += err
+        raise CommandFailed(error)
+    return p
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    p = DmSubprocess('ls -l', useExceptions=False)
+    p.run()
+    print p.getStdOut()
+    print p.getStdErr()
+    print p.getExitStatus()
+

src/python/dm/common/utility/loggingManager.py

+#!/usr/bin/env python
+
+#
+# Logging manager singleton.
+#
+
+#######################################################################
+
+import re
+import sys
+import os.path
+import logging
+import ConfigParser
+from dm.common.utility import configurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+
+#######################################################################
+
+# Get singleton instance.
+def getInstance():
+    from dm.common.utility.loggingManager import LoggingManager
+    try:
+        lm = LoggingManager()
+    except LoggingManager, ex:
+        lm = ex
+    return lm
+
+def getLogger(name='defaultLogger'):
+    """ Convenience function to obtain logger. """
+    return getInstance().getLogger(name)
+
+def setConsoleLogLevel(level):
+    """ Convenience function to set the console log level. """
+    return getInstance().setConsoleLogLevel(level)
+
+def setFileLogLevel(level):
+    """ Convenience function to set the file log level. """
+    return getInstance().setFileLogLevel(level)
+
+class LoggingManager:
+    """ 
+    Configuration:
+                                                                                    The log manager class is initialized via a configuration file 
+    that may have the following sections:
+    ConsoleLogging      # Used for output on the screen
+    FileLogging         # Used for logging into a file
+
+    Each section in the configuration file should have the following 
+    keys:
+        handler     # Indicates which handler class to use 
+        level       # Indicates logging level
+        format      # Indicates format for log messages
+        dateformat  # Indicates date format used for log messages
+
+    Given below is an example of a valid configuration file:
+
+    [ConsoleLogging]
+        handler=ConsoleLoggingHandler(sys.stdout,)
+        level=info
+        format=[%(levelname)s] %(message)s 
+        dateformat=%m/%d/%y %H:%M:%S
+                                                                                    [FileLogging]
+        handler=TimedRotatingFileLoggingHandler('/tmp/dm.log')
+        level=debug
+        format=%(asctime)s,%(msecs)d [%(levelname)s] %(module)s:%(lineno)d %(user)s@%(host)s %(name)s (%(process)d): %(message)s
+        dateformat=%m/%d/%y %H:%M:%S
+    """
+
+    # Singleton.
+    __instance = None
+
+    def __init__(self):
+        if LoggingManager.__instance:
+             raise LoggingManager.__instance
+        LoggingManager.__instance = self
+        self._consoleHandler = None
+        self._fileHandlerList = []
+        self._maxIntLevel = logging.CRITICAL
+        self._minIntLevel = logging.NOTSET
+        self._levelRegExList = []
+        self._logger = logging.getLogger(self.__class__.__name__)
+        self._initFlag = False
+
+    def setMinLogLevel(self, minLogLevel=logging.INFO):
+        self._minIntLevel = minLogLevel
+
+    def parseLevelRegEx(self, levelRegExList):
+        """ Parse a list of expressions of the form <regex>=<log-level>. """
+        lines = levelRegExList.split('\n')
+        for line in lines:
+            try:
+                 # Use the right split so we can have '='s in the regex
+                 (regex, level) = line.rsplit('=', 1)
+                 pattern = re.compile(regex)
+                 tuple = (pattern, logging.getLevelName(level.upper()))
+                 self._levelRegExList.append(tuple)
+            except Exception, ex:
+                 # Do not fail, but log an error.
+                 self._logger.error('Parser error in log configuration file: %s' % line)
+                 self._logger.exception(ex)
+
+    # Get Log Level based on a string representation
+    def getIntLogLevel(self, levelStr):
+        level = logging.getLevelName(levelStr)
+        # Level should be an integer
+        try:
+            return int(level) 
+        except ValueError, ex:
+            raise ConfigurationError('"%s" is not valid log level' % levelStr)
+
+    # Configure log handlers.
+    def configureHandlers(self):
+        """ Configure log handlers from the config file. """
+        cm = configurationManager.getInstance()
+        configFile = cm.getLogConfigFile()
+        configSections = self.__getConfigSections(configFile)
+
+        # Console handler.
+        defaults = { 
+            'level' : cm.getConsoleLogLevel(),
+            'format' : cm.getLogRecordFormat(),
+            'dateformat' : cm.getLogDateFormat(),
+            'handler' : 'ConsoleLoggingHandler(sys.stdout,)'
+        }
+        consoleHandler = self.__configureHandler(configFile, 'ConsoleLogging', defaults)
+
+        if consoleHandler != None:
+            self._consoleHandler = consoleHandler
+
+        # File logging. # Do not configure if log directory does
+        # not exist.
+        defaults['handler'] = None
+        defaults['level'] = cm.getFileLogLevel() 
+        if not os.path.exists(configFile):
+            # No config file, we'll configure default.
+            defaultLogFile = cm.getLogFile()
+            defaultLogDir = os.path.dirname(defaultLogFile)
+            if os.path.exists(defaultLogDir):
+                handler = 'TimedRotatingFileLoggingHandler("%s")' % defaultLogFile 
+                defaults['handler'] = handler
+                fileHandler = self.__configureHandler(configFile, 'FileLogging', defaults)
+                if fileHandler != None:
+                    self._fileHandlerList.append(fileHandler)
+
+        else:
+            # Parse all file loggers present in the config file
+            for configSection in configSections:
+                if configSection.startswith('FileLogging'):
+                     fileHandler = self.__configureHandler(configFile, configSection, defaults)
+                     if fileHandler != None:
+                         self._fileHandlerList.append(fileHandler)
+
+        # Add handlers to the root logger.  Use logging class here
+        # to make sure we can have a logger when we parse the 
+        # logger expressions
+        rootLogger = logging.getLogger('')
+        for handler in [self._consoleHandler] + self._fileHandlerList:
+            rootLogger.addHandler(handler)
+
+        # Get a logger factory based on our current config 
+        self.configureLoggers(configFile, defaultLevel=cm.getFileLogLevel())
+
+    def configureLoggers(self, configFile, defaultLevel='error'):
+        configParser = ConfigParser.ConfigParser()
+        configParser.read(configFile)
+
+        rootLogLevel = 'error'
+        levelRegEx = '^.*$=%s' % (defaultLevel)
+        if configParser.has_section('LoggerLevels'):
+            rootLogLevel = configParser.get('LoggerLevels', 'root', rootLogLevel)
+            levelRegEx = configParser.get('LoggerLevels', 'levelregex', levelRegEx)
+
+        rootLevelInt = logging.getLevelName(rootLogLevel.upper())
+        logging.getLogger('').root.setLevel(rootLevelInt)
+        logging.getLogger('').debug('Set root logger to %s' % rootLevelInt)
+
+        if not levelRegEx:
+            return
+
+        # Parse expressions of the form <regex>=<log-level>. """
+        lines = levelRegEx.split('\n')
+        for line in lines:
+            try:
+                # Use the right split so we can have '='s in the regex
+                (regex, level) = line.rsplit('=', 1)
+                pattern = re.compile(regex)
+                tuple = (pattern, logging.getLevelName(level.upper()))
+                self._levelRegExList.append(tuple)
+            except Exception, ex:
+                # Do not fail
+                self._logger.error('Parser error in log configuration file: %s' % line)
+                self._logger.exception(ex)
+
+    def __getOptionFromConfigFile(self, configParser, configSection, key, defaultValue=None):
+        """ Get specified option from the configuration file. """
+        if configParser.has_section(configSection):
+            return configParser.get(configSection, key, True)
+        else:
+            return defaultValue
+
+    # Get the sections in the config file
+    def __getConfigSections(self, configFile):
+        """ Return a list of the sections in the given config file """
+        configParser = ConfigParser.RawConfigParser()
+        configParser.read(configFile)
+        return configParser.sections()
+
+    # Configure particular handler with given defaults.
+    def __configureHandler(self, configFile, configSection, defaults):
+        """ Configure specified handler with a given defaults. """
+        configParser = ConfigParser.ConfigParser(defaults)
+        configParser.read(configFile)
+        handlerOption = defaults['handler']
+        try:
+            handlerOption = configParser.get(configSection, 'handler', True)
+        except Exception, ex:
+            pass
+
+        # If handlerOption is empty, handler cannot be instantiated.
+        handler = None
+        if handlerOption != None:
+            # Handler argument format: MyHandler(arg1, arg2, ...)
+            # Module will be in lowercase letters, but the class
+            # should be capitalized.
+            handlerName = re.sub('\(.*', '', handlerOption)
+            moduleName = handlerName[0].lower() + handlerName[1:]
+            try:
+                exec 'from dm.common.utility import %s' % (moduleName)
+                exec '_handler = %s.%s' % (moduleName, handlerOption)
+            except IOError, ex:
+                errno, _emsg = ex
+                import errno
+
+                # If the exception raised is an I/O permissions error, ignore
+                # it and disable this log handler.  This allows non-root users
+                # to use the (system-wide) default log configuration
+                if _errno != errno.EACCES:
+                    raise
+                _handler = None 
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+        # Only request setting from the config file if it was
+        # not set via environment variable, or programmatically.
+        if _handler != None:
+            try:
+                _level = self.__getOptionFromConfigFile(configParser,
+                configSection, 'level', defaults['level'])
+                intLevel = self.getIntLogLevel(_level.upper())
+                _handler.setLevel(intLevel)
+
+                _format = self.__getOptionFromConfigFile(configParser, configSection, 'format', defaults['format'])
+                _dateformat = self.__getOptionFromConfigFile(configParser, configSection, 'dateformat', defaults['dateformat'])
+
+                _handler.setFormatter(logging.Formatter(_format, _dateformat))
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+            # Look to see if there is a filter to apply to the handler
+            filter = None
+            try:
+                filter = configParser.get(configSection, 'filter')
+            except Exception, ex:
+                pass
+
+            if filter:
+                _handler.addFilter(logging.Filter(filter))
+        return _handler
+
+    def getLogger(self, name='defaultLogger'):
+        if not self._initFlag:
+            self._initFlag = True
+            self.configureHandlers()
+        logger = logging.getLogger(name)
+        logger.setLevel(self.getLevel(name))
+        return logger
+
+    def getLevel(self, name):
+        # Match from the known regex list.
+        level = logging.NOTSET
+
+        # The last regex is most important.
+        for e in reversed(self._levelRegExList):
+            (pattern, level) = e
+
+            # If we return not None it is a match
+            if not None == pattern.match(name):
+                break
+
+        if level > self._maxIntLevel:
+            level = self._maxIntLevel
+        if level < self._minIntLevel:
+            level = self._minIntLevel
+        return level
+
+    def setConsoleLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            self._consoleHandler.setLevel(intLevel)
+            self._maxIntLevel = intLevel
+            self._logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+    def setFileLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            for handler in self._fileHandlerList:
+                handler.setLevel(intLevel)
+            self._maxIntLevel = intLevel
+            self._logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    lm = getInstance()
+    logger = lm.getLogger('Main')
+    logger.info('Info In Main')
+    logger = lm.getLogger('Main')
+    logger.info('Info In Main 2')
+    logger = lm.getLogger('')
+    logger.info('Info using root logger')
+    logger = lm.getLogger('Main.2')
+    logger.info('Info in Main.2')
+    logger.debug('You should not see this message')
+    lm.setConsoleLogLevel('debug')
+    logger.debug('Debug in Main.2')

src/python/dm/common/utility/timedRotatingFileLoggingHandler.py

+#!/usr/bin/env python
+
+#######################################################################
+
+import socket
+import pwd
+import os
+from logging.handlers import TimedRotatingFileHandler
+
+#######################################################################
+
+class TimedRotatingFileLoggingHandler(TimedRotatingFileHandler):
+    """ Class that enables logging into files. """
+    def __init__(self, filename, when='D', interval=1, backupCount=0, encoding=None):
+        TimedRotatingFileHandler.__init__(self, filename, when, interval, backupCount, encoding)
+        self._user = pwd.getpwuid(os.getuid())[0]
+        self._host = socket.gethostname()
+
+    def emit(self, record):
+        record.__dict__['user'] = self._user
+        record.__dict__['host'] = self._host
+        return TimedRotatingFileHandler.emit(self, record)
+                                                                                
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass

src/python/dm/fs_service/impl/fsControllerImpl.py

+#!/usr/bin/env python
+
+#
+# Implementation for file system controller.
+#
+
+#######################################################################
+
+import threading
+
+from dm.common.objects.dmObject import DmObject
+from dm.common.objects.dmObjectManager import DmObjectManager
+from dm.common.utility.dmSubprocess import DmSubprocess
+
+#######################################################################
+
+class FsControllerImpl(DmObjectManager):
+    """ FS controller implementation class. """
+
+    def __init__(self):
+        DmObjectManager.__init__(self)
+
+    def getDirectoryList(self, path):
+        p = DmSubprocess('ls -l %s' % path)
+        p.run()
+        return DmObject({'path' : path, 'directoryList' : p.getStdOut()})

src/python/dm/fs_service/service/fsController.py

+#!/usr/bin/env python
+
+#######################################################################
+
+import cherrypy
+
+from dm.common.service.dmController import DmController
+from dm.common.objects.dmObject import DmObject
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.internalError import InternalError
+from dm.common.exceptions.invalidRequest import InvalidRequest
+
+from dm.fs_service.impl.fsControllerImpl import FsControllerImpl
+
+#######################################################################
+
+class FsController(DmController):
+
+    def __init__(self):
+        DmController.__init__(self)
+        self._fsControllerImpl = FsControllerImpl()
+
+    @cherrypy.expose
+    def getDirectoryList(self, **kwargs):
+        try:
+            if not kwargs.has_key('path'):
+                raise InvalidRequest('Missing directory path.')
+            path = kwargs.get('path')
+            response = '%s' % self._fsControllerImpl.getDirectoryList(path).getJsonRep()
+            self.getLogger().debug('Returning: %s' % response)
+        except DmException, ex:
+            self.getLogger().error('%s' % ex)
+            self.handleException(ex)
+            response = ex.getJsonRep()
+        except Exception, ex:
+            self.getLogger().error('%s' % ex)
+            self.handleException(ex)
+            response = InternalError(ex).getJsonRep()
+        return self.formatJsonResponse(response)
+

src/python/dm/fs_service/service/fsService.py

+#!/usr/bin/env python
+
+#
+# File system controller service
+#
+
+####################################################################
+
+from dm.common.service.dmRestWebService import DmRestWebService
+from dm.common.utility import dmModuleManager
+from dm.fs_service.service import fsServiceRouteMapper
+
+####################################################################
+
+class FsService(DmRestWebService):
+ 
+    def __init__(self):
+        DmRestWebService.__init__(self, fsServiceRouteMapper)
+
+    def initDmModules(self):
+        self._logger.debug('Initializing dm modules')
+
+        # Add modules that will be started.
+        moduleManager = dmModuleManager.getInstance()
+        self._logger.debug('Initialized dm modules')
+
+####################################################################
+# Run service
+
+if __name__ == '__main__':
+    service = FsService();
+    service.run()

src/python/dm/fs_service/service/fsServiceRouteMapper.py

+#!/usr/bin/env python
+
+#
+# Route mapper for file system service.
+#
+
+#######################################################################
+
+import sys
+import os
+
+import cherrypy
+from dm.common.utility import loggingManager
+from dm.fs_service.service.fsController import FsController
+
+#######################################################################
+
+def setupRoutes():
+    """ Setup RESTFul routes. """
+
+    # Static instances shared between different routes
+    fsController = FsController()
+
+    # Define routes.
+    # Make sure to have leading '/' for consistency.
+    routes = [
+        # ('GET') routes do not require authorization.
+        # ('PUT', 'POST', 'DELETE') routes require authorization.
+
+        #
+        # FS Controller routes
+        #
+
+        # Get directory listing
+        {
+            'name' : 'getDirectoryList',
+            'path' : '/directory/list',
+            'controller' : fsController,
+            'action' : 'getDirectoryList', 
+            'method' : ['GET']
+        },
+    ]
+       
+    # Add routes to dispatcher. 
+    d = cherrypy.dispatch.RoutesDispatcher()
+    logger = loggingManager.getLogger('setupRoutes')
+    for route in routes:
+        logger.debug('Connecting route: %s' % route)
+        d.connect(route['name'], route['path'], action=route['action'], controller=route['controller'], conditions=dict(method=route['method']))
+    return d
+

tools/make/RULES_DM

+all:
+	for d in $(SUBDIRS); do $(MAKE) -C $$d $(ARCH); done
+
+dist:
+	for d in $(SUBDIRS); do $(MAKE) -C $$d $(ARCH); done
+
+clean:
+	for d in $(SUBDIRS); do $(MAKE) -C $$d clean; done
+
+distclean:
+	for d in $(SUBDIRS); do $(MAKE) -C $$d distclean; done
+	for d in $(SUBDIRS); do rm -f `find $$d -name 'RELEASE.local'`; done
+
+tidy: distclean
+