src/python/dm/common/service/dmRestWebServiceBase.py

+#!/usr/bin/env python
+
+#
+# Base web service class.
+#
+
+####################################################################
+import sys
+import os
+import cherrypy
+from cherrypy.process import plugins
+from cherrypy import server
+
+from dm.common.constants import dmStatus
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.dmModuleManager import DmModuleManager
+from dm.common.exceptions.configurationError import ConfigurationError
+
+####################################################################
+
+class DmRestWebServiceBase:
+
+    DEFAULT_N_SERVER_REQUEST_THREADS = 50
+    DEFAULT_SERVER_SOCKET_TIMEOUT = 30
+    CONFIG_SECTION_NAME = 'WebService'
+    CONFIG_OPTION_NAME_LIST = [ 'serviceHost', 'servicePort', 
+        'sslCertFile', 'sslKeyFile', 'sslCaCertFile', 'stationName' ]
+
+    class SignalHandler:
+        def __init__(self, signal, oldSignalHandler):
+            self.signal = signal
+            self.oldSignalHandler = oldSignalHandler
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+
+        def signalHandler(self):
+            self.logger.debug('%s signal handler called' % self.signal)
+            DmModuleManager.getInstance().stopModules()
+            self.oldSignalHandler()
+
+    def __init__(self, routeMapper):
+        self.configurationManager = ConfigurationManager.getInstance()
+        self.routeMapper = routeMapper
+        self.options = None
+        self.args = None
+        self.logger = None
+
+    def prepareOptions(self):
+        from optparse import OptionParser
+        p = OptionParser()
+        p.add_option('-d', '--daemon', action="store_true",
+            dest='daemonFlag', default=False, 
+            help="Run as a daemon.")
+        p.add_option('-p', '--pid-file',
+            dest='pidFile', default=None,
+            help="Store process id in the given file.")
+        p.add_option('', '--config-file',
+            dest='configFile', default=None,
+            help="Service configuration file.")
+        p.add_option('-P', '--port',
+            dest='servicePort', default=None,
+            help="Service port.")
+        p.add_option('-H', '--host',
+            dest='serviceHost', default=None,
+            help="Service host.")
+        p.add_option('-C', '--ssl-ca-cert', 
+            dest='sslCaCertFile', default=None,
+            help='SSL CA certificate path (used for client SSL certificate verification). Requires --ssl-key and --ssl-cert.')
+        p.add_option('-c', '--ssl-cert', 
+            dest='sslCertFile', default=None,
+            help='SSL certificate path. SSL operation requires both --ssl-key and --ssl-cert. Client SSL certificate verification also requires --ssl-ca-cert.')
+        p.add_option('-k', '--ssl-key', 
+            dest='sslKeyFile', default=None, 
+            help='SSL key path. SSL operation requires both --ssl-key and --ssl-cert. Client SSL certificate verification also requires --ssl-ca-cert.')
+        p.add_option('', '--n-server-threads', 
+            dest='nServerThreads', default=DmRestWebServiceBase.DEFAULT_N_SERVER_REQUEST_THREADS,
+            help='Number of service request handler threads (defaut: %s).' % DmRestWebServiceBase.DEFAULT_N_SERVER_REQUEST_THREADS)
+        return p
+
+    def initDmModules(self):
+        return None
+
+    def getDefaultServerHost(self):
+        return None
+
+    def getDefaultServerPort(self):
+        return None
+
+    # Instantiate modified signal handler that stops dm modules first, 
+    # and then does the default action.
+    def modifySignalHandlers(self, engine):
+        pluginsSignalHandler = plugins.SignalHandler(engine)
+        handlers = pluginsSignalHandler.handlers
+
+        # Add handler for interrupt
+        handlers['SIGINT'] = engine.exit
+
+        # Modify all signal handlers
+        for signal in handlers.keys():
+            self.logger.debug('Modifying signal: %s' % signal)
+            oldSignalHandler = handlers[signal]
+            self.logger.debug('Old signal handler: %s' % oldSignalHandler)
+            signalHandler = DmRestWebServiceBase.SignalHandler(signal, oldSignalHandler)
+            self.logger.debug('Setting signal handler to: %s' % signalHandler.signalHandler)
+            handlers[signal] = signalHandler.signalHandler
+            pluginsSignalHandler.subscribe()
+
+    def initServerLog(self):
+        cherrypyLogLevel = self.configurationManager.getCherrypyLogLevel()
+        cherrypy.log.error_log.setLevel(cherrypyLogLevel)
+        cherrypy.log.error_file = self.configurationManager.getCherrypyLogFile()
+        cherrypy.log.error_log.propagate = False
+        cherrypy.log.access_log.setLevel(cherrypyLogLevel)
+        cherrypy.log.access_file = self.configurationManager.getCherrypyAccessFile()
+        cherrypy.log.access_log.propagate = False
+
+    def updateServerConfig(self):
+        serviceHost = self.configurationManager.getServiceHost()
+        servicePort = int(self.configurationManager.getServicePort())
+        nServerThreads = int(self.options.nServerThreads)
+        configDict = {
+            'server.socket_host' : serviceHost,
+            'server.socket_port' : servicePort,
+            'server.thread_pool' : nServerThreads,
+            'log.screen' : (self.options.daemonFlag != True),
+        }
+        cherrypy.config.update(configDict)
+
+    def readConfigFile(self, configFile):
+        configFile = self.options.configFile
+        if not configFile:
+            configFile = self.configurationManager.getConfigFile()
+        else:
+            self.configurationManager.setConfigFile(configFile)
+
+        if not os.path.exists(configFile):
+            raise ConfigurationError('Configuration file %s does not exist.' % configFile)
+        # Read file and set config options
+        self.configurationManager.setOptionsFromConfigFile(DmRestWebServiceBase.CONFIG_SECTION_NAME, DmRestWebServiceBase.CONFIG_OPTION_NAME_LIST, configFile)
+
+    def readCommandLineOptions(self):
+        # This method should be called after reading config file
+        # in case some options are overridden
+        if self.options.sslCaCertFile != None:
+            self.configurationManager.setSslCaCertFile(self.options.sslCaCertFile)
+        if self.options.sslCertFile != None:
+            self.configurationManager.setSslCertFile(self.options.sslCertFile)
+
+        if self.options.sslKeyFile != None:
+            self.configurationManager.setSslKeyFile(self.options.sslKeyFile)
+
+        if self.options.serviceHost != None:
+            self.configurationManager.setServiceHost(self.options.serviceHost)
+
+        if self.options.servicePort != None:
+            self.configurationManager.setServicePort(self.options.servicePort)
+
+    def prepareServer(self):
+        try:
+            optionParser = self.prepareOptions()
+            (self.options, self.args) = optionParser.parse_args()
+      
+            # Read config file and override with command line options
+            self.readConfigFile(self.options.configFile)
+            self.readCommandLineOptions()
+             
+            # Turn off console log for daemon mode.
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+            if self.options.daemonFlag:
+                LoggingManager.getInstance().setConsoleLogLevel('CRITICAL')
+
+            dispatch = self.routeMapper.setupRoutes()
+            self.logger.debug('Using route dispatch: %s' % dispatch)
+
+            config = {
+                '/' : {
+                    'request.dispatch' : dispatch,
+                },
+            }
+                   
+            # No root controller as we provided our own.
+            cherrypy.tree.mount(root=None, config=config)
+            self.initServerLog()
+            self.updateServerConfig()
+
+            self.logger.info('Using host %s' % self.configurationManager.getServiceHost())
+            self.logger.info('Using port %s' % self.configurationManager.getServicePort())
+            self.logger.debug('Using %s request handler threads' % self.options.nServerThreads)
+        except Exception, ex:
+            if self.logger is not None:
+                self.logger.exception(ex)
+            else:
+                import traceback
+                print '\n%s' % sys.exc_info()[1]
+                traceback.print_exc(file=sys.stderr)
+            sys.exit(dmStatus.DM_ERROR)
+
+    # Run server.
+    def __runServer(self):
+        self.logger.info('Starting service')
+        engine = cherrypy.engine
+
+        # Set up Deamonization
+        if self.options.daemonFlag:
+            plugins.Daemonizer(engine).subscribe()
+        self.logger.debug('Daemon mode: %s' % self.options.daemonFlag)
+
+        if self.options.pidFile != None:
+            plugins.PIDFile(engine, self.options.pidFile).subscribe()
+        self.logger.debug('Using PID file: %s' % self.options.pidFile)
+
+        sslCertFile = self.configurationManager.getSslCertFile()
+        sslKeyFile = self.configurationManager.getSslKeyFile()
+        sslCaCertFile = self.configurationManager.getSslCaCertFile()
+        if sslCertFile != None and sslKeyFile != None:
+            server.ssl_ca_certificate = None
+            if sslCaCertFile != None:
+                server.ssl_ca_certificate = self.options.sslCaCertFile
+                self.logger.info('Using SSL CA cert file: %s' % sslCaCertFile)
+            server.ssl_certificate = sslCertFile
+            self.logger.info('Using SSL cert file: %s' % sslCertFile)
+
+            server.ssl_private_key = sslKeyFile
+            self.logger.info('Using SSL key file: %s' % sslKeyFile)
+
+        server.ssl_module = 'builtin'
+        #server.ssl_module = 'pyopenssl'
+
+        # Increase timeout to prevent early SSL connection terminations
+        server.socket_timeout = DmRestWebServiceBase.DEFAULT_SERVER_SOCKET_TIMEOUT
+        
+        # Setup the signal handler to stop the application while running.
+        if hasattr(engine, 'signal_handler'):
+            self.logger.debug('Subscribing signal handler')
+            engine.signal_handler.subscribe()
+            self.modifySignalHandlers(engine)
+
+        # Turn off autoreloader.
+        self.logger.debug('Turning off autoreloader')
+        engine.autoreload.unsubscribe()
+
+        # Start the engine.
+        try:
+            self.logger.debug('Starting engine')
+            engine.start()
+
+            # Prepare dm services.
+            self.logger.debug('Starting modules')
+            self.initDmModules()
+            DmModuleManager.getInstance().startModules()
+
+        except Exception, ex:
+            self.logger.exception('Service exiting: %s' % ex)
+            DmModuleManager.getInstance().stopModules()
+            return dmStatus.DM_ERROR
+        self.logger.info('Service ready')
+        engine.block()
+        DmModuleManager.getInstance().stopModules()
+        self.logger.info('Service done')
+        return dmStatus.DM_OK
+
+    # Run server instance.
+    def run(self):
+        self.prepareServer()
+        sys.exit(self.__runServer())
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/service/dmSessionController.py

+#!/usr/bin/env python
+
+#
+# Base DM session controller class.
+#
+
+import cherrypy
+from dm.common.constants import dmRole
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.service.dmController import DmController
+from dm.common.service.loginController import LoginController
+
+class DmSessionController(DmController):
+    """ Base session controller class. """
+
+    _cp_config = {
+        'tools.sessions.on': True,
+        'tools.auth.on': True
+    }
+
+    #auth = LoginController()
+    # Add before_handler for authorization
+    cherrypy.tools.auth = cherrypy.Tool('before_handler', LoginController.authCheck)
+
+    def __init__(self):
+        DmController.__init__(self)
+
+    @classmethod
+    def require(cls, *conditions):
+        """
+        Decorator that appends conditions to the auth.require config
+        variable.
+        """
+        def decorate(f):
+            if not hasattr(f, '_cp_config'):
+                f._cp_config = dict()
+            if 'auth.require' not in f._cp_config:
+                f._cp_config['auth.require'] = []
+                f._cp_config['auth.require'].extend(conditions)
+            return f
+        return decorate
+
+    @classmethod
+    def anyOf(cls, *conditions):
+        """ Returns True if any of the conditions match. """
+        def check():
+            for c in conditions:
+                if c():
+                    return True
+            return False
+        return check
+
+    @classmethod
+    def allOf(cls, *conditions):
+        """ Returns True if all of the conditions match. """
+        def check():
+            for c in conditions:
+                if not c():
+                    return False
+            return True
+        return check
+
+    @classmethod
+    def isLoggedIn(cls):
+        """ Returns True if session has been established. """
+        def userIsLoggedIn():
+            role = cherrypy.session.get(LoginController.SESSION_ROLE_KEY, None)
+            if role is not None:
+                return True
+            return False
+        return userIsLoggedIn
+
+    @classmethod
+    def isAdministrator(cls):
+        def userIsAdministrator():
+            result = (cherrypy.session.get(LoginController.SESSION_ROLE_KEY, None) == dmRole.DM_ADMIN_SESSION_ROLE)
+            return result
+        return userIsAdministrator
+
+    @classmethod
+    def hasAdministratorRole(cls):
+        sessionRole = cherrypy.session.get(LoginController.SESSION_ROLE_KEY, None)
+        return (sessionRole == dmRole.DM_ADMIN_SESSION_ROLE)
+        
+    @classmethod
+    def hasManagerRole(cls, experimentStationIdOrName):
+        systemRoleDict = cherrypy.session.get(LoginController.SESSION_SYSTEM_ROLE_DICT_KEY, None)
+        experimentStationIdOrNameList = systemRoleDict.get(dmRole.DM_MANAGER_SYSTEM_ROLE_ID, [])
+        if not experimentStationIdOrNameList:
+            # Remote sessions may come with string key
+            experimentStationIdOrNameList = systemRoleDict.get(str(dmRole.DM_MANAGER_SYSTEM_ROLE_ID), [])
+        return (experimentStationIdOrNameList.count(experimentStationIdOrName) > 0)
+        
+    @classmethod
+    def hasPiRole(cls, experimentIdOrName):
+        experimentRoleDict = cherrypy.session.get(LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY, None)
+        experimentIdOrNameList = experimentRoleDict.get(dmRole.DM_PI_EXPERIMENT_ROLE_ID, [])
+        return (experimentIdOrNameList.count(experimentIdOrName) > 0)
+        
+    @classmethod
+    def hasUserRole(cls, experimentIdOrName):
+        experimentRoleDict = cherrypy.session.get(LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY, None)
+        experimentIdOrNameList = experimentRoleDict.get(dmRole.DM_USER_EXPERIMENT_ROLE_ID, [])
+        return (experimentIdOrNameList.count(experimentIdOrName) > 0)
+        
+    @classmethod
+    def canManageStation(cls):
+        def userCanManageStation():
+            if cls.hasAdministratorRole():
+                return True
+            stationName = ConfigurationManager.getInstance().getStationName()
+            return cls.hasManagerRole(stationName)
+        return userCanManageStation
+        
+

src/python/dm/common/service/loginController.py

+#!/usr/bin/env python
+
+import cherrypy
+import datetime
+import urllib
+from cherrypy.lib import httpauth
+
+from dm.common.constants import dmStatus
+from dm.common.constants import dmRole
+from dm.common.constants import dmHttpStatus
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.dmHttpError import DmHttpError
+from dm.common.exceptions.authorizationError import AuthorizationError
+from dm.common.exceptions.invalidSession import InvalidSession
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.service.dmController import DmController
+from dm.common.service.auth.authorizationPrincipalManager import AuthorizationPrincipalManager
+from dm.common.service.auth.singleSignOnManager import SingleSignOnManager
+
+class LoginController(DmController):
+    """ Controller to provide login and logout actions. """
+
+    SESSION_USERNAME_KEY = '_cp_username'
+    SESSION_ROLE_KEY = 'sessionRole'
+    SESSION_SYSTEM_ROLE_DICT_KEY = 'systemRoleDict'
+    SESSION_EXPERIMENT_ROLE_DICT_KEY = 'experimentRoleDict'
+    ORIGINAL_SESSION_ID_KEY = 'originalid'
+    INVALID_SESSION_KEY = 'invalidSession'
+
+    _cp_config = {
+        'tools.sessions.on' : True,
+        'tools.auth.on' : True
+    }
+
+    def __init__(self):
+        DmController.__init__(self)
+
+    def onLogin(self, username):
+        """ Called on successful login. """
+        return
+
+    def onLogout(self, username):
+        """ Called on logout. """
+        return
+
+    @classmethod
+    def getLoginForm(cls, msg='Enter username and password:', username='', fromPage='/'):
+        return """
+        <html>
+            <body>
+                <form method='post' action='/auth/login'>
+                    <input type='hidden' name='fromPage' value='%(fromPage)s' />
+                    <h2>DM Service</h2>
+                    <p/>
+                    %(msg)s
+                    <p/>
+                    <table border='0'>
+                        <tr>
+                            <td>Username:</td><td><input type='text' name="username" value='%(username)s'/></td>
+                        </tr>
+                        <tr>
+                            <td>Password:</td><td><input type='password' name='password' /></td>
+                        </tr>
+                        <p/>
+                        <tr>
+                            <td></td>
+                            <td><input type='submit' value='Log In' /></td>
+                        </tr>
+                    </table>
+                </form>
+            </body>
+        </html>""" % locals()
+
+    @classmethod
+    def parseBasicAuthorizationHeaders(cls):
+        try:
+            username = None
+            password = None
+            authorization = cherrypy.request.headers['authorization']
+            authorizationHeader = httpauth.parseAuthorization(authorization)
+            if authorizationHeader['auth_scheme'] == 'basic':
+                username = authorizationHeader['username']
+                password = authorizationHeader['password']
+            if username and password:
+                return (username, password)
+            else:
+                raise AuthorizationError('Username and/or password not supplied.')
+        except Exception, ex:
+            errorMsg = 'Could not extract username/password from authorization header: %s' % ex
+            raise AuthorizationError(errorMsg)
+
+    @classmethod
+    def checkCredentials(cls, username, password):
+        """ Verifies credentials for username and password."""
+        logger = LoggingManager.getInstance().getLogger('LoginController:checkCredentials')
+        logger.debug('Checking credential for User: %s' % (username))
+        #logger.debug('Checking credential for User: %s, Password: %s' % (username, password))
+        logger.debug('Session id: %s' % cherrypy.serving.session.id)
+        principal = AuthorizationPrincipalManager.getInstance().getAuthenticatedAuthorizationPrincipal(username, password)
+        logger.debug('Principal: %s' % (principal))
+        if principal:
+            cherrypy.session[LoginController.SESSION_ROLE_KEY] = principal.getSessionRole()
+            cherrypy.session[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY] = principal['userSystemRoleDict']
+            cherrypy.session[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY] = principal['userExperimentRoleDict']
+            logger.debug('Successful login from user: %s (role: %s)' % (username, principal.getSessionRole()))
+            # Try adding to SingleSignOnManager
+            sessionId = cherrypy.serving.session.id
+            sessionCache = cherrypy.session.cache
+            sessionInfo = {LoginController.SESSION_ROLE_KEY : principal.getSessionRole()}
+            sessionInfo[LoginController.SESSION_USERNAME_KEY] = username
+            sessionInfo[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY] = principal.get('userSystemRoleDict', {})
+            sessionInfo[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY] = principal.get('userExperimentRoleDict', {})
+            ssoManager = SingleSignOnManager.getInstance()
+            ssoManager.addSession(sessionId, sessionInfo)
+        else:
+            logger.debug('Login denied for user: %s' % username)
+            username = cherrypy.session.get(LoginController.SESSION_USERNAME_KEY, None)
+            if username is not None:
+                cherrypy.request.login = None
+                cherrypy.session[LoginController.INVALID_DM_SESSION_KEY] = True
+            raise AuthorizationError('Incorrect username or password.')
+        return principal
+
+    @classmethod
+    def authCheck(cls, *args, **kwargs):
+        """
+        A tool that looks in config for 'auth.require'. If found and it
+        is not None, a login is required and the entry is evaluated as a list of
+        conditions that the user must fulfill.
+        """
+        logger = LoggingManager.getInstance().getLogger('LoginController:authCheck')
+        conditions = cherrypy.request.config.get('auth.require', None)
+        #logger.debug('Headers: %s' % (cherrypy.request.headers))
+        #logger.debug('Request params: %s' % (cherrypy.request.params))
+        #logger.debug('Request query string: %s' % (cherrypy.request.query_string))
+        method = urllib.quote(cherrypy.request.request_line.split()[0])
+        params = urllib.quote(cherrypy.request.request_line.split()[1])
+
+        if conditions is None:
+            logger.debug('No conditions imposed')
+            return
+
+        sessionId = cherrypy.serving.session.id
+        sessionCache = cherrypy.session.cache
+        # If session cache does not have current session id, reuse original
+        # session id
+        if not sessionCache.has_key(sessionId) and cherrypy.serving.session.__dict__.has_key(LoginController.ORIGINAL_SESSION_ID_KEY):
+            logger.debug('Reusing original session id: %s' % sessionId)
+            sessionId = cherrypy.serving.session.__dict__.get(LoginController.ORIGINAL_SESSION_ID_KEY)
+        #logger.debug('Session: %s' % ((cherrypy.session.__dict__)))
+        logger.debug('Session cache length: %s' % (len(sessionCache)))
+        #logger.debug('Session cache: %s' % (sessionCache))
+
+        # Check session.
+        # Try SingleSignOnManager first
+        ssoManager = SingleSignOnManager.getInstance()
+
+        # SSO Manager returns session info
+        sessionInfo = ssoManager.checkSession(sessionId)
+        if not sessionInfo:
+            # Cache has tuple (sessionInfo, updateTime)
+            sessionTuple = sessionCache.get(sessionId)
+            if sessionTuple:
+                sessionInfo = sessionTuple[0]
+                logger.debug('Retrieved session info from cache: %s' % sessionInfo)
+        else:
+            logger.debug('Retrieved session %s from SSO Manager' % sessionId)
+            sessionCache[sessionId] = (sessionInfo, datetime.datetime.now())
+
+        if not sessionInfo:
+            errorMsg = 'Invalid or expired session id: %s.' % sessionId
+            logger.debug(errorMsg)
+            raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', InvalidSession(errorMsg))
+
+        username = sessionInfo.get(LoginController.SESSION_USERNAME_KEY)
+        if not username:
+            errorMsg = 'Invalid session id: %s (no username supplied).' % sessionId
+            logger.debug(errorMsg)
+            raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', InvalidSession(errorMsg))
+
+        cherrypy.session[LoginController.SESSION_ROLE_KEY] = sessionInfo[LoginController.SESSION_ROLE_KEY]
+        cherrypy.session[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY] = sessionInfo[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY] 
+        cherrypy.session[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY] = sessionInfo[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY] 
+        logger.debug('Session id %s is valid (username: %s)' % (sessionId, username))
+        cherrypy.request.login = username
+        for condition in conditions:
+            # A condition is just a callable that returns true or false
+            if not condition():
+                logger.debug('Authorization check %s() failed for username %s' % (condition.func_name, username))
+                errorMsg = 'Authorization check %s() failed for user %s.' % (condition.func_name, username)
+                raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', AuthorizationError(errorMsg))
+
+
+    @cherrypy.expose
+    def login(self, username=None, password=None, fromPage='/'):
+        self.logger.debug('Attempting login from username %s' % (username))
+        try:
+            if username is None or password is None:
+                self.logger.debug('Parsing auth headers for username %s' % (username))
+                (username, password) = LoginController.parseBasicAuthorizationHeaders()
+                self.logger.debug('Retrieving principal for username %s' % (username))
+            principal = LoginController.checkCredentials(username, password)
+        except DmHttpError, ex:
+            raise
+        except DmException, ex:
+            self.logger.debug('Authorization failed (username %s): %s' % (username, ex))
+            self.addDmExceptionHeaders(ex)
+            raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', ex)
+
+        # Authorization worked.
+        cherrypy.session[LoginController.SESSION_USERNAME_KEY] = cherrypy.request.login = username
+        self.onLogin(username)
+        self.addDmSessionRoleHeaders(principal.getSessionRole())
+        self.addDmResponseHeaders()
+
+    @cherrypy.expose
+    def logout(self, fromPage='/'):
+        sess = cherrypy.session
+        username = sess.get(LoginController.SESSION_USERNAME_KEY, None)
+        if username:
+            del sess[LoginController.SESSION_USERNAME_KEY]
+            cherrypy.request.login = None
+        self.onLogout(username)
+

src/python/dm/common/service/loginRouteDescriptor.py

+#!/usr/bin/env python
+
+#
+# Login route descriptor.
+#
+
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.service.loginController import LoginController
+
+class LoginRouteDescriptor:
+
+    @classmethod
+    def getRoutes(cls):
+        contextRoot = ConfigurationManager.getInstance().getContextRoot()
+        loginController = LoginController()
+
+        routes = [
+
+            # Login
+            {
+                'name'          : 'login',
+                'path'          : '%s/login' % contextRoot,
+                'controller'    : loginController,
+                'action'        : 'login',
+                'method'        : [ 'PUT', 'POST' ]
+            },
+
+        ]
+       
+        return routes
+
+

src/python/dm/common/utility/configurationManager.py

+#!/usr/bin/env python
+
+# 
+# Configuration manager singleton.
+#
+
+#######################################################################
+
+import os
+import socket
+import pwd
+import UserDict
+import ConfigParser
+
+from dm.common.constants import dmServiceConstants
+from dm.common.exceptions.configurationError import ConfigurationError
+
+#######################################################################
+
+# Defaults.
+
+DEFAULT_DM_ROOT_DIR = '/opt/dm'
+DEFAULT_DM_INSTALL_DIR = '%s'                     # requires install dir
+DEFAULT_DM_CONFIG_FILE = '%s/etc/%s.%s.conf'      # requires install dir/db name/service name
+DEFAULT_DM_SERVICE_NAME= 'web-service'
+
+DEFAULT_DM_LOG_FILE = '%s/var/log/%s.%s.log'         # requires install dir/db name/service name
+DEFAULT_DM_CONSOLE_LOG_LEVEL = 'CRITICAL'
+DEFAULT_DM_FILE_LOG_LEVEL = 'INFO'
+#DEFAULT_DM_LOG_RECORD_FORMAT = '%(asctime)s,%(msecs)03d [%(levelname)s] %(module)s:%(lineno)d %(user)s@%(host)s %(name)s (%(process)d): %(message)s'
+#DEFAULT_DM_LOG_RECORD_FORMAT = '%(asctime)s,%(msecs)03d %(levelname)s %(module)s:%(lineno)d %(process)d:  %(message)s'
+DEFAULT_DM_LOG_RECORD_FORMAT = '%(asctime)s,%(msecs)03d %(levelname)s %(process)d:  %(message)s'
+DEFAULT_DM_LOG_DATE_FORMAT = '%Y/%m/%d %H:%M:%S'
+
+DEFAULT_DM_CHERRYPY_LOG_LEVEL = 'ERROR'
+DEFAULT_DM_CHERRYPY_LOG_FILE = '%s/var/log/%s.%s.cherrypy.error'     # requires install dir/db name/service name
+DEFAULT_DM_CHERRYPY_ACCESS_FILE = '%s/var/log/%s.%s.cherrypy.access' # requires install dir/db name/service name
+
+DEFAULT_DM_SERVICE_PORT = 22236    # 222DM
+DEFAULT_DM_SERVICE_HOST = '0.0.0.0'
+DEFAULT_DM_SERVICE_PROTOCOL = dmServiceConstants.DM_SERVICE_PROTOCOL_HTTP
+
+DEFAULT_DM_WEB_SERVICE_PROTOCOL = DEFAULT_DM_SERVICE_PROTOCOL 
+DEFAULT_DM_DS_WEB_SERVICE_PORT = 22236    # 222DM
+DEFAULT_DM_DS_WEB_SERVICE_HOST = '127.0.0.1'
+DEFAULT_DM_DAQ_WEB_SERVICE_PORT = 33336    # 333DM
+DEFAULT_DM_DAQ_WEB_SERVICE_HOST = '127.0.0.1'
+DEFAULT_DM_CAT_WEB_SERVICE_PORT = 44436    # 444DM
+DEFAULT_DM_CAT_WEB_SERVICE_HOST = '127.0.0.1'
+DEFAULT_DM_PROC_WEB_SERVICE_PORT = 55536    # 555DM
+DEFAULT_DM_PROC_WEB_SERVICE_HOST = '127.0.0.1'
+
+DEFAULT_DM_DB = 'postgresql'
+DEFAULT_DM_DB_HOST = '127.0.0.1'
+DEFAULT_DM_DB_PORT = 11136
+DEFAULT_DM_DB_PASSWORD = ''
+DEFAULT_DM_DB_NAME = 'dm'
+DEFAULT_DM_DB_SCHEMA = DEFAULT_DM_DB_NAME
+DEFAULT_DM_DB_USER = DEFAULT_DM_DB_NAME
+DEFAULT_DM_DB_PASSWORD_FILE = '%s/etc/%s.db.passwd' # requires install dir/db name
+
+DEFAULT_DM_CONTEXT_ROOT = '/dm'
+
+# Session cache file
+DEFAULT_DM_SESSION_CACHE_FILE = None
+
+# Enforce session credentials.
+DEFAULT_DM_REQUIRE_SESSION_CREDENTIALS = False
+
+# SSL variables
+DEFAULT_DM_SSL_CA_CERT_FILE = None
+DEFAULT_DM_SSL_CERT_FILE = None
+DEFAULT_DM_SSL_KEY_FILE = None
+
+# Login (user|password) file
+DEFAULT_DM_LOGIN_FILE = None
+
+# Station name
+DEFAULT_DM_STATION_NAME = None
+
+# Allowed experiment types
+DEFAULT_DM_ALLOWED_EXPERIMENT_TYPES = None
+
+class ConfigurationManager(UserDict.UserDict):
+    """ 
+    Singleton class used for keeping system configuration data. The class
+    initializes its data using predefined defaults, or from certain 
+    environment variables.
+
+    Usage:
+        from dm.common.utility import configurationManager
+        cm = configurationManager.getInstance()
+        cm.setConsoleLogLevel('info')
+        level = cm.getConsoleLogLevel()
+        cm['myKey'] = 'myValue'
+        value = cm.get('myKey')
+    """
+
+    # Get singleton instance.
+    @classmethod
+    def getInstance(cls):
+        """ Get configuration manager singleton instance. """
+        from dm.common.utility.configurationManager import ConfigurationManager
+        try:
+            cm = ConfigurationManager()
+        except ConfigurationManager, ex:
+            cm = ex
+        return cm
+
+    # Singleton.
+    __instance = None
+
+    def __init__(self):
+        if ConfigurationManager.__instance:
+            raise ConfigurationManager.__instance
+        ConfigurationManager.__instance = self
+        UserDict.UserDict.__init__(self)
+        self['user'] = pwd.getpwuid(os.getuid())[0]
+        self['host'] = socket.gethostname()
+
+        self['defaultRootDir'] = DEFAULT_DM_ROOT_DIR
+        self.__setFromEnvVar('rootDir', 'DM_ROOT_DIR')
+        self['defaultInstallDir'] = DEFAULT_DM_INSTALL_DIR % self.getRootDir()
+        self.__setFromEnvVar('installDir', 'DM_INSTALL_DIR')
+        self['defaultServiceName'] = DEFAULT_DM_SERVICE_NAME 
+        self.__setFromEnvVar('serviceName', 'DM_SERVICE_NAME')
+        self['defaultDbName'] = DEFAULT_DM_DB_NAME 
+        self.__setFromEnvVar('dbName', 'DM_DB_NAME')
+        self.__resetDynamicDefaults()
+
+        self['defaultConsoleLogLevel'] = DEFAULT_DM_CONSOLE_LOG_LEVEL
+        self['defaultFileLogLevel'] = DEFAULT_DM_FILE_LOG_LEVEL
+        self['defaultLogRecordFormat'] = DEFAULT_DM_LOG_RECORD_FORMAT
+        self['defaultLogDateFormat'] = DEFAULT_DM_LOG_DATE_FORMAT
+
+        self['defaultCherrypyLogLevel'] = DEFAULT_DM_CHERRYPY_LOG_LEVEL
+
+        self['defaultServicePort'] = DEFAULT_DM_SERVICE_PORT
+        self['defaultServiceHost'] = DEFAULT_DM_SERVICE_HOST
+        self['defaultServiceProtocol'] = DEFAULT_DM_SERVICE_PROTOCOL
+
+        self['defaultWebServiceProtocol'] = DEFAULT_DM_WEB_SERVICE_PROTOCOL
+        self['defaultDaqWebServicePort'] = DEFAULT_DM_DAQ_WEB_SERVICE_PORT
+        self['defaultDaqWebServiceHost'] = DEFAULT_DM_DAQ_WEB_SERVICE_HOST
+        self['defaultDsWebServicePort'] = DEFAULT_DM_DS_WEB_SERVICE_PORT
+        self['defaultDsWebServiceHost'] = DEFAULT_DM_DS_WEB_SERVICE_HOST
+        self['defaultCatWebServicePort'] = DEFAULT_DM_CAT_WEB_SERVICE_PORT
+        self['defaultCatWebServiceHost'] = DEFAULT_DM_CAT_WEB_SERVICE_HOST
+        self['defaultProcWebServicePort'] = DEFAULT_DM_PROC_WEB_SERVICE_PORT
+        self['defaultProcWebServiceHost'] = DEFAULT_DM_PROC_WEB_SERVICE_HOST
+
+        self['defaultDb'] = DEFAULT_DM_DB
+        self['defaultDbHost'] = DEFAULT_DM_DB_HOST
+        self['defaultDbPort'] = DEFAULT_DM_DB_PORT
+        self['defaultDbPassword'] = DEFAULT_DM_DB_PASSWORD
+
+        self['defaultContextRoot'] = DEFAULT_DM_CONTEXT_ROOT
+
+        self['defaultSessionCacheFile'] = DEFAULT_DM_SESSION_CACHE_FILE
+        self['defaultRequireSessionCredentials'] = DEFAULT_DM_REQUIRE_SESSION_CREDENTIALS
+
+        self['defaultSslCaCertFile'] = DEFAULT_DM_SSL_CA_CERT_FILE
+        self['defaultSslCertFile'] = DEFAULT_DM_SSL_CERT_FILE
+        self['defaultSslKeyFile'] = DEFAULT_DM_SSL_KEY_FILE
+
+        self['defaultLoginFile'] = DEFAULT_DM_LOGIN_FILE
+
+        self['defaultStationName'] = DEFAULT_DM_STATION_NAME
+
+        self['defaultAllowedExperimentTypes'] = DEFAULT_DM_ALLOWED_EXPERIMENT_TYPES
+
+        # Settings that might come from environment variables.
+        self.__setFromEnvVar('logFile', 'DM_LOG_FILE')
+        self.__setFromEnvVar('consoleLogLevel', 'DM_CONSOLE_LOG_LEVEL')
+        self.__setFromEnvVar('fileLogLevel', 'DM_FILE_LOG_LEVEL')
+        self.__setFromEnvVar('logRecordFormat', 'DM_LOG_RECORD_FORMAT')
+        self.__setFromEnvVar('logDateFormat', 'DM_LOG_DATE_FORMAT')
+
+        self.__setFromEnvVar('cherrypyLogLevel', 'DM_CHERRYPY_LOG_LEVEL')
+        self.__setFromEnvVar('cherrypyLogFile', 'DM_CHERRYPY_LOG_FILE')
+        self.__setFromEnvVar('cherrypyAccessFile', 'DM_CHERRYPY_ACCESS_FILE')
+
+        self.__setFromEnvVar('serviceHost', 'DM_SERVICE_HOST')
+        self.__setFromEnvVar('servicePort', 'DM_SERVICE_PORT')
+        self.__setFromEnvVar('serviceProtocol', 'DM_SERVICE_PROTOCOL')
+
+        self.__setFromEnvVar('webServiceProtocol', 'DM_WEB_SERVICE_PROTOCOL')
+        self.__setFromEnvVar('daqWebServiceHost', 'DM_DAQ_WEB_SERVICE_HOST')
+        self.__setFromEnvVar('daqWebServicePort', 'DM_DAQ_WEB_SERVICE_PORT')
+        self.__setFromEnvVar('dsWebServiceHost', 'DM_DS_WEB_SERVICE_HOST')
+        self.__setFromEnvVar('dsWebServicePort', 'DM_DS_WEB_SERVICE_PORT')
+        self.__setFromEnvVar('catWebServiceHost', 'DM_CAT_WEB_SERVICE_HOST')
+        self.__setFromEnvVar('catWebServicePort', 'DM_CAT_WEB_SERVICE_PORT')
+        self.__setFromEnvVar('procWebServiceHost', 'DM_PROC_WEB_SERVICE_HOST')
+        self.__setFromEnvVar('procWebServicePort', 'DM_PROC_WEB_SERVICE_PORT')
+
+        self.__setFromEnvVar('contextRoot', 'DM_CONTEXT_ROOT')
+
+        self.__setFromEnvVar('sessionCacheFile', 'DM_SESSION_CACHE_FILE')
+
+        self.__setFromEnvVar('sslCaCertFile', 'DM_SSL_CA_CERT_FILE')
+        self.__setFromEnvVar('sslCertFile', 'DM_SSL_CERT_FILE')
+        self.__setFromEnvVar('sslKeyFile', 'DM_SSL_KEY_FILE')
+
+        self.__setFromEnvVar('configFile', 'DM_CONFIG_FILE')
+        self.__setFromEnvVar('dbPasswordFile', 'DM_DB_PASSWORD_FILE')
+        self.__setFromEnvVar('loginFile', 'DM_LOGIN_FILE')
+
+        self.__setFromEnvVar('stationName', 'DM_STATION_NAME')
+        self.__setFromEnvVar('allowedExperimentTypes', 'DM_ALLOWED_EXPERIMENT_TYPES')
+
+        # Settings that might come from file.
+        self.__setFromVarFile('dbPassword', self.getDbPasswordFile())
+
+        # Variables we do not keep in a dictionary
+        self.configParser = None
+
+    # Reset defaults that depend on install dir and service name
+    def __resetDynamicDefaults(self):
+        self['defaultConfigFile'] = DEFAULT_DM_CONFIG_FILE % (self.getInstallDir(), self.getDbName(), self.getServiceName())
+        self['defaultLogFile'] = DEFAULT_DM_LOG_FILE % (self.getInstallDir(), self.getDbName(), self.getServiceName())
+        self['defaultCherrypyAccessFile'] = DEFAULT_DM_CHERRYPY_ACCESS_FILE % (self.getInstallDir(), self.getDbName(), self.getServiceName())
+        self['defaultCherrypyLogFile'] = DEFAULT_DM_CHERRYPY_LOG_FILE % (self.getInstallDir(), self.getDbName(), self.getServiceName())
+        self['defaultDbUser'] = self.getDbName()
+        self['defaultDbSchema'] = self.getDbName()
+        self['defaultDbPasswordFile'] = DEFAULT_DM_DB_PASSWORD_FILE % (self.getInstallDir(), self.getDbName())
+
+    # This function will ignore errors if environment variable is not set.
+    def __setFromEnvVar(self, key, envVar):
+        """ 
+        Set value for the specified key from a given environment variable.
+        This function ignores errors for env. variables that are not set.
+        """
+        try:
+            self[key] = os.environ[envVar]
+        except:
+            pass
+
+    # This function will ignore errors if variable file is not present.
+    def __setFromVarFile(self, key, varFile):
+        """ 
+        Set value for the specified key from a given file. The first line  
+        in the file is variable value.
+        This function ignores errors.
+        """
+        try:
+            v = open(varFile, 'r').readline()
+            self[key] = v.lstrip().rstrip()
+        except Exception, ex:
+            pass
+
+    def __getKeyValue(self, key, default='__dm_default__'):
+        """
+        Get value for a given key.
+        Keys will be of the form 'logFile', and the default keys have
+        the form 'defaultLogFile'.
+        """
+        defaultKey = "default" + key[0].upper() + key[1:]
+        defaultValue = self.get(defaultKey, None)
+        if default != '__dm_default__':
+            defaultValue = default
+        return self.get(key, defaultValue)
+
+    def setOptionsFromConfigFile(self, configSection, keyList, configFile=None):
+        _configFile = configFile
+        if _configFile is None:
+           _configFile = self.getConfigFile()
+            
+        if _configFile is not None and os.path.exists(_configFile):
+            configParser = ConfigParser.RawConfigParser()
+            configParser.read(_configFile)
+            if not configParser.has_section(configSection):
+                return
+            for key in keyList:
+                if configParser.has_option(configSection, key):
+                    self[key] = configParser.get(configSection, key)
+
+    def clearConfigParser(self):
+        self.configParser = None
+
+    def getConfigParser(self, defaults={}):
+        if self.configParser is None:
+            configFile = self.getConfigFile()
+            if os.path.exists(configFile):
+                self.configParser = ConfigParser.ConfigParser(defaults)
+                self.configParser.read(configFile)
+                self.configParser.defaults = defaults
+        return self.configParser
+
+    def setConfigDefaults(self, defaults={}):
+        configParser = self.getConfigParser()
+        if configParser is not None:
+            configParser.defaults = defaults
+
+    def getConfigOption(self, configSection, key, defaultValue=None):
+        configParser = self.getConfigParser()
+        if self.hasConfigSection(configSection):
+            try:
+                return configParser.get(configSection, key)
+            except ConfigParser.NoOptionError, ex:
+                # ok, return default value
+                pass
+        return defaultValue
+
+    def getConfigSections(self):
+        configParser = self.getConfigParser()
+        if configParser is not None:
+            return configParser.sections()
+        return []
+
+    def hasConfigSection(self, name):
+        configSections = self.getConfigSections()
+        if name in configSections:
+            return True
+        return False
+
+    def getConfigItems(self, configSection):
+        configParser = self.getConfigParser()
+        if configParser is not None and configParser.has_section(configSection):
+            return configParser.items(configSection)
+        return []
+
+    @classmethod
+    def getConfigParserFromConfigFile(cls, configFile):
+        if not os.path.exists(configFile):
+            return None
+        configParser = ConfigParser.RawConfigParser()
+        configParser.read(configFile)
+        return configParser
+
+    @classmethod
+    def getOptionFromConfigParser(cls, configParser, configSection, key, defaultValue=None):
+        if configParser is not None and configParser.has_section(configSection):
+            return configParser.get(configSection, key)
+        else:
+            return defaultValue
+
+    @classmethod
+    def getConfigSectionsFromConfigParser(cls, configParser):
+        if configParser is not None:
+            return configParser.sections()
+        return []
+
+    @classmethod
+    def getModuleClassConstructorTuple(cls, value, creatorClass=None):
+        """ Extract (module,class,constructor) tuple from the given value. """
+        itemList = value.split('(')
+        if not itemList:
+            return ()
+        itemList2 = itemList[0].split('.')
+        moduleNameList = itemList2[0:-1]
+        className = itemList2[-1]
+        moduleName = className[0].lower() + className[1:]
+
+        # If module name list is empty, use convention
+        # for determining import module name (use creator class module)
+        if not len(moduleNameList) and creatorClass is not None:
+            moduleNameList = creatorClass.__module__.split('.')
+            moduleNameList[-1] = moduleName
+
+        if len(moduleNameList): 
+            moduleName = '.'.join(moduleNameList)
+        constructor = '%s(%s' % (className, ''.join(itemList[1:]))
+        return (moduleName,className,constructor)
+
+    def getHost(self):
+        return self['host']
+    
+    def getUser(self):
+        return self['user']
+
+    def getDefaultRootDir(self):
+        return self['defaultRootDir']
+
+    def setRootDir(self, rootDir):
+        self['rootDir'] = rootDir
+
+    def getRootDir(self, default='__dm_default__'):
+        return self.__getKeyValue('rootDir', default)
+
+    def getDefaultInstallDir(self):
+        return self['defaultInstallDir']
+
+    def setInstallDir(self, installDir):
+        self['installDir'] = installDir
+        self.__resetDynamicDefaults()
+
+    def getInstallDir(self, default='__dm_default__'):
+        return self.__getKeyValue('installDir', default)
+
+    def getDefaultServiceName(self):
+        return self['defaultServiceName']
+
+    def setServiceName(self, serviceName):
+        self['serviceName'] = serviceName
+        self.__resetDynamicDefaults()
+
+    def getServiceName(self, default='__dm_default__'):
+        return self.__getKeyValue('serviceName', default)
+
+    def getDefaultDbName(self):
+        return self['defaultDbName']
+
+    def setDbName(self, dbName):
+        self['dbName'] = dbName
+        self.__resetDynamicDefaults()
+
+    def getDbName(self, default='__dm_default__'):
+        return self.__getKeyValue('dbName', default)
+
+    def getDefaultLogFile(self):
+        return self['defaultLogFile']
+
+    def setLogFile(self, logFile):
+        self['logFile'] = logFile
+
+    def getLogFile(self, default='__dm_default__'):
+        return self.__getKeyValue('logFile', default)
+
+    def hasLogFile(self):
+        return self.has_key('logFile')
+
+    def getDefaultConsoleLogLevel(self):
+        return self['defaultConsoleLogLevel']
+
+    def setConsoleLogLevel(self, level):
+        self['consoleLogLevel'] = level 
+
+    def getConsoleLogLevel(self, default='__dm_default__'):
+        return self.__getKeyValue('consoleLogLevel', default)
+
+    def getConsoleLogLevelFromEnvVar(self):
+        return os.environ.get('DM_CONSOLE_LOG_LEVEL')
+
+    def hasConsoleLogLevel(self):
+        return self.has_key('consoleLogLevel')
+
+    def getDefaultFileLogLevel(self):
+        return self['defaultFileLogLevel']
+
+    def setFileLogLevel(self, level):
+        self['fileLogLevel'] = level 
+
+    def getFileLogLevel(self, default='__dm_default__'):
+        return self.__getKeyValue('fileLogLevel', default)
+
+    def hasFileLogLevel(self):
+        return self.has_key('fileLogLevel')
+
+    def getDefaultLogRecordFormat(self):
+        return self['defaultLogRecordFormat']
+
+    def setLogRecordFormat(self, format):
+        self['logRecordFormat'] = format
+
+    def getLogRecordFormat(self, default='__dm_default__'):
+        return self.__getKeyValue('logRecordFormat', default)
+
+    def hasLogRecordFormat(self):
+        return self.has_key('logRecordFormat')
+
+    def getDefaultLogDateFormat(self):
+        return self['defaultLogDateFormat']
+
+    def setLogDateFormat(self, format):
+        self['logDateFormat'] = format
+
+    def getLogDateFormat(self, default='__dm_default__'):
+        return self.__getKeyValue('logDateFormat', default)
+
+    def hasLogDateFormat(self):
+        return self.has_key('logDateFormat')
+
+    def getDefaultCherrypyLogLevel(self):
+        return self['defaultCherrypyLogLevel']
+
+    def setCherrypyLogLevel(self, level):
+        self['cherrypyLogLevel'] = level 
+
+    def getCherrypyLogLevel(self, default='__dm_default__'):
+        return self.__getKeyValue('cherrypyLogLevel', default)
+
+    def hasCherrypyLogLevel(self):
+        return self.has_key('cherrypyLogLevel')
+
+    def getDefaultCherrypyLogCherrypy(self):
+        return self['defaultCherrypyLogFile']
+
+    def setCherrypyLogFile(self, cherrypyLogFile):
+        self['cherrypyLogFile'] = cherrypyLogFile
+
+    def getCherrypyLogFile(self, default='__dm_default__'):
+        return self.__getKeyValue('cherrypyLogFile', default)
+
+    def hasCherrypyLogFile(self):
+        return self.has_key('cherrypyLogFile')
+
+    def getDefaultCherrypyAccessFile(self):
+        return self['defaultCherrypyAccessFile']
+
+    def setCherrypyAccessFile(self, cherrypyAccessFile):
+        self['cherrypyAccessFile'] = cherrypyAccessFile
+
+    def getCherrypyAccessFile(self, default='__dm_default__'):
+        return self.__getKeyValue('cherrypyAccessFile', default)
+
+    def hasCherrypyAccessFile(self):
+        return self.has_key('cherrypyAccessFile')
+
+    def isDbAvailable(self):
+        if os.access(self.getDbPasswordFile(), os.R_OK):
+            return True
+        return False
+
+    def getDefaultServiceProtocol(self):
+        return self['defaultServiceProtocol']
+
+    def setServiceProtocol(self, serviceProtocol):
+        self['serviceProtocol'] = serviceProtocol
+
+    def getServiceProtocol(self, default='__dm_default__'):
+        return self.__getKeyValue('serviceProtocol', default)
+
+    def hasServiceProtocol(self):
+        return self.has_key('serviceProtocol')
+
+    def getDefaultServicePort(self):
+        return self['defaultServicePort']
+
+    def setServicePort(self, servicePort):
+        self['servicePort'] = servicePort 
+
+    def getServicePort(self, default='__dm_default__'):
+        return int(self.__getKeyValue('servicePort', default))
+
+    def hasServicePort(self):
+        return self.has_key('servicePort')
+
+    def getDefaultServiceHost(self):
+        return self['defaultServiceHost']
+
+    def setServiceHost(self, serviceHost):
+        self['serviceHost'] = serviceHost
+
+    def getServiceHost(self, default='__dm_default__'):
+        return self.__getKeyValue('serviceHost', default)
+
+    def hasServiceHost(self):
+        return self.has_key('serviceHost')
+
+    def getDefaultWebServiceProtocol(self):
+        return self['defaultWebServiceProtocol']
+
+    def setWebServiceProtocol(self, webServiceProtocol):
+        self['webServiceProtocol'] = webServiceProtocol
+
+    def getWebServiceProtocol(self, default='__dm_default__'):
+        return self.__getKeyValue('webServiceProtocol', default)
+
+    def hasWebServiceProtocol(self):
+        return self.has_key('webServiceProtocol')
+
+    def getDefaultDaqWebServicePort(self):
+        return self['defaultDaqWebServicePort']
+
+    def setDaqWebServicePort(self, daqWebServicePort):
+        self['daqWebServicePort'] = daqWebServicePort 
+
+    def getDaqWebServicePort(self, default='__dm_default__'):
+        return int(self.__getKeyValue('daqWebServicePort', default))
+
+    def hasDaqWebServicePort(self):
+        return self.has_key('daqWebServicePort')
+
+    def getDefaultDaqWebServiceHost(self):
+        return self['defaultDaqWebServiceHost']
+
+    def setDaqWebServiceHost(self, daqWebServiceHost):
+        self['daqWebServiceHost'] = daqWebServiceHost
+
+    def getDaqWebServiceHost(self, default='__dm_default__'):
+        return self.__getKeyValue('daqWebServiceHost', default)
+
+    def hasDaqWebServiceHost(self):
+        return self.has_key('daqWebServiceHost')
+
+    def getDefaultDsWebServicePort(self):
+        return self['defaultDsWebServicePort']
+
+    def setDsWebServicePort(self, dsWebServicePort):
+        self['dsWebServicePort'] = dsWebServicePort 
+
+    def getDsWebServicePort(self, default='__dm_default__'):
+        return int(self.__getKeyValue('dsWebServicePort', default))
+
+    def hasDsWebServicePort(self):
+        return self.has_key('dsWebServicePort')
+
+    def getDefaultDsWebServiceHost(self):
+        return self['defaultDsWebServiceHost']
+
+    def setDsWebServiceHost(self, dsWebServiceHost):
+        self['dsWebServiceHost'] = dsWebServiceHost
+
+    def getDsWebServiceHost(self, default='__dm_default__'):
+        return self.__getKeyValue('dsWebServiceHost', default)
+
+    def hasDsWebServiceHost(self):
+        return self.has_key('dsWebServiceHost')
+
+    def getDefaultCatWebServicePort(self):
+        return self['defaultCatWebServicePort']
+
+    def setCatWebServicePort(self, catWebServicePort):
+        self['catWebServicePort'] = catWebServicePort 
+
+    def getCatWebServicePort(self, default='__dm_default__'):
+        return int(self.__getKeyValue('catWebServicePort', default))
+
+    def hasCatWebServicePort(self):
+        return self.has_key('catWebServicePort')
+
+    def getDefaultCatWebServiceHost(self):
+        return self['defaultCatWebServiceHost']
+
+    def setCatWebServiceHost(self, catWebServiceHost):
+        self['catWebServiceHost'] = catWebServiceHost
+
+    def getCatWebServiceHost(self, default='__dm_default__'):
+        return self.__getKeyValue('catWebServiceHost', default)
+
+    def hasCatWebServiceHost(self):
+        return self.has_key('catWebServiceHost')
+
+    def getDefaultProcWebServicePort(self):
+        return self['defaultProcWebServicePort']
+
+    def setProcWebServicePort(self, procWebServicePort):
+        self['procWebServicePort'] = procWebServicePort 
+
+    def getProcWebServicePort(self, default='__dm_default__'):
+        return int(self.__getKeyValue('procWebServicePort', default))
+
+    def hasProcWebServicePort(self):
+        return self.has_key('procWebServicePort')
+
+    def getDefaultProcWebServiceHost(self):
+        return self['defaultProcWebServiceHost']
+
+    def setProcWebServiceHost(self, procWebServiceHost):
+        self['procWebServiceHost'] = procWebServiceHost
+
+    def getProcWebServiceHost(self, default='__dm_default__'):
+        return self.__getKeyValue('procWebServiceHost', default)
+
+    def hasProcWebServiceHost(self):
+        return self.has_key('procWebServiceHost')
+
+    def getDefaultDb(self):
+        return self['defaultDb']
+
+    def setDb(self, db):
+        self['db'] = db
+
+    def getDb(self, default='__dm_default__'):
+        return self.__getKeyValue('db', default) 
+
+    def hasDb(self):
+        return self.has_key('db')
+
+    def getDefaultDbHost(self):
+        return self['defaultDbHost']
+
+    def setDbHost(self, dbHost):
+        self['dbHost'] = dbHost
+
+    def getDbHost(self, default='__dm_default__'):
+        return self.__getKeyValue('dbHost', default) 
+
+    def hasDbHost(self):
+        return self.has_key('dbHost')
+
+    def getDefaultDbPort(self):
+        return self['defaultDbPort']
+
+    def setDbPort(self, dbPort):
+        self['dbPort'] = dbPort
+
+    def getDbPort(self, default='__dm_default__'):
+        return self.__getKeyValue('dbPort', default) 
+
+    def hasDbPort(self):
+        return self.has_key('dbPort')
+
+    def getDefaultDbPassword(self):
+        return self['defaultDbPassword']
+
+    def setDbPassword(self, dbPassword):
+        self['dbPassword'] = dbPassword
+
+    def getDbPassword(self, default='__dm_default__'):
+        return self.__getKeyValue('dbPassword', default) 
+
+    def hasDbPassword(self):
+        return self.has_key('dbPassword')
+
+    def getDefaultDbPasswordFile(self):
+        return self['defaultDbPasswordFile']
+
+    def getDbPasswordFile(self, default='__dm_default__'):
+        return self.__getKeyValue('dbPasswordFile', default) 
+
+    def setDbPasswordFile(self, f):
+        self['dbPasswordFile'] = f
+
+    def hasDbPasswordFile(self):
+        return self.has_key('dbPasswordFile')
+
+    def getDefaultDbUser(self):
+        return self['defaultDbUser']
+
+    def getDbUser(self, default='__dm_default__'):
+        return self.__getKeyValue('dbUser', default) 
+
+    def setDbUser(self, dbUser):
+        self['dbUser'] = dbUser
+
+    def hasDbUser(self):
+        return self.has_key('dbUser')
+
+    def getDbSchema(self, default='__dm_default__'):
+        return self.__getKeyValue('dbSchema', default)
+
+    def getDefaultConfigFile(self):
+        return self['defaultConfigFile']
+
+    def setConfigFile(self, configFile):
+        self['configFile'] = configFile
+        # Must reinitialize config parser at this point
+        self.configParser = None
+
+    def getConfigFile(self, default='__dm_default__'):
+        return self.__getKeyValue('configFile', default)
+
+    def hasConfigFile(self):
+        return self.has_key('configFile')
+
+    def getDefaultContextRoot(self):
+        return self['defaultContextRoot']
+
+    def setContextRoot(self, contextRoot):
+        self['contextRoot'] = contextRoot
+
+    def getContextRoot(self, default='__dm_default__'):
+        return self.__getKeyValue('contextRoot', default)
+
+    def hasContextRoot(self):
+        return self.has_key('contextRoot')
+
+    def getDefaultSessionCacheFile(self):
+        return self['defaultSessionCacheFile']
+
+    def setSessionCacheFile(self, sessionCacheFile):
+        self['sessionCacheFile'] = sessionCacheFile
+
+    def getSessionCacheFile(self, default='__dm_default__'):
+        return self.__getKeyValue('sessionCacheFile', default)
+
+    def hasSessionCacheFile(self):
+        return self.has_key('sessionCacheFile')
+
+    def getDefaultRequireSessionCredentials(self):
+        return self['defaultRequireSessionCredentials']
+
+    def setRequireSessionCredentials(self, requireSessionCredentials):
+        self['requireSessionCredentials'] = requireSessionCredentials
+
+    def getRequireSessionCredentials(self, default='__dm_default__'):
+        return self.__getKeyValue('requireSessionCredentials', default)
+
+    def hasRequireSessionCredentials(self):
+        return self.has_key('requireSessionCredentials')
+
+    def getDefaultSslCaCertFile(self):
+        return self['defaultSslCaCertFile']
+
+    def setSslCaCertFile(self, sslCaCertFile):
+        self['sslCaCertFile'] = sslCaCertFile
+
+    def getSslCaCertFile(self, default='__dm_default__'):
+        return self.__getKeyValue('sslCaCertFile', default)
+
+    def hasSslCaCertFile(self):
+        return self.has_key('sslCaCertFile')
+
+    def getDefaultSslCertFile(self):
+        return self['defaultSslCertFile']
+
+    def setSslCertFile(self, sslCertFile):
+        self['sslCertFile'] = sslCertFile
+
+    def getSslCertFile(self, default='__dm_default__'):
+        return self.__getKeyValue('sslCertFile', default)
+
+    def hasSslCertFile(self):
+        return self.has_key('sslCertFile')
+
+    def getDefaultSslKeyFile(self):
+        return self['defaultSslKeyFile']
+
+    def setSslKeyFile(self, sslKeyFile):
+        self['sslKeyFile'] = sslKeyFile
+
+    def getSslKeyFile(self, default='__dm_default__'):
+        return self.__getKeyValue('sslKeyFile', default)
+
+    def hasSslKeyFile(self):
+        return self.has_key('sslKeyFile')
+
+    def getDefaultUsername(self):
+        return self['defaultUsername']
+
+    def setUsername(self, username):
+        self['username'] = username
+
+    def getUsername(self, default='__dm_default__'):
+        return self.__getKeyValue('username', default)
+
+    def hasUsername(self):
+        return self.has_key('username')
+
+    def getDefaultPassword(self):
+        return self['defaultPassword']
+
+    def setPassword(self, password):
+        self['password'] = password
+
+    def getPassword(self, default='__dm_default__'):
+        return self.__getKeyValue('password', default)
+
+    def hasPassword(self):
+        return self.has_key('password')
+
+    def getDefaultLoginFile(self):
+        return self['defaultLoginFile']
+
+    def getLoginFile(self, default='__dm_default__'):
+        return self.__getKeyValue('loginFile', default) 
+
+    def setLoginFile(self, f):
+        self['loginFile'] = f
+
+    def hasLoginFile(self):
+        return self.has_key('loginFile')
+
+    def getDefaultStationName(self):
+        return self['defaultStationName']
+
+    def getStationName(self, default='__dm_default__'):
+        return self.__getKeyValue('stationName', default) 
+
+    def setStationName(self, f):
+        self['stationName'] = f
+
+    def hasStationName(self):
+        return self.has_key('stationName')
+
+    def getDefaultAllowedExperimentTypes(self):
+        return self['defaultAllowedExperimentTypes']
+
+    def getAllowedExperimentTypes(self, default='__dm_default__'):
+        return self.__getKeyValue('allowedExperimentTypes', default) 
+
+    def setAllowedExperimentTypes(self, f):
+        self['allowedExperimentTypes'] = f
+
+    def hasAllowedExperimentTypes(self):
+        return self.has_key('allowedExperimentTypes')
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    cm = ConfigurationManager.getInstance()
+    print cm
+    print cm.getUsername()

src/python/dm/common/utility/consoleLoggingHandler.py

+#!/usr/bin/env python
+
+#
+# Console logging handler class
+#
+
+#######################################################################
+
+import socket
+import pwd
+import os
+from logging import StreamHandler
+
+#######################################################################
+
+class ConsoleLoggingHandler(StreamHandler):
+    """ Class that enables console logging. """
+    def __init__(self, *args):
+        StreamHandler.__init__(self, *args)
+        self.user = pwd.getpwuid(os.getuid())[0]
+        self.host = socket.gethostname()
+
+    def emit(self, record):
+        record.__dict__['user'] = self.user
+        record.__dict__['host'] = self.host
+        return StreamHandler.emit(self, record)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    import sys
+    import logging
+    exec 'sh = ConsoleLoggingHandler(sys.stdout,)'
+    sh.setLevel(logging.INFO)
+    rootLogger = logging.getLogger('')
+    logging.basicConfig(level=logging.DEBUG)
+
+    mainLogger = logging.getLogger('main')
+    mainLogger.debug("main debug")
+    mainLogger.info("main info")
+                                                                                                                        

src/python/dm/common/utility/cryptUtility.py

+#!/usr/bin/env python
+
+import random
+import string
+import crypt
+#import md5
+import hashlib
+import base64
+
+class CryptUtility:
+
+    CRYPT_TYPE = 6  # SHA-512 (man crypt)
+    SALT_CHARACTERS = string.lowercase + string.uppercase + string.digits
+    SALT_DELIMITER = '$'
+    SALT_LENGTH_IN_BYTES = 4
+
+    PBKDF2_ENCRYPTION = 'sha1' # use SHA-1 for compatibility with java
+    PBKDF2_KEY_LENGTH_IN_BYTES = 24
+    PBKDF2_ITERATIONS = 1003
+
+    @classmethod
+    def getRandomWord(cls, length):
+        return ''.join(random.choice(CryptUtility.SALT_CHARACTERS) for i in range(length))
+
+    @classmethod
+    def cryptPassword(cls, password):
+        """ Return crypted password. """
+        #calculator = md5.md5()
+        #calculator.update(salt)
+        #md5Salt = calculator.hexdigest()
+        #return crypt.crypt(cleartext, md5Salt)
+
+        salt = CryptUtility.getRandomWord(CryptUtility.SALT_LENGTH_IN_BYTES)
+        salt = '%s%s%s%s%s'.format(CryptUtility.SALT_DELIMITER, CryptUtility.CRYPT_TYPE, CryptUtility.SALT_DELIMITER, salt, CryptUtility.SALT_DELIMITER)
+        return crypt.crypt(password, salt)
+
+    @classmethod
+    def verifyPassword(cls, password, cryptedPassword):
+        """ Verify crypted password. """
+        return cryptedPassword == crypt.crypt(password, cryptedPassword)
+
+    @classmethod
+    def cryptPasswordWithPbkdf2(cls, password):
+        """ Crypt password with pbkdf2 package and encode with b64. """
+        salt = CryptUtility.getRandomWord(CryptUtility.SALT_LENGTH_IN_BYTES)
+        return cls.saltAndCryptPasswordWithPbkdf2(password, salt)
+
+    @classmethod
+    def saltAndCryptPasswordWithPbkdf2(cls, password, salt):
+        """ Crypt salted password with pbkdf2 package and encode with b64. """
+        cryptedPassword = hashlib.pbkdf2_hmac(
+            CryptUtility.PBKDF2_ENCRYPTION, 
+            password, salt, 
+            CryptUtility.PBKDF2_ITERATIONS, 
+            CryptUtility.PBKDF2_KEY_LENGTH_IN_BYTES)
+        encodedPassword = base64.b64encode(cryptedPassword)
+        return '%s%s%s' % (salt, CryptUtility.SALT_DELIMITER, encodedPassword)
+
+    @classmethod
+    def verifyPasswordWithPbkdf2(cls, password, cryptedPassword):
+        """ Verify crypted password. """
+        # Get salt
+        salt = '%s' % cryptedPassword.split(CryptUtility.SALT_DELIMITER)[0]
+        # Verify crypted password
+        return cryptedPassword == cls.saltAndCryptPasswordWithPbkdf2(password, salt)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    import sys
+    #password = "dm"
+    password = sys.argv[1]
+    print 'Clear text: ', password
+    #cryptedPassword = CryptUtility.cryptPassword(password)
+    #print 'Crypted: ', cryptedPassword
+    #isVerified = CryptUtility.verifyPassword(password, cryptedPassword)
+    #print 'Verify: ', isVerified
+
+    cryptedPassword = CryptUtility.cryptPasswordWithPbkdf2(password)
+    print 'Crypted: ', cryptedPassword
+    isVerified = CryptUtility.verifyPasswordWithPbkdf2(password, cryptedPassword)
+    print 'Verify: ', isVerified
+

src/python/dm/common/utility/dictUtility.py

+#!/usr/bin/env python
+
+import copy
+
+class DictUtility:
+
+    @classmethod
+    def getAndRemoveKey(cls, dict, key, default=None):
+        value = dict.get(key, default)
+        if dict.has_key(key):
+            del dict[key]
+        return value
+
+    @classmethod
+    def deepCopy(cls, dict, includeKeys=[], excludeKeys=[]):
+        dict2 = {}
+        if len(includeKeys):
+            for key in includeKeys:
+                value = dict.get(key)
+                if value is not None:
+                    dict2[key] = copy.deepcopy(value)
+        elif len(excludeKeys):
+            for key in dict.keys():
+                if key not in excludeKeys:
+                    dict2[key] = copy.deepcopy(dict[key])
+        else:
+            dict2 = copy.deepcopy(dict)
+        return dict2
+
+    @classmethod
+    def deepCopy2(cls, dict, includeKeys=[], excludeKeys=[]):
+        dict2 = copy.deepcopy(dict)
+        if len(includeKeys):
+            for key in dict2.keys():
+                if not key in includeKeys:
+                    del dict2[key]
+        elif len(excludeKeys):
+            for key in excludeKeys:
+                if dict2.has_key(key):
+                    del dict2[key]
+        return dict2
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    print DictUtility.deepCopy({'k1': '1', 'k2' : '2'}, excludeKeys=['k1'])
+    print  DictUtility.deepCopy({'k1': '1', 'k2' : '2'}, includeKeys=['k1'])
+    print  DictUtility.deepCopy({'k1': '1', 'k2' : '2'})
+

src/python/dm/common/utility/dmModuleManager.py

+#!/usr/bin/env python
+
+#
+# Module manager class.
+#
+
+#######################################################################
+
+import threading
+
+#######################################################################
+
+class DmModuleManager:
+    """ Singleton class used for managing dm modules. """
+
+    # Get singleton instance.
+    @classmethod
+    def getInstance(cls):
+        from dm.common.utility.dmModuleManager import DmModuleManager
+        try:
+            mgr = DmModuleManager()
+        except DmModuleManager, ex:
+            mgr = ex
+        return mgr
+
+    # Singleton.
+    __instanceLock = threading.RLock()
+    __instance = None
+
+    def __init__(self):
+        DmModuleManager.__instanceLock.acquire()
+        try:
+            if DmModuleManager.__instance:
+                raise DmModuleManager.__instance
+            DmModuleManager.__instance = self
+            from dm.common.utility.loggingManager import LoggingManager
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+            self.lock = threading.RLock()
+            self.moduleList = []
+            self.modulesRunning = False
+        finally:
+            DmModuleManager.__instanceLock.release()
+
+    def addModule(self, m):
+        self.lock.acquire()
+        try:
+            self.logger.debug('Adding dm module: %s' % m.__class__.__name__)
+            self.moduleList.append(m)
+        finally:
+            self.lock.release()
+
+    def startModules(self):
+        self.lock.acquire()
+        try:
+            if self.modulesRunning:
+                return
+            for m in self.moduleList:
+                self.logger.debug('Starting dm module: %s' % m.__class__.__name__)
+                m.start()
+            self.modulesRunning = True
+        finally:
+            self.lock.release()
+
+    def stopModules(self):
+        self.lock.acquire()
+        try:
+            if not self.modulesRunning:
+                return
+            n = len(self.moduleList)
+            for i in range(0, n):
+                m = self.moduleList[n-1-i]
+                self.logger.debug('Stopping dm module: %s' % m.__class__.__name__)
+                m.stop()
+            self.modulesRunning = False
+        finally:
+            self.lock.release()
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/utility/dmSubprocess.py

+#!/usr/bin/env python
+
+# 
+# Subprocess class
+#
+
+import os
+import subprocess
+import platform
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.commandFailed import CommandFailed
+
+#######################################################################
+
+class DmSubprocess(subprocess.Popen):
+
+    # Get subprocess instance.
+    @classmethod
+    def getSubprocess(cls, command, cwd=None):
+        close_fds = True
+        if platform.system() != 'Windows':
+            close_fds = False
+        p = DmSubprocess(command, close_fds=close_fds, cwd=cwd)
+        return p
+
+    # Execute command
+    @classmethod
+    def executeCommand(cls, command, cwd=None):
+        """ Create subprocess and run it, return subprocess object. """
+        p = cls.getSubprocess(command, cwd)
+        p.run()
+        return p
+
+    # Execute command, ignore errors.
+    @classmethod
+    def executeCommandAndIgnoreFailure(cls, command, cwd=None):
+        """ Create subprocess, run it, igore any failures, and return subprocess object. """
+        p = cls.getSubprocess(command, cwd)
+        try:
+            p.run()
+        except CommandFailed, ex:
+            p.getLogger().debug('Command failed, stdout: %s, stderr: %s' % (p.getStdOut(), p.getStdErr()))
+        return p
+
+    @classmethod
+    def executeCommandAndLogToStdOut(cls, command, cwd=None):
+        """ Execute command, display output to stdout, maintain log file and return subprocess object. """
+        p = cls.getSubprocess(command, cwd)
+        p.__commandLog()
+
+        while True:
+            outp = p.stdout.readline()
+            if not outp:
+                break
+            print outp,
+
+        retval = p.wait()
+
+        p.logger.debug('Exit status: %s' % retval)
+
+        if retval != 0:
+            error = ''
+            while True:
+                err = p.stderr.readline()
+                if not err:
+                    break
+                error += err
+            raise CommandFailed(error)
+        return p
+
+    def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=None, close_fds=False, shell=True, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, useExceptions=True, quietMode=False):
+        """ Overrides Popen constructor with more appropriate defaults. """
+        subprocess.Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags)
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        self._stdout = None
+        self._stderr = None
+        self._args = args
+        self._cwd = cwd
+        self.useExceptions = useExceptions
+        self.quietMode = quietMode
+
+    def __commandLog(self):
+        # Not very useful to show the name of this file.
+        # Walk up the stack to find the caller.
+        import traceback
+        stack =  traceback.extract_stack()
+        for i in range(2, len(stack)):
+            if stack[-i][0] != stack[-1][0]:
+                fileName, lineNumber, functionName, text = stack[-i]
+                break
+            else:
+                fileName = lineNumber = functionName = text = '?'
+
+        self.logger.debug('From [%s:%s] Invoking: [%s]' % (os.path.basename(fileName), lineNumber, self._args))
+
+    def run(self, input=None):
+        """ Run subprocess. """
+        if not self.quietMode:
+            self.__commandLog()
+        (self._stdout, self._stderr) = subprocess.Popen.communicate(self, input)
+        if not self.quietMode:
+            self.logger.debug('Exit status: %s' % self.returncode)
+        if self.returncode != 0 and self.useExceptions:
+            if not self.quietMode:
+                self.logger.debug('StdOut: %s' % self._stdout)
+                self.logger.debug('StdErr: %s' % self._stderr)
+            error = self._stderr.strip()
+            if error == '':
+                error = self._stdout.strip()
+            raise CommandFailed('%s' % (error))
+        return (self._stdout, self._stderr) 
+
+    def getLogger(self):
+        return self.logger
+
+    def getArgs(self):
+        return self._args
+
+    def getStdOut(self):
+        return self._stdout
+
+    def getStdErr(self):
+        return self._stderr
+
+    def getExitStatus(self):
+        return self.returncode
+
+    def toDict(self):
+        result = { 
+            'command' : self._args,
+            'exitStatus' : self.returncode,
+            'stdErr' : self._stderr,
+            'stdOut' : self._stdout,
+            'workingDir' : self._cwd
+        }
+        return result
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    p = DmSubprocess('ls -l', useExceptions=False)
+    print p.toDict()
+    p.run()
+    #print p.getStdOut()
+    #print p.getStdErr()
+    #print p.getExitStatus()
+    print p.toDict()
+

src/python/dm/common/utility/encoder.py

+#!/usr/bin/env python
+
+import base64
+
+class Encoder:
+
+    @classmethod
+    def encode(cls, data):
+        # Encode twice, in order to avoid issues like '+' being
+        # interpreted as space after decoding
+        if not data:
+            return data
+        encodedData = base64.b64encode(base64.encodestring('%s' % data))
+        return encodedData
+
+    @classmethod
+    def decode(cls, encodedData):
+        if not encodedData:
+            return encodedData
+        data = base64.decodestring(base64.b64decode('%s' % encodedData))
+        return data
+
+    @classmethod
+    def encodeDict(cls, dict, excludedKeyList=[]):
+        encodedDict = {}
+        for (key,value) in dict.items():
+            if key not in excludedKeyList:
+                encodedDict[key] = cls.encode(value)
+            else:
+                encodedDict[key] = value
+        return encodedDict
+
+    @classmethod
+    def decodeDict(cls, encodedDict, excludedKeyList=[]):
+        dict = {}
+        for (key,value) in encodedDict.items():
+            if key not in excludedKeyList:
+                dict[key] = cls.decode(value)
+            else:
+                dict[key] = value
+        return dict

src/python/dm/common/utility/fileUtility.py

+#!/usr/bin/env python
+
+import os
+import stat
+from dm.common.utility.dmSubprocess import DmSubprocess
+from dm.common.utility.timeUtility import TimeUtility
+
+class FileUtility:
+
+    @classmethod
+    def getMd5Sum(cls, filePath, fileInfo={}):
+        p = DmSubprocess.getSubprocess('md5sum "%s"' % filePath)
+        p.run()
+        md5Sum = p.getStdOut().split()[0]
+        fileInfo['md5Sum'] = md5Sum
+        fileInfo['filePath'] = filePath
+        return fileInfo
+
+    @classmethod
+    def statFile(cls, filePath, fileInfo={}):
+        statResult = os.stat(filePath)
+        fileInfo['filePath'] = filePath
+        fileInfo['fileSize'] = statResult[stat.ST_SIZE]
+        fileInfo['fileCreationTime'] = statResult[stat.ST_CTIME]
+        fileInfo['fileAccessTime'] = statResult[stat.ST_ATIME]
+        fileInfo['fileModificationTime'] = statResult[stat.ST_MTIME]
+        return fileInfo
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    print FileUtility.getMd5Sum('/tmp/file')
+    print FileUtility.statFile('/tmp/file')
+

src/python/dm/common/utility/ftpUtility.py

+#!/usr/bin/env python
+
+import copy
+import os
+import time
+from ftplib import FTP
+from dm.common.utility.timeUtility import TimeUtility
+from dm.common.utility.loggingManager import LoggingManager
+import urlparse
+
+class FtpUtility:
+
+    def __init__(self, host, port, username=None, password=None, serverUsesUtcTime=True):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.ftpClient = None
+        self.serverUsesUtcTime = serverUsesUtcTime 
+        self.mlsdFileStatDict = {}
+
+    @classmethod
+    def parseFtpUrl(cls, url, defaultHost=None, defaultPort=None):
+        host = defaultHost
+        port = defaultPort
+        scheme = None
+        dirPath = url
+        if url.startswith('ftp://'):
+            parseResult = urlparse.urlparse(url)
+            scheme = parseResult.scheme
+            netlocTokens = parseResult.netloc.split(':')
+            host = netlocTokens[0]
+            if len(netlocTokens) > 1:
+                port = int(netlocTokens[1])
+            dirPath = parseResult.path
+        return (scheme, host, port, dirPath)
+
+    @classmethod
+    def getFtpClient(cls, host, port, username=None, password=None):
+        ftp = FTP()
+        ftp.connect(host, port)
+        ftp.login(username, password)
+        return ftp
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+
+    def __parseKeyValue(cls, origKeyValue, outputDict={}, newKey=None):
+        key,value = origKeyValue.split('=')
+        value = value.strip()
+        if newKey is not None:
+            key = newKey
+        outputDict[key] = value
+        return outputDict
+        
+    def __parseMlsdOutput(self, line):
+        # ['Type=dir', 'Modify=20151018024430', 'Size=4096', 'Perm=el', 'UNIX.mode=0775', 'UNIX.owner=sveseli', 'UNIX.uid=500', 'UNIX.group=sveseli', 'UNIX.gid=500', 'Unique=fd00-c2e3e', ' dir2\r']
+        parts = line.split(';')
+        parseDict = {}
+        self.__parseKeyValue(parts[0], parseDict)
+        self.__parseKeyValue(parts[1], parseDict)
+        self.__parseKeyValue(parts[2], parseDict, 'fileSize')
+        name = parts[-1].strip()
+        parseDict['Name'] = name
+        type = parseDict.get('Type', '')
+        if type == 'dir' :
+            self.mlsdDirList.append(name)
+        elif type == 'file':
+            self.mlsdFileDict[name] = parseDict
+
+    def __parseMlsdFileStat(self, line):
+        # ['Type=dir', 'Modify=20151018024430', 'Size=4096', 'Perm=el', 'UNIX.mode=0775', 'UNIX.owner=sveseli', 'UNIX.uid=500', 'UNIX.group=sveseli', 'UNIX.gid=500', 'Unique=fd00-c2e3e', ' dir2\r']
+        parts = line.split(';')
+        parseDict = {}
+        self.__parseKeyValue(parts[0], parseDict)
+        self.__parseKeyValue(parts[1], parseDict)
+        self.__parseKeyValue(parts[2], parseDict, 'fileSize')
+        name = parts[-1].strip()
+        self.mlsdFileStatDict[name] = parseDict
+
+    def __processFileStatDict(self, fileStatDict):
+        modifyTime = fileStatDict.get('Modify')
+        modifyTime = time.mktime(time.strptime(modifyTime, '%Y%m%d%H%M%S'))
+        if self.serverUsesUtcTime:
+            modifyTime = TimeUtility.utcToLocalTime(modifyTime)
+        fileStatDict['fileModificationTime'] = modifyTime
+        fileStatDict['fileSize'] = int(fileStatDict.get('fileSize'))
+        del fileStatDict['Modify']
+        del fileStatDict['Type']
+
+    def getFiles2(self, dirPath, fileDict={}, replacementDirPath=None, initialCall=True):
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        # Need these to be class members for the callback function
+        self.mlsdFileDict = {}
+        self.mlsdDirList = []
+        self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput)
+        if not replacementDirPath:
+            replacementDirPath = dirPath
+        for (fileName,fileInfo) in self.mlsdFileDict.items():
+            self.__processFileStatDict(fileInfo)
+            del fileInfo['Name']
+            filePath = '%s/%s' % (replacementDirPath, fileName)
+            fileDict[filePath] = fileInfo
+        mlsdDirList = copy.copy(self.mlsdDirList)
+        for d in mlsdDirList:
+            dirPath2 = '%s/%s' % (dirPath,d)
+            replacementDirPath2 = '%s/%s' % (replacementDirPath,d)
+            self.getFiles2(dirPath2,fileDict, replacementDirPath2, initialCall=False)
+        if initialCall:
+            del self.mlsdFileDict
+            del self.mlsdDirList
+        return fileDict
+
+    def getFiles(self, dirPath, fileDict={}, replacementDirPath=None):
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        # Need these to be class members for the callback function
+        self.mlsdFileDict = {}
+        self.mlsdDirList = []
+        self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput)
+        if not replacementDirPath:
+            replacementDirPath = dirPath
+        for (fileName,fileInfo) in self.mlsdFileDict.items():
+            filePath = '%s/%s' % (replacementDirPath, fileName)
+            modifyTime = fileInfo.get('Modify')
+            modifyTime = time.mktime(time.strptime(modifyTime, '%Y%m%d%H%M%S'))
+            if self.serverUsesUtcTime:
+                modifyTime = TimeUtility.utcToLocalTime(modifyTime)
+            fileSize = int(fileInfo.get('fileSize'))
+            #print '%s %s %s' % (filePath, fileSize, modifyTime)
+            fileDict[filePath] = {'fileSize' : fileSize, 'fileModificationTime' : modifyTime}
+        self.mlsdFileDict.clear()
+        mlsdDirList = copy.copy(self.mlsdDirList)
+        for d in mlsdDirList:
+            dirPath2 = '%s/%s' % (dirPath,d)
+            replacementDirPath2 = '%s/%s' % (replacementDirPath,d)
+            self.getFiles(dirPath2, fileDict, replacementDirPath2)
+        self.mlsdDirList = []
+        return fileDict
+
+    def getMd5Sum(self, filePath, fileInfo={}):
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        md5Sum = self.ftpClient.sendcmd('CKSM MD5 0 -1 %s' % filePath).split()[-1]
+        fileInfo['md5Sum'] = md5Sum
+        return md5Sum
+
+    def statFile(self, filePath, fileInfo={}):
+        fileName = os.path.basename(filePath)
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        # Need this to be class members for the callback function
+        self.ftpClient.retrlines('MLSD %s' % filePath, self.__parseMlsdFileStat)
+        fileStatDict = self.mlsdFileStatDict.get(fileName)
+        if fileStatDict:
+            self.__processFileStatDict(fileStatDict)
+            fileInfo.update(fileStatDict)
+            del self.mlsdFileStatDict[fileName]
+        return fileInfo
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    print "Round 1: "
+    ftpUtility = FtpUtility('s33dserv', 2811)
+    #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
+    files = ftpUtility.getFiles2('/export/dm/test')
+    print files
+    files = ftpUtility.getFiles('/export/dm/test')
+    print ftpUtility.parseFtpUrl('/export/dm/test')
+    print files
+    #files = ftpUtility.getFiles('/export/7IDSprayimage/Cummins/Data')
+    #files = ftpUtility.getFiles2('/export/8-id-i/test', replacementDirPath='/data/testing/8-id-i')
+    #print "Number of files: ", len(files)
+    #time.sleep(60)
+    #print "Removing files"
+    #del files
+    #print "Files removed"
+    #time.sleep(60)
+    #del ftpUtility
+    #print "Utility removed"
+    #time.sleep(60)
+    #print "Round 2: "
+    #ftpUtility = FtpUtility('s7dserv', 2811)
+    #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
+    print ftpUtility.getMd5Sum('/export/dm/test/testfile01')
+    #print ftpUtility.statFile('/export/8-id-i/test/testfile01')
+    #ftpUtility = FtpUtility('xstor-devel', 22)
+    #files = ftpUtility.getFiles2('/data/testing')
+    #print files

src/python/dm/common/utility/ldapClient.py

+#!/usr/bin/env python
+
+import ldap
+import re
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.exceptions.internalError import InternalError
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+from dm.common.exceptions.dmException import DmException
+
+class LdapClient:
+
+    LDAP_ERROR_REGEX_LIST = [
+        (re.compile('.*No such object.*'),ObjectNotFound),
+        (re.compile('.*'),DmException)
+    ]
+
+    def __init__(self, serverUrl, adminDn, adminPasswordFile):
+        self.serverUrl = serverUrl
+        self.adminDn = adminDn
+        self.adminPassword = open(adminPasswordFile, 'r').readline().strip()
+        if not self.adminPassword:
+            raise ConfigurationError('LDAP password could not be found in %s file' % adminPasswordFile) 
+        self.ldapClient = None
+
+    # Decorator for all LDAP transactions
+    @classmethod
+    def executeLdapCall(cls, func):
+        def ldapCall(*args, **kwargs):
+            try:
+                result = func(*args, **kwargs)
+                return result
+            except ldap.NO_SUCH_OBJECT, ex:
+                raise ObjectNotFound(str(ex))
+            except DmException, ex:
+                raise
+            except Exception, ex:
+                error = str(ex)
+                for (pattern,dmExClass) in LdapClient.LDAP_ERROR_REGEX_LIST:
+                    if pattern.match(error):
+                        raise dmExClass(exception=ex)
+                raise DmException(exception=ex)
+        return ldapCall
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    def getLdapClient(self):
+        if self.ldapClient is not None:
+            try:
+                self.ldapClient.simple_bind_s(self.adminDn, self.adminPassword)
+            except Exception, ex:
+                self.getLogger().error('Invalidating LDAP client due to error: %s' % ex)
+                self.unbind(self.ldapClient)
+                self.ldapClient = None
+
+        if not self.ldapClient:
+            self.ldapClient = self.bind(self.serverUrl, self.adminDn, self.adminPassword)
+        return self.ldapClient
+
+    @classmethod 
+    def bind(cls, serverUrl, adminDn, adminPassword):
+        try:
+            ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+            ldapClient = ldap.initialize(serverUrl)
+            ldapClient.set_option(ldap.OPT_REFERRALS,0)
+            ldapClient.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
+            ldapClient.simple_bind_s(adminDn, adminPassword)
+            cls.getLogger().debug('Successful binding with LDAP DN: %s' % adminDn)
+            return ldapClient
+        except ldap.INVALID_CREDENTIALS, ex:
+            ldapClient.unbind()
+            raise AuthenticationError('Invalid LDAP credentials for admin user %s' % adminDn)
+        except ldap.SERVER_DOWN, ex:
+            raise CommunicationError('Cannot reach LDAP server %s' % serverUrl)
+        except Exception, ex:
+            raise InternalError('Unrecognized exception while binding to LDAP server %s: %s' % (serverUrl, ex))
+
+    @classmethod 
+    def unbind(cls, ldapClient):
+        try:
+            ldapClient.unbind()
+        except Exception, ex:
+            cls.getLogger().error('Could not unbind LDAP client: %s' % ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass
+

src/python/dm/common/utility/ldapLinuxPlatformUtility.py

+#!/usr/bin/env python
+
+import grp
+import ldap
+import ldap.modlist
+import copy
+import threading
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.exceptions.internalError import InternalError
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+from dmSubprocess import DmSubprocess
+
+class LdapLinuxPlatformUtility:
+
+    SUDO_CMD = '/usr/bin/sudo'
+    GROUPADD_CMD = '/usr/sbin/groupadd'
+    USERMOD_CMD = '/usr/sbin/usermod'
+    SETFACL_CMD = '/usr/bin/setfacl'
+    CHOWN_CMD = '/bin/chown'
+    GPASSWD_CMD = '/usr/bin/gpasswd'
+    NSCD_CMD = '/usr/sbin/nscd'
+    FIND_CMD = '/bin/find'
+
+    CONFIG_SECTION_NAME = 'LdapLinuxPlatformUtility'
+    REFRESH_AUTH_FILES_COMMAND_KEY = 'refreshauthfilescommand'
+    TIMER_DELAY_IN_SECONDS = 10
+
+    def __init__(self, serverUrl, adminDn, adminPasswordFile, groupDnFormat, minGidNumber=None):
+        self.serverUrl = serverUrl
+        self.adminDn = adminDn
+        self.groupDnFormat = groupDnFormat
+        self.minGidNumber = minGidNumber
+        self.refreshAuthFilesCommand = None
+        self.refreshAuthFilesTimer = None
+        self.getLogger().debug('Min GID number: %s' % minGidNumber)
+        # Remove first entry from the dn format to get tree base
+        self.groupBaseDn = ','.join(groupDnFormat.split(',')[1:])
+        self.getLogger().debug('Group base DN: %s' % self.groupBaseDn)
+        self.getLogger().debug('Using LDAP Admin password file: %s' % adminPasswordFile)
+        self.adminPassword = open(adminPasswordFile, 'r').readline().strip()
+        if not self.adminPassword:
+            raise ConfigurationError('LDAP password could not be found in %s file' % adminPasswordFile) 
+        self.ldapClient = None
+        self.__configure()
+
+    def __configure(self):
+        cm = ConfigurationManager.getInstance()
+        configItems = cm.getConfigItems(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME)
+        self.getLogger().debug('Got config items: %s' % configItems)
+        self.refreshAuthFilesCommand = cm.getConfigOption(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME, LdapLinuxPlatformUtility.REFRESH_AUTH_FILES_COMMAND_KEY)
+        self.getLogger().debug('Refresh auth files command: %s' % self.refreshAuthFilesCommand)
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    @classmethod
+    def executeSudoCommand(cls, cmd):
+        p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
+        p.run()
+
+    @classmethod
+    def executeCommand(cls, cmd):
+        p = DmSubprocess('%s' % (cmd))
+        p.run()
+
+    def getLdapClient(self):
+        if self.ldapClient is not None:
+            try:
+                self.ldapClient.simple_bind_s(self.adminDn, self.adminPassword)
+            except Exception, ex:
+                self.getLogger().error('Invalidating LDAP client due to error: %s' % ex)
+                self.unbind(self.ldapClient)
+                self.ldapClient = None
+
+        if not self.ldapClient:
+            self.ldapClient = self.bind(self.serverUrl, self.adminDn, self.adminPassword)
+        return self.ldapClient
+
+    @classmethod 
+    def bind(cls, serverUrl, adminDn, adminPassword):
+        try:
+            ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+            ldapClient = ldap.initialize(serverUrl)
+            ldapClient.set_option(ldap.OPT_REFERRALS,0)
+            ldapClient.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
+            ldapClient.simple_bind_s(adminDn, adminPassword)
+            cls.getLogger().debug('Successful binding with LDAP DN: %s' % adminDn)
+            return ldapClient
+        except ldap.INVALID_CREDENTIALS, ex:
+            ldapClient.unbind()
+            raise AuthenticationError('Invalid LDAP credentials for admin user %s' % adminDn)
+        except ldap.SERVER_DOWN, ex:
+            raise CommunicationError('Cannot reach LDAP server %s' % serverUrl)
+        except Exception, ex:
+            raise InternalError('Unrecognized exception while binding to LDAP server %s: %s' % (serverUrl, ex))
+
+    @classmethod 
+    def unbind(cls, ldapClient):
+        try:
+            ldapClient.unbind()
+        except Exception, ex:
+            cls.getLogger().error('Could not unbind LDAP client: %s' % ex)
+
+    def createGroup(self, name):
+        """ Create group if it does not exist. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        name = str(name)
+        try:
+            groupDn = self.groupDnFormat % name
+            logger.debug('Looking for group DN: %s' % groupDn)
+            # this method will throw exception if group is not found
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            logger.debug('Group %s already exists' % groupTuple[0])
+            return
+        except ldap.NO_SUCH_OBJECT, ex:
+            logger.debug('Group DN %s must be created' % groupDn)
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+        # determine gidNumber: look through all entries to get max value,
+        # then increment it
+        # ldap should really be configured to handle gid's automatically,
+        # and should prevent invalid entries
+        try:
+            logger.debug('Looking for max group id')
+            resultList = ldapClient.search_s(self.groupBaseDn, ldap.SCOPE_ONELEVEL, attrlist=['gidNumber'])
+            maxGid = 0
+            if self.minGidNumber:
+                maxGid = self.minGidNumber 
+            for result in resultList:
+                gidList = result[1].get('gidNumber', [])
+                gid = 0
+                if gidList:
+                    gid = int(gidList[0])
+
+                if gid > maxGid:
+                    maxGid = gid
+            gidNumber = str(maxGid + 1)
+            logger.debug('Max GID is %s, new group id will be %s' % (maxGid, gidNumber)) 
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        
+        attrs = {}
+        attrs['objectclass'] = ['posixGroup','top']
+        attrs['cn'] = name
+        attrs['gidNumber'] = [gidNumber]
+        attrs['memberUid'] = []
+        try:
+            groupLdif = ldap.modlist.addModlist(attrs)
+            ldapClient.add_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add group %s: %s' % (groupDn, ex))
+            raise InternalError(exception=ex)
+
+    def addUserToGroup(self, username, groupName):
+        """ Add user to group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username in memberUidList:
+                logger.debug('Group %s already contains user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Adding user %s to group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.append(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+        # Refresh auth files
+        self.refreshAuthFiles()
+
+    def deleteUserFromGroup(self, username, groupName):
+        """ Remove user from group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username not in memberUidList:
+                logger.debug('Group %s does not contain user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Removing user %s from group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.remove(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+        # Refresh auth files
+        self.refreshAuthFiles()
+
+    @classmethod
+    def createLocalGroup(cls, name):
+        """ Create local group if it does not exist. """
+        logger = cls.getLogger()
+        try:
+            group = grp.getgrnam(name)
+            logger.debug('Group %s already exists' % name)
+            return
+        except KeyError, ex:
+            # ok, we need to create group
+            pass
+        logger.debug('Creating group %s' % name)
+        cmd = '%s %s' % (cls.GROUPADD_CMD, name)
+        cls.executeSudoCommand(cmd)
+
+    def addLocalUserToGroup(self, username, groupName):
+        """ Add local user to group. """
+        self.addUserToGroup(username, groupName)
+
+    def deleteLocalUserFromGroup(self, username, groupName):
+        """ Remove local user from group. """
+        self.deleteUserFromGroup(username, groupName)
+
+    def getGroupInfo(self, groupName):
+        """ Get given group info. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            return groupTuple
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+    def setGroupUsers(self, groupName, usernameList):
+        """ Set list of users for a given group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Setting users %s for group %s' % (usernameList, groupName))
+        memberUidList = []
+        for username in usernameList:
+            memberUidList.append(str(username))
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not set users %s for group %s: %s' % (usernameList, groupName, ex))
+            raise InternalError(exception=ex)
+
+        # Refresh auth files 
+        self.refreshAuthFiles()
+
+    @classmethod
+    def setPathReadExecutePermissionsForGroup(cls, path, groupName):
+        """ Set path permissions for the given group. """
+        logger = cls.getLogger()
+        logger.debug('Allowing group %s to read/execute path %s' % (groupName, path))
+        cmd = '%s -m group\:%s\:rx "%s"' % (cls.SETFACL_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def changePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def recursivelyChangePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Recursively changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s -R \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def refreshNscdGroupCache(cls):
+        logger = cls.getLogger()
+        try:
+            logger.debug('Refreshing NCSD secondary group membership cache')
+            cmd = '%s -i group' % (cls.NSCD_CMD)
+            cls.executeSudoCommand(cmd)
+        except Exception, ex:
+            logger.warn('Failed to refresh NCSD group cache: %s' % (str(ex)))
+
+    # Refresh auth files in a timer, to avoid running command too often
+    def refreshAuthFiles(self):
+        if not self.refreshAuthFilesCommand:
+            return
+        if self.refreshAuthFilesTimer and self.refreshAuthFilesTimer.is_alive():
+            return
+        self.refreshAuthFilesTimer = threading.Timer(self.TIMER_DELAY_IN_SECONDS, self.__refreshAuthFiles)
+        self.refreshAuthFilesTimer.start()
+
+    def __refreshAuthFiles(self):
+        logger = self.getLogger()
+        try:
+            logger.debug('Refreshing auth files')
+            self.executeCommand(self.refreshAuthFilesCommand)
+        except Exception, ex:
+            logger.warn('Failed to refresh auth files: %s' % (str(ex)))
+
+    @classmethod
+    def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+        logger = cls.getLogger()
+        logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+        cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+        cls.executeCommand(cmd)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    utility = LdapLinuxPlatformUtility('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=dmadmin,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', '/tmp/ldapPassword', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
+    print utility.getGroupInfo(u's1id-test03')
+    #utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+    #utility.deleteLocalUserFromGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+

src/python/dm/common/utility/ldapUserManager.py

+#!/usr/bin/env python
+
+import ldap
+import ldap.modlist
+import copy
+import re
+import types
+from base64 import b16encode
+from base64 import b16decode
+from base64 import b64encode
+from base64 import b64decode
+
+from dm.common.objects.ldapUserInfo import LdapUserInfo
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.exceptions.internalError import InternalError
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.dmException import DmException
+from ldapClient import LdapClient
+
+class LdapUserManager(LdapClient):
+
+    CONFIG_SECTION_NAME = 'LdapUserManager'
+    SERVER_URL_KEY = 'serverurl'
+    ADMIN_DN_KEY = 'admindn'
+    ADMIN_PASSWORD_FILE_KEY = 'adminpasswordfile'
+    USER_DN_FORMAT_KEY = 'userdnformat'
+    GROUP_DN_FORMAT_KEY = 'groupdnformat'
+    MIN_GID_NUMBER_KEY = 'mingidnumber'
+
+    def __init__(self, serverUrl, adminDn, adminPasswordFile, userDnFormat, groupDnFormat, minGidNumber=None):
+        LdapClient.__init__(self, serverUrl, adminDn, adminPasswordFile)
+        self.userDnFormat = userDnFormat
+        self.groupDnFormat = groupDnFormat
+        self.minGidNumber = minGidNumber
+        #self.getLogger().debug('Min GID number: %s' % minGidNumber)
+        # Remove first entry from the dn format to get tree base
+        self.groupBaseDn = ','.join(groupDnFormat.split(',')[1:])
+        #self.getLogger().debug('Group base DN: %s' % self.groupBaseDn)
+
+    @classmethod
+    def createInstance(cls):
+        cm = ConfigurationManager.getInstance()
+        logger = cls.getLogger()
+        logger.debug('Creating LDAP User Manager instance')
+        serverUrl = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.SERVER_URL_KEY)
+        #logger.debug('Using server URL: %s' % serverUrl)
+        adminDn = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.ADMIN_DN_KEY)
+        #logger.debug('Using admin DN: %s' % adminDn)
+        adminPasswordFile = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.ADMIN_PASSWORD_FILE_KEY)
+        #logger.debug('Using admin password file: %s' % adminPasswordFile)
+        userDnFormat = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.USER_DN_FORMAT_KEY)
+        #logger.debug('Using user DN format: %s' % userDnFormat)
+        groupDnFormat = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.GROUP_DN_FORMAT_KEY)
+        #logger.debug('Using group DN format: %s' % groupDnFormat)
+        minGidNumber = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.MIN_GID_NUMBER_KEY)
+        #logger.debug('Using min GID number: %s' % minGidNumber)
+        return LdapUserManager(serverUrl, adminDn, adminPasswordFile, userDnFormat, groupDnFormat, minGidNumber)
+
+    @classmethod
+    def decodePasswordHash(cls, b64EncodedString):
+        beginEncoding = b64EncodedString.find('}')
+        decodedString = b64EncodedString[beginEncoding+1:]
+        decodedString = b16encode(b64decode(decodedString)).upper()
+        return decodedString
+        
+    @classmethod
+    def encodePasswordHash(cls, passwordHash):
+        encodedString = '{SHA}'+b64encode(b16decode(passwordHash))
+        return encodedString
+        
+    @LdapClient.executeLdapCall
+    def getUserInfo(self, username):
+        """ Get user info. """
+        userDn = self.userDnFormat % str(username)
+        ldapClient = self.getLdapClient()
+        resultList = ldapClient.search_s(userDn, ldap.SCOPE_BASE, attrlist=['*','createTimeStamp','modifyTimestamp'])
+        userTuple = resultList[0]
+        return LdapUserInfo({'userDn' : userTuple[0], 'userAttrs' : userTuple[1]})
+
+    @LdapClient.executeLdapCall
+    def getUserInfos(self):
+        """ Get user infos. """
+        baseDnPos = self.userDnFormat.find(',')
+        baseDn = self.userDnFormat[baseDnPos+1:]
+        searchFilter = self.userDnFormat[0:baseDnPos] % '*'
+        ldapClient = self.getLdapClient()
+        resultList = ldapClient.search_s(baseDn, ldap.SCOPE_SUBTREE, searchFilter, attrlist=['*','createTimeStamp','modifyTimestamp'])
+        userList = []
+        for result in resultList:
+            userDn = result[0]
+            userAttrs = result[1]
+            userList.append(LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs}))
+        return userList
+
+    @LdapClient.executeLdapCall
+    def getUserInfoMapByUid(self):
+        """ Get user infos. """
+        baseDnPos = self.userDnFormat.find(',')
+        baseDn = self.userDnFormat[baseDnPos+1:]
+        searchFilter = self.userDnFormat[0:baseDnPos] % '*'
+        ldapClient = self.getLdapClient()
+        resultList = ldapClient.search_s(baseDn, ldap.SCOPE_SUBTREE, searchFilter)
+        userMap = {}
+        for result in resultList:
+            userDn = result[0]
+            userAttrs = result[1]
+            uid = userAttrs['uid'][0]
+            userMap[uid] = LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs})
+        return userMap
+
+    @LdapClient.executeLdapCall
+    def modifyUserInfo(self, username, attrDict):
+        """ Modify user. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        ldapUserInfo = self.getUserInfo(username)
+        userDn = ldapUserInfo.get('userDn')
+        userAttrs = ldapUserInfo.get('userAttrs')
+
+        # Remove internal LDAP attributes before creating copy for modifications
+        for key in [ 'modifyTimestamp', 'createTimeStamp' ]:
+            if userAttrs.has_key(key):
+                del userAttrs[key]
+        userAttrs2 = copy.copy(userAttrs)
+
+        middleName = attrDict.get('middleName', '')
+        fullName = '%s, %s' % (attrDict.get('lastName', ''), attrDict.get('firstName', ''))
+        if middleName:
+            fullName = '%s %s' % (fullName, middleName)
+
+        if attrDict.has_key('name'):
+            userAttrs2['cn'] = [attrDict.get('name')]
+            userAttrs2['revcn'] = ['%s %s' % (attrDict.get('lastName'), attrDict.get('firstName'))]
+            userAttrs2['sn'] = [attrDict.get('lastName')]
+            userAttrs2['givenName'] = [attrDict.get('firstName')]
+            userAttrs2['gecos'] = [attrDict.get('name')]
+        for key in ['loginShell', 'allowed-host', 'homeDirectory', 'home7Directory', 'gidNumber' ]:
+            if attrDict.has_key(key):
+                userAttrs2[key] = [attrDict.get(key)]
+        if attrDict.has_key('grp'):
+            userAttrs2['o'] = [attrDict.get('grp')]
+        if attrDict.has_key('inactive'):
+            if attrDict.get('inactive', 'N') == 'Y':
+                userAttrs2['inetUserStatus'] = ['inactive']
+            else:
+                userAttrs2['inetUserStatus'] = ['active']
+        passwordHash = attrDict.get('userPassword')
+        if not passwordHash:
+            passwordHash = attrDict.get('passwordHashValue')
+        if passwordHash:
+            userAttrs2['userPassword'] = [self.encodePasswordHash(passwordHash)]
+            #logger.debug('Encoded password entry: %s' % passwordHash)
+        logger.debug('Old user %s attrs: %s' % (username, userAttrs))
+        logger.debug('Modified user %s attrs: %s' % (username, userAttrs2))
+        userLdif = ldap.modlist.modifyModlist(userAttrs, userAttrs2)
+        ldapClient.modify_s(userDn, userLdif)
+        return LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs2})
+
+    @LdapClient.executeLdapCall
+    def createUserInfo(self, username, attrDict):
+        """ Create user. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        try:
+            ldapUserInfo = self.getUserInfo(username)
+            raise ObjectAlreadyExists('User %s already exists.' % username)
+        except ObjectNotFound, ex:
+            pass
+        middleName = attrDict.get('middleName') 
+        fullName = '%s, %s' % (attrDict.get('lastName', ''), attrDict.get('firstName', ''))
+        if middleName:
+            fullName = '%s %s' % (fullName, middleName)
+        badgeNumber = attrDict.get('badgeNumber')
+        uid = 'd%s' % badgeNumber
+        if username != uid:
+            raise InvalidArgument('Username %s and badge number %s are not consistent.' % (username, badgeNumber))
+
+        userAttrs = {}
+        userAttrs['objectclass'] = ['top', 'person', 'organizationalPerson', 'inetOrgPerson', 'inetUser', 'shadowAccount', 'posixAccount', 'passwordObject', 'APSexten', 'apple-user']
+        userAttrs['cn'] = [attrDict.get('name', fullName)]
+        userAttrs['revcn'] = ['%s %s' % (attrDict.get('lastName', ''), attrDict.get('firstName', ''))]
+        userAttrs['loginShell'] = ['/sbin/nologin']
+        userAttrs['allowed-host'] = ['dmid-vm']
+        userAttrs['uidNumber'] = ['10%s' % badgeNumber]
+        userAttrs['uid'] = [uid]
+        userAttrs['homeDirectory'] = ['/data']
+        userAttrs['home7Directory'] = ['/data']
+        userAttrs['o'] = [attrDict.get('grp')]
+        if attrDict.get('inactive', 'N') == 'Y':
+            userAttrs['inetUserStatus'] = ['inactive']
+        else:
+            userAttrs['inetUserStatus'] = ['active']
+        userAttrs['gidNumber'] = ['66001']
+        userAttrs['gecos'] = [attrDict.get('name', fullName)]
+        userAttrs['sn'] = [attrDict.get('lastName', '')]
+        userAttrs['givenName'] = [attrDict.get('firstName', '')]
+        passwordHash = attrDict.get('userPassword')
+        if not passwordHash:
+            passwordHash = attrDict.get('passwordHashValue')
+        if passwordHash:
+            userAttrs['userPassword'] = [self.encodePasswordHash(passwordHash)]
+        logger.debug('Creating user %s with attrs %s' % (username, userAttrs))
+
+        userDn = self.userDnFormat % str(username)
+        userLdif = ldap.modlist.addModlist(userAttrs)
+        ldapClient.add_s(userDn, userLdif)
+        return LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs})
+
+    def createGroup(self, name):
+        """ Create group if it does not exist. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        name = str(name)
+        try:
+            groupDn = self.groupDnFormat % name
+            logger.debug('Looking for group DN: %s' % groupDn)
+            # this method will throw exception if group is not found
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            logger.debug('Group %s already exists' % groupTuple[0])
+            return
+        except ldap.NO_SUCH_OBJECT, ex:
+            logger.debug('Group DN %s must be created' % groupDn)
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+        # determine gidNumber: look through all entries to get max value,
+        # then increment it
+        # ldap should really be configured to handle gid's automatically,
+        # and should prevent invalid entries
+        try:
+            logger.debug('Looking for max group id')
+            resultList = ldapClient.search_s(self.groupBaseDn, ldap.SCOPE_ONELEVEL, attrlist=['gidNumber'])
+            maxGid = 0
+            if self.minGidNumber:
+                maxGid = self.minGidNumber 
+            for result in resultList:
+                gidList = result[1].get('gidNumber', [])
+                gid = 0
+                if gidList:
+                    gid = int(gidList[0])
+
+                if gid > maxGid:
+                    maxGid = gid
+            gidNumber = str(maxGid + 1)
+            logger.debug('Max GID is %s, new group id will be %s' % (maxGid, gidNumber)) 
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        
+        attrs = {}
+        attrs['objectclass'] = ['posixGroup','top']
+        attrs['cn'] = name
+        attrs['gidNumber'] = [gidNumber]
+        attrs['memberUid'] = []
+        try:
+            groupLdif = ldap.modlist.addModlist(attrs)
+            ldapClient.add_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add group %s: %s' % (groupDn, ex))
+            raise InternalError(exception=ex)
+
+    def addUserToGroup(self, username, groupName):
+        """ Add user to group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username in memberUidList:
+                logger.debug('Group %s already contains user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Adding user %s to group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.append(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+    def deleteUserFromGroup(self, username, groupName):
+        """ Remove user from group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username not in memberUidList:
+                logger.debug('Group %s does not contain user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Removing user %s from group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.remove(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+    def getGroupInfo(self, groupName):
+        """ Get given group info. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            return groupTuple
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+    def setGroupUsers(self, groupName, usernameList):
+        """ Set list of users for a given group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Setting users %s for group %s' % (usernameList, groupName))
+        memberUidList = []
+        for username in usernameList:
+            memberUidList.append(str(username))
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not set users %s for group %s: %s' % (usernameList, groupName, ex))
+            raise InternalError(exception=ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    utility = LdapUserManager('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=dmadmin,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', '/tmp/ldapPassword', userDnFormat='uid=%s,ou=DM,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
+    #print utility.getGroupInfo(u's1id-test03')
+    user = utility.getUserInfo(u'd225159')
+    print user
+    print user.getUserPassword()
+    print utility.decodePasswordHash(user.getUserPassword())
+    userMap = utility.getUserInfoMapByUid()
+    print 'N entries: ', len(userMap)
+    print userMap['d225159']
+    #for user in userList:
+    #    print user
+
+    utility.modifyUserInfo(u'd225159', {'homeDirectory' : '/data/'})
+    user = utility.getUserInfo(u'd225159')
+    print user
+    #user = utility.getUserInfo(u'd65114')
+    #print user
+
+    #passwordHash = LdapUserManager.decodePasswordHash(user['userAttrs']['userPassword'][0])
+    #print passwordHash
+
+    #print 'Creating user d000001'
+    #user = utility.getUserInfo(u'd000001')
+    #print user
+    #attrDict = {'badgeNumber' : '000001', 'name' : 'Test, User 1.', 'firstName' : 'User', 'middleName' : '1.', 'lastName' : 'Test', 'passwordHashValue' : '84673F4A8774846B1C096511C7F6B1329CEE5CCC', 'inactive' : 'N', 'grp' : 'XSD-TEST', 'lastChangeDate' : '2015-02-03 15:52:15', 'email' : 'UTEST1@APS.ANL.GOV', 'isUserNotAnlEmployee' : 'N' }
+    #try:
+    #    user = utility.createUserInfo(u'd000001', attrDict)
+    #except ObjectAlreadyExists, ex:
+    #    print 'User d000001 already exists, will modify it'
+    #    user = utility.updateUserInfo(u'd000001', attrDict)
+
+    #print user
+
+    #print LdapUserManager.encodePasswordHash(passwordHash)
+
+    #utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+    #utility.deleteLocalUserFromGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+

src/python/dm/common/utility/ldapUtility.py

+#!/usr/bin/env python
+
+import ldap
+
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+
+class LdapUtility:
+
+    def __init__(self, serverUrl, dnFormat):
+        self.serverUrl = serverUrl
+        self.dnFormat = dnFormat
+         
+    def checkCredentials(self, username, password):
+        """Verifies credentials for username and password. """
+        ldapUsername = self.dnFormat % username
+        try:
+            # build client
+            ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+            ldapClient = ldap.initialize(self.serverUrl)
+            ldapClient.set_option(ldap.OPT_REFERRALS,0)
+            ldapClient.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
+
+            #ldapClient.ststart_tls_s()
+            #ldapClient.set_option(ldap.OPT_X_TLS,ldap.OPT_X_TLS_DEMAND)
+            #ldapClient.set_option( ldap.OPT_X_TLS_DEMAND, True)
+            #ldapClient.set_option( ldap.OPT_DEBUG_LEVEL, 255)
+
+            # perform a synchronous bind
+            ldapClient.simple_bind_s(ldapUsername, password)
+            #ldapClient.whoami_s()
+        except ldap.INVALID_CREDENTIALS, ex:
+            ldapClient.unbind()
+            raise AuthenticationError('Invalid LDAP credentials for user %s' % username)
+        except ldap.SERVER_DOWN, ex:
+            raise CommunicationError('Cannot reach LDAP server %s' % self.serverUrl)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    ldapUtility = LdapUtility(serverUrl='ldaps://ldap.anl.gov:636', dnFormat='uid=%s,dc=anl,dc=gov')
+    ldapUtility.checkCredentials('sveseli', 'sv')

src/python/dm/common/utility/linuxUtility.py

+#!/usr/bin/env python
+
+import grp
+from dm.common.utility.loggingManager import LoggingManager
+from dmSubprocess import DmSubprocess
+
+class LinuxUtility:
+
+    SUDO_CMD = '/usr/bin/sudo'
+    GROUPADD_CMD = '/usr/sbin/groupadd'
+    USERMOD_CMD = '/usr/sbin/usermod'
+    SETFACL_CMD = '/usr/bin/setfacl'
+    CHOWN_CMD = '/bin/chown'
+    GPASSWD_CMD = '/usr/bin/gpasswd'
+    FIND_CMD = '/bin/find'
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    @classmethod
+    def executeSudoCommand(cls, cmd):
+        p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
+        p.run()
+
+    @classmethod
+    def executeCommand(cls, cmd):
+        p = DmSubprocess('%s' % (cmd))
+        p.run()
+
+    @classmethod
+    def createGroup(cls, name):
+        """ Create group if it does not exist. """
+        logger = cls.getLogger()
+        try:
+            group = grp.getgrnam(name)
+            logger.debug('Group %s already exists' % name)
+            return 
+        except KeyError, ex:
+            # ok, we need to create group 
+            pass
+        logger.debug('Creating group %s' % name)
+        cmd = '%s %s' % (cls.GROUPADD_CMD, name)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def addUserToGroup(cls, username, groupName):
+        """ Add user to group. """
+        logger = cls.getLogger()
+        logger.debug('Adding user %s to group %s' % (username, groupName))
+        cmd = '%s -a -G %s %s' % (cls.USERMOD_CMD, groupName, username)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def addLocalUserToGroup(cls, username, groupName):
+        """ Add local user to group. """
+        logger = cls.getLogger()
+        logger.debug('Adding local user %s to group %s' % (username, groupName))
+        cmd = '%s -a %s %s' % (cls.GPASSWD_CMD, username, groupName)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def deleteLocalUserFromGroup(cls, username, groupName):
+        """ Remove local user from group. """
+        logger = cls.getLogger()
+        logger.debug('Removing local user %s from group %s' % (username, groupName))
+        cmd = '%s -d %s %s' % (cls.GPASSWD_CMD, username, groupName)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def setGroupUsers(cls, groupName, usernameList):
+        """ Set list of users for a given group. """
+        logger = cls.getLogger()
+        logger.debug('Setting group %s users to: %s' % (groupName, usernameList))
+        cmd = '%s -M "%s" %s' % (cls.GPASSWD_CMD, ','.join(usernameList), groupName)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def setPathReadExecutePermissionsForGroup(cls, path, groupName):
+        """ Set path permissions for the given group. """
+        logger = cls.getLogger()
+        logger.debug('Allowing group %s to read/execute path %s' % (groupName, path))
+        cmd = '%s -m group\:%s\:rx "%s"' % (cls.SETFACL_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def changePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def recursivelyChangePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Recursively changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s -R \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+        logger = cls.getLogger()
+        logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+        cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+        cls.executeCommand(cmd)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    LinuxUtility.createGroup('exp3')
+    LinuxUtility.addUserToGroup('hpc', 'exp3')
+    LinuxUtility.setPathReadExecutePermissionsForGroup('/home/sveseli/Work/DM/data/ESAF/exp3', 'exp3')
+

src/python/dm/common/utility/loggingManager.py

+#!/usr/bin/env python
+
+#
+# Logging manager singleton.
+#
+
+import re
+import sys
+import os.path
+import logging
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+
+class LoggingManager:
+    """ 
+    The log manager class is initialized via a configuration file 
+    that may have the following sections:
+
+    ConsoleLogging      # Used for output on the screen
+    FileLogging         # Used for logging into a file
+
+    Each section in the configuration file should have the following 
+    keys:
+        handler     # Indicates which handler class to use 
+        level       # Indicates logging level
+        format      # Indicates format for log messages
+        dateFormat  # Indicates date format used for log messages
+
+    Given below is an example of a valid configuration file:
+
+    [ConsoleLogging]
+        handler=ConsoleLoggingHandler(sys.stdout,)
+        level=info
+        format=[%(levelname)s] %(message)s 
+        dateFormat=%m/%d/%y %H:%M:%S
+    [FileLogging]
+        handler=TimedRotatingFileLoggingHandler('/tmp/dm.log')
+        level=debug
+        format=%(asctime)s,%(msecs)d [%(levelname)s] %(module)s:%(lineno)d %(user)s@%(host)s %(name)s (%(process)d): %(message)s
+        dateFormat=%m/%d/%y %H:%M:%S
+    """
+
+    # Get singleton instance.
+    @classmethod
+    def getInstance(cls):
+        from dm.common.utility.loggingManager import LoggingManager
+        try:
+            lm = LoggingManager()
+        except LoggingManager, ex:
+            lm = ex
+        return lm
+
+    # Singleton.
+    __instance = None
+
+    def __init__(self):
+        if LoggingManager.__instance:
+             raise LoggingManager.__instance
+        LoggingManager.__instance = self
+        self.consoleHandler = None
+        self.fileHandlerList = []
+        self.maxIntLevel = logging.CRITICAL
+        self.minIntLevel = logging.NOTSET
+        self.levelRegExList = []
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.initFlag = False
+
+    def setMinLogLevel(self, minLogLevel=logging.INFO):
+        self.minIntLevel = minLogLevel
+
+    def parseLevelRegEx(self, levelRegExList):
+        """ Parse expressions of the form <regex>=<log level>. """
+        lines = levelRegExList.split('\n')
+        for line in lines:
+            try:
+                 (regex, level) = line.rsplit('=', 1)
+                 pattern = re.compile(regex)
+                 tuple = (pattern, logging.getLevelName(level.upper()))
+                 self.levelRegExList.append(tuple)
+            except Exception, ex:
+                 self.logger.error('Parser error in log configuration file: %s' % line)
+                 self.logger.exception(ex)
+
+    # Get Log Level based on a string representation
+    def getIntLogLevel(self, levelStr):
+        level = logging.getLevelName(levelStr)
+        # Level should be an integer
+        try:
+            return int(level) 
+        except ValueError, ex:
+            raise ConfigurationError('"%s" is not valid log level' % levelStr)
+
+    # Configure log handlers.
+    def configure(self):
+        """ Configure log handlers from the config file. """
+        cm = ConfigurationManager.getInstance()
+        configFile = cm.getConfigFile()
+        configParser = cm.getConfigParserFromConfigFile(configFile)
+        configSections = cm.getConfigSectionsFromConfigParser(configParser)
+
+        # Console handler.
+        defaults = { 
+           'level' : cm.getConsoleLogLevel(),
+           'format' : cm.getLogRecordFormat(),
+           'dateFormat' : cm.getLogDateFormat(),
+           'handler' : 'ConsoleLoggingHandler(sys.stdout,)'
+        }
+        if not self.consoleHandler:
+            consoleHandler = self.__configureHandler(configParser, 'ConsoleLogging', defaults)
+
+            if consoleHandler != None:
+                self.consoleHandler = consoleHandler
+
+        # File logging. 
+        # Do not configure if log directory does not exist.
+        defaults['handler'] = None
+        defaults['level'] = cm.getFileLogLevel() 
+        if not os.path.exists(configFile):
+            # No config file, we'll configure default.
+            defaultLogFile = cm.getLogFile()
+            defaultLogDir = os.path.dirname(defaultLogFile)
+            if os.path.exists(defaultLogDir):
+                handler = 'TimedRotatingFileLoggingHandler("%s")' % defaultLogFile 
+                defaults['handler'] = handler
+                fileHandler = self.__configureHandler(configParser, 'FileLogging', defaults)
+                if fileHandler != None:
+                    self.fileHandlerList.append(fileHandler)
+
+        else:
+            # Parse all file loggers present in the config file
+            for configSection in configSections:
+                if configSection.startswith('FileLogging'):
+                     fileHandler = self.__configureHandler(configParser, configSection, defaults)
+                     if fileHandler != None:
+                         self.fileHandlerList.append(fileHandler)
+
+        # Add handlers to the root logger.  Use logging class here
+        # to make sure we can have a logger when we parse the 
+        # logger expressions
+        rootLogger = logging.getLogger('')
+        for handler in [self.consoleHandler] + self.fileHandlerList:
+            rootLogger.addHandler(handler)
+
+        # Get a logger factory based on our current config 
+        self.configureLoggers(configParser, defaultLevel='debug')
+
+    def configureLoggers(self, configParser, defaultLevel='error'):
+        rootLogLevel = 'error'
+        levelRegEx = '^.*$=%s' % (defaultLevel)
+        if configParser is not None and configParser.has_section('LoggerLevels'):
+            rootLogLevel = ConfigurationManager.getOptionFromConfigParser(configParser, 'LoggerLevels', 'root', rootLogLevel)
+            levelRegEx = ConfigurationManager.getOptionFromConfigParser(configParser, 'LoggerLevels', 'levelregex', levelRegEx)
+
+        rootLevelInt = logging.getLevelName(rootLogLevel.upper())
+        logging.getLogger('').root.setLevel(rootLevelInt)
+        logging.getLogger('').debug('Set root logger to %s' % rootLevelInt)
+
+        if not levelRegEx:
+            return
+
+        # Parse expressions of the form <regex>=<log-level>. """
+        lines = levelRegEx.split('\n')
+        for line in lines:
+            try:
+                # Use the right split so we can have '='s in the regex
+                (regex, level) = line.rsplit('=', 1)
+                pattern = re.compile(regex)
+                tuple = (pattern, logging.getLevelName(level.upper()))
+                self.levelRegExList.append(tuple)
+            except Exception, ex:
+                # Do not fail
+                self.logger.error('Parser error in log configuration file: %s' % line)
+                self.logger.exception(ex)
+
+    # Configure particular handler with given defaults.
+    def __configureHandler(self, configParser, configSection, defaults):
+        """ Configure specified handler with a given defaults. """
+        handlerOption = defaults['handler']
+        try:
+            if configParser is not None:
+                handlerOption = configParser.get(configSection, 'handler')
+        except Exception, ex:
+            pass
+
+        # If handlerOption is empty, handler cannot be instantiated.
+        handler = None
+        if handlerOption != None:
+            # Handler argument format: MyHandler(arg1, arg2, ...)
+            # Module will be in lowercase letters, but the class
+            # should be capitalized.
+            handlerName = re.sub('\(.*', '', handlerOption)
+            moduleName = handlerName[0].lower() + handlerName[1:]
+            try:
+                exec 'from dm.common.utility import %s' % (moduleName)
+                exec 'handler = %s.%s' % (moduleName, handlerOption)
+            except IOError, ex:
+                errNo, errMsg = ex
+                import errno
+
+                # If the exception raised is an I/O permissions error, ignore
+                # it and disable this log handler.  This allows non-root users
+                # to use the (system-wide) default log configuration
+                if errNo != errno.EACCES:
+                    raise
+                handler = None 
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+        # Only request setting from the config file if it was
+        # not set via environment variable, or programmatically.
+        cm = ConfigurationManager.getInstance()
+        if handler != None:
+            try:
+                level = cm.getOptionFromConfigParser(configParser, configSection, 'level', defaults['level'])
+                intLevel = self.getIntLogLevel(level.upper())
+                handler.setLevel(intLevel)
+
+                format = cm.getOptionFromConfigParser(configParser, configSection, 'format', defaults['format'])
+                dateFormat = cm.getOptionFromConfigParser(configParser, configSection, 'dateFormat', defaults['dateFormat'])
+
+                handler.setFormatter(logging.Formatter(format, dateFormat))
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+            # Apply filters to handler
+            filter = None
+            try:
+                filter = configParser.get(configSection, 'filter')
+                if filter:
+                    handler.addFilter(logging.Filter(filter))
+            except Exception, ex:
+                pass
+        return handler
+
+    def getLogger(self, name='defaultLogger'):
+        if not self.initFlag:
+            self.initFlag = True
+            self.configure()
+        logger = logging.getLogger(name)
+        logger.setLevel(self.getLevel(name))
+        return logger
+
+    def getLevel(self, name):
+        # Match from the known regex list.
+        level = logging.NOTSET
+
+        # The last regex is most important.
+        for e in reversed(self.levelRegExList):
+            (pattern, level) = e
+
+            # If we return not None it is a match
+            if not None == pattern.match(name):
+                break
+
+        if level > self.maxIntLevel:
+            level = self.maxIntLevel
+        if level < self.minIntLevel:
+            level = self.minIntLevel
+        return level
+
+    def setConsoleLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            self.consoleHandler.setLevel(intLevel)
+            self.maxIntLevel = intLevel
+            self.logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+    def setFileLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            for handler in self.fileHandlerList:
+                handler.setLevel(intLevel)
+            self.maxIntLevel = intLevel
+            self.logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    lm = LoggingManager.getInstance()
+    logger = lm.getLogger('Main')
+    logger.error('Error In Main')
+    logger.debug('Debug In Main')
+    logger.warn('Warn In Main')
+    logger.info('Info In Main')
+    logger = lm.getLogger('Main')
+    logger.info('Info In Main')
+    logger = lm.getLogger('')
+    logger.info('Info using root logger')
+    logger = lm.getLogger('Main.2')
+    logger.info('Info in Main.2')
+    lm.setConsoleLogLevel('info')
+    logger.debug('You should not see this message')
+    lm.setConsoleLogLevel('debug')
+    logger.debug('Debug in Main.2')