src/python/dm/common/utility/dictUtility.py

+#!/usr/bin/env python
+
+import copy
+
+class DictUtility:
+
+    @classmethod
+    def getAndRemoveKey(cls, dict, key, default=None):
+        value = dict.get(key, default)
+        if dict.has_key(key):
+            del dict[key]
+        return value
+
+    @classmethod
+    def deepCopy(cls, dict, includeKeys=[], excludeKeys=[]):
+        dict2 = {}
+        if len(includeKeys):
+            for key in includeKeys:
+                value = dict.get(key)
+                if value is not None:
+                    dict2[key] = copy.deepcopy(value)
+        elif len(excludeKeys):
+            for key in dict.keys():
+                if key not in excludeKeys:
+                    dict2[key] = copy.deepcopy(dict[key])
+        else:
+            dict2 = copy.deepcopy(dict)
+        return dict2
+
+    @classmethod
+    def deepCopy2(cls, dict, includeKeys=[], excludeKeys=[]):
+        dict2 = copy.deepcopy(dict)
+        if len(includeKeys):
+            for key in dict2.keys():
+                if not key in includeKeys:
+                    del dict2[key]
+        elif len(excludeKeys):
+            for key in excludeKeys:
+                if dict2.has_key(key):
+                    del dict2[key]
+        return dict2
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    print DictUtility.deepCopy({'k1': '1', 'k2' : '2'}, excludeKeys=['k1'])
+    print  DictUtility.deepCopy({'k1': '1', 'k2' : '2'}, includeKeys=['k1'])
+    print  DictUtility.deepCopy({'k1': '1', 'k2' : '2'})
+

src/python/dm/common/utility/dmModuleManager.py

+#!/usr/bin/env python
+
+#
+# Module manager class.
+#
+
+#######################################################################
+
+import threading
+
+#######################################################################
+
+class DmModuleManager:
+    """ Singleton class used for managing dm modules. """
+
+    # Get singleton instance.
+    @classmethod
+    def getInstance(cls):
+        from dm.common.utility.dmModuleManager import DmModuleManager
+        try:
+            mgr = DmModuleManager()
+        except DmModuleManager, ex:
+            mgr = ex
+        return mgr
+
+    # Singleton.
+    __instanceLock = threading.RLock()
+    __instance = None
+
+    def __init__(self):
+        DmModuleManager.__instanceLock.acquire()
+        try:
+            if DmModuleManager.__instance:
+                raise DmModuleManager.__instance
+            DmModuleManager.__instance = self
+            from dm.common.utility.loggingManager import LoggingManager
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+            self.lock = threading.RLock()
+            self.moduleList = []
+            self.modulesRunning = False
+        finally:
+            DmModuleManager.__instanceLock.release()
+
+    def addModule(self, m):
+        self.lock.acquire()
+        try:
+            self.logger.debug('Adding dm module: %s' % m.__class__.__name__)
+            self.moduleList.append(m)
+        finally:
+            self.lock.release()
+
+    def startModules(self):
+        self.lock.acquire()
+        try:
+            if self.modulesRunning:
+                return
+            for m in self.moduleList:
+                self.logger.debug('Starting dm module: %s' % m.__class__.__name__)
+                m.start()
+            self.modulesRunning = True
+        finally:
+            self.lock.release()
+
+    def stopModules(self):
+        self.lock.acquire()
+        try:
+            if not self.modulesRunning:
+                return
+            n = len(self.moduleList)
+            for i in range(0, n):
+                m = self.moduleList[n-1-i]
+                self.logger.debug('Stopping dm module: %s' % m.__class__.__name__)
+                m.stop()
+            self.modulesRunning = False
+        finally:
+            self.lock.release()
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/utility/dmSubprocess.py

+#!/usr/bin/env python
+
+# 
+# Subprocess class
+#
+
+import os
+import subprocess
+import platform
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.commandFailed import CommandFailed
+
+#######################################################################
+
+class DmSubprocess(subprocess.Popen):
+
+    # Get subprocess instance.
+    @classmethod
+    def getSubprocess(cls, command, cwd=None):
+        close_fds = True
+        if platform.system() != 'Windows':
+            close_fds = False
+        p = DmSubprocess(command, close_fds=close_fds, cwd=cwd)
+        return p
+
+    # Execute command
+    @classmethod
+    def executeCommand(cls, command, cwd=None):
+        """ Create subprocess and run it, return subprocess object. """
+        p = cls.getSubprocess(command, cwd)
+        p.run()
+        return p
+
+    # Execute command, ignore errors.
+    @classmethod
+    def executeCommandAndIgnoreFailure(cls, command, cwd=None):
+        """ Create subprocess, run it, igore any failures, and return subprocess object. """
+        p = cls.getSubprocess(command, cwd)
+        try:
+            p.run()
+        except CommandFailed, ex:
+            p.getLogger().debug('Command failed, stdout: %s, stderr: %s' % (p.getStdOut(), p.getStdErr()))
+        return p
+
+    @classmethod
+    def executeCommandAndLogToStdOut(cls, command, cwd=None):
+        """ Execute command, display output to stdout, maintain log file and return subprocess object. """
+        p = cls.getSubprocess(command, cwd)
+        p.__commandLog()
+
+        while True:
+            outp = p.stdout.readline()
+            if not outp:
+                break
+            print outp,
+
+        retval = p.wait()
+
+        p.logger.debug('Exit status: %s' % retval)
+
+        if retval != 0:
+            error = ''
+            while True:
+                err = p.stderr.readline()
+                if not err:
+                    break
+                error += err
+            raise CommandFailed(error)
+        return p
+
+    def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=None, close_fds=False, shell=True, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, useExceptions=True, quietMode=False):
+        """ Overrides Popen constructor with more appropriate defaults. """
+        subprocess.Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags)
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        self._stdout = None
+        self._stderr = None
+        self._args = args
+        self._cwd = cwd
+        self.useExceptions = useExceptions
+        self.quietMode = quietMode
+
+    def __commandLog(self):
+        # Not very useful to show the name of this file.
+        # Walk up the stack to find the caller.
+        import traceback
+        stack =  traceback.extract_stack()
+        for i in range(2, len(stack)):
+            if stack[-i][0] != stack[-1][0]:
+                fileName, lineNumber, functionName, text = stack[-i]
+                break
+            else:
+                fileName = lineNumber = functionName = text = '?'
+
+        self.logger.debug('From [%s:%s] Invoking: [%s]' % (os.path.basename(fileName), lineNumber, self._args))
+
+    def run(self, input=None):
+        """ Run subprocess. """
+        if not self.quietMode:
+            self.__commandLog()
+        (self._stdout, self._stderr) = subprocess.Popen.communicate(self, input)
+        if not self.quietMode:
+            self.logger.debug('Exit status: %s' % self.returncode)
+        if self.returncode != 0 and self.useExceptions:
+            if not self.quietMode:
+                self.logger.debug('StdOut: %s' % self._stdout)
+                self.logger.debug('StdErr: %s' % self._stderr)
+            error = self._stderr.strip()
+            if error == '':
+                error = self._stdout.strip()
+            raise CommandFailed('%s' % (error))
+        return (self._stdout, self._stderr) 
+
+    def getLogger(self):
+        return self.logger
+
+    def getArgs(self):
+        return self._args
+
+    def getStdOut(self):
+        return self._stdout
+
+    def getStdErr(self):
+        return self._stderr
+
+    def getExitStatus(self):
+        return self.returncode
+
+    def toDict(self):
+        result = { 
+            'command' : self._args,
+            'exitStatus' : self.returncode,
+            'stdErr' : self._stderr,
+            'stdOut' : self._stdout,
+            'workingDir' : self._cwd
+        }
+        return result
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    p = DmSubprocess('ls -l', useExceptions=False)
+    print p.toDict()
+    p.run()
+    #print p.getStdOut()
+    #print p.getStdErr()
+    #print p.getExitStatus()
+    print p.toDict()
+

src/python/dm/common/utility/encoder.py

+#!/usr/bin/env python
+
+import base64
+
+class Encoder:
+
+    @classmethod
+    def encode(cls, data):
+        # Encode twice, in order to avoid issues like '+' being
+        # interpreted as space after decoding
+        if not data:
+            return data
+        encodedData = base64.b64encode(base64.encodestring('%s' % data))
+        return encodedData
+
+    @classmethod
+    def decode(cls, encodedData):
+        if not encodedData:
+            return encodedData
+        data = base64.decodestring(base64.b64decode('%s' % encodedData))
+        return data
+
+    @classmethod
+    def encodeDict(cls, dict, excludedKeyList=[]):
+        encodedDict = {}
+        for (key,value) in dict.items():
+            if key not in excludedKeyList:
+                encodedDict[key] = cls.encode(value)
+            else:
+                encodedDict[key] = value
+        return encodedDict
+
+    @classmethod
+    def decodeDict(cls, encodedDict, excludedKeyList=[]):
+        dict = {}
+        for (key,value) in encodedDict.items():
+            if key not in excludedKeyList:
+                dict[key] = cls.decode(value)
+            else:
+                dict[key] = value
+        return dict

src/python/dm/common/utility/fileUtility.py

+#!/usr/bin/env python
+
+import os
+import stat
+from dm.common.utility.dmSubprocess import DmSubprocess
+from dm.common.utility.timeUtility import TimeUtility
+
+class FileUtility:
+
+    @classmethod
+    def getMd5Sum(cls, filePath, fileInfo={}):
+        p = DmSubprocess.getSubprocess('md5sum "%s"' % filePath)
+        p.run()
+        md5Sum = p.getStdOut().split()[0]
+        fileInfo['md5Sum'] = md5Sum
+        fileInfo['filePath'] = filePath
+        return fileInfo
+
+    @classmethod
+    def statFile(cls, filePath, fileInfo={}):
+        statResult = os.stat(filePath)
+        fileInfo['filePath'] = filePath
+        fileInfo['fileSize'] = statResult[stat.ST_SIZE]
+        fileInfo['fileCreationTime'] = statResult[stat.ST_CTIME]
+        fileInfo['fileAccessTime'] = statResult[stat.ST_ATIME]
+        fileInfo['fileModificationTime'] = statResult[stat.ST_MTIME]
+        return fileInfo
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    print FileUtility.getMd5Sum('/tmp/file')
+    print FileUtility.statFile('/tmp/file')
+

src/python/dm/common/utility/ftpUtility.py

+#!/usr/bin/env python
+
+import copy
+import os
+import time
+from ftplib import FTP
+from dm.common.utility.timeUtility import TimeUtility
+from dm.common.utility.loggingManager import LoggingManager
+import urlparse
+
+class FtpUtility:
+
+    def __init__(self, host, port, username=None, password=None, serverUsesUtcTime=True):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.ftpClient = None
+        self.serverUsesUtcTime = serverUsesUtcTime 
+        self.mlsdFileStatDict = {}
+
+    @classmethod
+    def parseFtpUrl(cls, url, defaultHost=None, defaultPort=None):
+        host = defaultHost
+        port = defaultPort
+        scheme = None
+        dirPath = url
+        if url.startswith('ftp://'):
+            parseResult = urlparse.urlparse(url)
+            scheme = parseResult.scheme
+            netlocTokens = parseResult.netloc.split(':')
+            host = netlocTokens[0]
+            if len(netlocTokens) > 1:
+                port = int(netlocTokens[1])
+            dirPath = parseResult.path
+        return (scheme, host, port, dirPath)
+
+    @classmethod
+    def parseUrl(cls, url, defaultHost=None, defaultPort=None):
+        host = defaultHost
+        port = defaultPort
+        parseResult = urlparse.urlparse(url)
+        scheme = parseResult.scheme
+        netlocTokens = parseResult.netloc
+        if netlocTokens: 
+	    netlocTokens = netlocTokens.split(':')
+            host = netlocTokens[0]
+            if len(netlocTokens) > 1:
+                port = int(netlocTokens[1])
+        dirPath = parseResult.path
+        dirPath = os.path.normpath(dirPath)
+        return (scheme, host, port, dirPath)
+
+    @classmethod
+    def assembleUrl(cls, scheme, host, port, dirPath):
+        url = ''
+	if scheme:
+	    url += '%s://' % scheme
+	if host:
+	    url += '%s' % (host)
+	    if port:
+	        url += ':%s' % (port)
+        if dirPath:
+	    if len(url) and not dirPath.startswith('/'):
+	        url += '/%s' % (os.path.normpath(dirPath))
+	    else:
+	        url += '%s' % (os.path.normpath(dirPath))
+        return url
+
+    @classmethod
+    def getFtpClient(cls, host, port, username=None, password=None):
+        ftp = FTP()
+        ftp.connect(host, port)
+        ftp.login(username, password)
+        return ftp
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+
+    def __parseKeyValue(cls, origKeyValue, outputDict={}, newKey=None):
+        key,value = origKeyValue.split('=')
+        value = value.strip()
+        if newKey is not None:
+            key = newKey
+        outputDict[key] = value
+        return outputDict
+        
+    def __parseMlsdOutput(self, line):
+        # ['Type=dir', 'Modify=20151018024430', 'Size=4096', 'Perm=el', 'UNIX.mode=0775', 'UNIX.owner=sveseli', 'UNIX.uid=500', 'UNIX.group=sveseli', 'UNIX.gid=500', 'Unique=fd00-c2e3e', ' dir2\r']
+        parts = line.split(';')
+        parseDict = {}
+        self.__parseKeyValue(parts[0], parseDict)
+        self.__parseKeyValue(parts[1], parseDict)
+        self.__parseKeyValue(parts[2], parseDict, 'fileSize')
+        name = parts[-1].strip()
+        parseDict['Name'] = name
+        type = parseDict.get('Type', '')
+        if type == 'dir' :
+            self.mlsdDirList.append(name)
+        elif type == 'file':
+            self.mlsdFileDict[name] = parseDict
+
+    def __parseMlsdFileStat(self, line):
+        # ['Type=dir', 'Modify=20151018024430', 'Size=4096', 'Perm=el', 'UNIX.mode=0775', 'UNIX.owner=sveseli', 'UNIX.uid=500', 'UNIX.group=sveseli', 'UNIX.gid=500', 'Unique=fd00-c2e3e', ' dir2\r']
+        parts = line.split(';')
+        parseDict = {}
+        self.__parseKeyValue(parts[0], parseDict)
+        self.__parseKeyValue(parts[1], parseDict)
+        self.__parseKeyValue(parts[2], parseDict, 'fileSize')
+        name = parts[-1].strip()
+        self.mlsdFileStatDict[name] = parseDict
+
+    def __processFileStatDict(self, fileStatDict):
+        modifyTime = fileStatDict.get('Modify')
+        modifyTime = time.mktime(time.strptime(modifyTime, '%Y%m%d%H%M%S'))
+        if self.serverUsesUtcTime:
+            modifyTime = TimeUtility.utcToLocalTime(modifyTime)
+        fileStatDict['fileModificationTime'] = modifyTime
+        fileStatDict['fileSize'] = int(fileStatDict.get('fileSize'))
+        del fileStatDict['Modify']
+        del fileStatDict['Type']
+
+    def getFiles2(self, dirPath, fileDict={}, replacementDirPath=None, initialCall=True):
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        # Need these to be class members for the callback function
+        self.mlsdFileDict = {}
+        self.mlsdDirList = []
+        self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput)
+        if not replacementDirPath:
+            replacementDirPath = dirPath
+        for (fileName,fileInfo) in self.mlsdFileDict.items():
+            self.__processFileStatDict(fileInfo)
+            del fileInfo['Name']
+            filePath = '%s/%s' % (replacementDirPath, fileName)
+            fileDict[filePath] = fileInfo
+        mlsdDirList = copy.copy(self.mlsdDirList)
+        for d in mlsdDirList:
+            dirPath2 = '%s/%s' % (dirPath,d)
+            replacementDirPath2 = '%s/%s' % (replacementDirPath,d)
+            self.getFiles2(dirPath2,fileDict, replacementDirPath2, initialCall=False)
+        if initialCall:
+            del self.mlsdFileDict
+            del self.mlsdDirList
+        return fileDict
+
+    def getFiles(self, dirPath, fileDict={}, replacementDirPath=None):
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        # Need these to be class members for the callback function
+        self.mlsdFileDict = {}
+        self.mlsdDirList = []
+        self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput)
+        if not replacementDirPath:
+            replacementDirPath = dirPath
+        for (fileName,fileInfo) in self.mlsdFileDict.items():
+            filePath = '%s/%s' % (replacementDirPath, fileName)
+            modifyTime = fileInfo.get('Modify')
+            modifyTime = time.mktime(time.strptime(modifyTime, '%Y%m%d%H%M%S'))
+            if self.serverUsesUtcTime:
+                modifyTime = TimeUtility.utcToLocalTime(modifyTime)
+            fileSize = int(fileInfo.get('fileSize'))
+            #print '%s %s %s' % (filePath, fileSize, modifyTime)
+            fileDict[filePath] = {'fileSize' : fileSize, 'fileModificationTime' : modifyTime}
+        self.mlsdFileDict.clear()
+        mlsdDirList = copy.copy(self.mlsdDirList)
+        for d in mlsdDirList:
+            dirPath2 = '%s/%s' % (dirPath,d)
+            replacementDirPath2 = '%s/%s' % (replacementDirPath,d)
+            self.getFiles(dirPath2, fileDict, replacementDirPath2)
+        self.mlsdDirList = []
+        return fileDict
+
+    def getMd5Sum(self, filePath, fileInfo={}):
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        md5Sum = self.ftpClient.sendcmd('CKSM MD5 0 -1 %s' % filePath).split()[-1]
+        fileInfo['md5Sum'] = md5Sum
+        return md5Sum
+
+    def statFile(self, filePath, fileInfo={}):
+        fileName = os.path.basename(filePath)
+        if not self.ftpClient:
+            self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
+        # Need this to be class members for the callback function
+        self.ftpClient.retrlines('MLSD %s' % filePath, self.__parseMlsdFileStat)
+        fileStatDict = self.mlsdFileStatDict.get(fileName)
+        if fileStatDict:
+            self.__processFileStatDict(fileStatDict)
+            fileInfo.update(fileStatDict)
+            del self.mlsdFileStatDict[fileName]
+        return fileInfo
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    print "Round 1: "
+    ftpUtility = FtpUtility('s33dserv', 2811)
+    #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
+    files = ftpUtility.getFiles2('/export/dm/test')
+    print files
+    files = ftpUtility.getFiles('/export/dm/test')
+    print ftpUtility.parseFtpUrl('/export/dm/test')
+    print files
+    #files = ftpUtility.getFiles('/export/7IDSprayimage/Cummins/Data')
+    #files = ftpUtility.getFiles2('/export/8-id-i/test', replacementDirPath='/data/testing/8-id-i')
+    #print "Number of files: ", len(files)
+    #time.sleep(60)
+    #print "Removing files"
+    #del files
+    #print "Files removed"
+    #time.sleep(60)
+    #del ftpUtility
+    #print "Utility removed"
+    #time.sleep(60)
+    #print "Round 2: "
+    #ftpUtility = FtpUtility('s7dserv', 2811)
+    #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
+    print ftpUtility.getMd5Sum('/export/dm/test/testfile01')
+    #print ftpUtility.statFile('/export/8-id-i/test/testfile01')
+    #ftpUtility = FtpUtility('xstor-devel', 22)
+    #files = ftpUtility.getFiles2('/data/testing')
+    #print files

src/python/dm/common/utility/ldapClient.py

+#!/usr/bin/env python
+
+import ldap
+import re
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.exceptions.internalError import InternalError
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+from dm.common.exceptions.dmException import DmException
+
+class LdapClient:
+
+    LDAP_ERROR_REGEX_LIST = [
+        (re.compile('.*No such object.*'),ObjectNotFound),
+        (re.compile('.*'),DmException)
+    ]
+
+    def __init__(self, serverUrl, adminDn, adminPasswordFile):
+        self.serverUrl = serverUrl
+        self.adminDn = adminDn
+        self.adminPassword = open(adminPasswordFile, 'r').readline().strip()
+        if not self.adminPassword:
+            raise ConfigurationError('LDAP password could not be found in %s file' % adminPasswordFile) 
+        self.ldapClient = None
+
+    # Decorator for all LDAP transactions
+    @classmethod
+    def executeLdapCall(cls, func):
+        def ldapCall(*args, **kwargs):
+            try:
+                result = func(*args, **kwargs)
+                return result
+            except ldap.NO_SUCH_OBJECT, ex:
+                raise ObjectNotFound(str(ex))
+            except DmException, ex:
+                raise
+            except Exception, ex:
+                error = str(ex)
+                for (pattern,dmExClass) in LdapClient.LDAP_ERROR_REGEX_LIST:
+                    if pattern.match(error):
+                        raise dmExClass(exception=ex)
+                raise DmException(exception=ex)
+        return ldapCall
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    def getLdapClient(self):
+        if self.ldapClient is not None:
+            try:
+                self.ldapClient.simple_bind_s(self.adminDn, self.adminPassword)
+            except Exception, ex:
+                self.getLogger().error('Invalidating LDAP client due to error: %s' % ex)
+                self.unbind(self.ldapClient)
+                self.ldapClient = None
+
+        if not self.ldapClient:
+            self.ldapClient = self.bind(self.serverUrl, self.adminDn, self.adminPassword)
+        return self.ldapClient
+
+    @classmethod 
+    def bind(cls, serverUrl, adminDn, adminPassword):
+        try:
+            ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+            ldapClient = ldap.initialize(serverUrl)
+            ldapClient.set_option(ldap.OPT_REFERRALS,0)
+            ldapClient.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
+            ldapClient.simple_bind_s(adminDn, adminPassword)
+            cls.getLogger().debug('Successful binding with LDAP DN: %s' % adminDn)
+            return ldapClient
+        except ldap.INVALID_CREDENTIALS, ex:
+            ldapClient.unbind()
+            raise AuthenticationError('Invalid LDAP credentials for admin user %s' % adminDn)
+        except ldap.SERVER_DOWN, ex:
+            raise CommunicationError('Cannot reach LDAP server %s' % serverUrl)
+        except Exception, ex:
+            raise InternalError('Unrecognized exception while binding to LDAP server %s: %s' % (serverUrl, ex))
+
+    @classmethod 
+    def unbind(cls, ldapClient):
+        try:
+            ldapClient.unbind()
+        except Exception, ex:
+            cls.getLogger().error('Could not unbind LDAP client: %s' % ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass
+

src/python/dm/common/utility/ldapLinuxPlatformUtility.py

+#!/usr/bin/env python
+
+import grp
+import ldap
+import ldap.modlist
+import copy
+import threading
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.exceptions.internalError import InternalError
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+from dmSubprocess import DmSubprocess
+
+class LdapLinuxPlatformUtility:
+
+    SUDO_CMD = '/usr/bin/sudo'
+    GROUPADD_CMD = '/usr/sbin/groupadd'
+    USERMOD_CMD = '/usr/sbin/usermod'
+    SETFACL_CMD = '/usr/bin/setfacl'
+    CHOWN_CMD = '/bin/chown'
+    GPASSWD_CMD = '/usr/bin/gpasswd'
+    NSCD_CMD = '/usr/sbin/nscd'
+    FIND_CMD = '/bin/find'
+
+    CONFIG_SECTION_NAME = 'LdapLinuxPlatformUtility'
+    REFRESH_AUTH_FILES_COMMAND_KEY = 'refreshauthfilescommand'
+    TIMER_DELAY_IN_SECONDS = 10
+
+    def __init__(self, serverUrl, adminDn, adminPasswordFile, groupDnFormat, minGidNumber=None):
+        self.serverUrl = serverUrl
+        self.adminDn = adminDn
+        self.groupDnFormat = groupDnFormat
+        self.minGidNumber = minGidNumber
+        self.refreshAuthFilesCommand = None
+        self.refreshAuthFilesTimer = None
+        self.getLogger().debug('Min GID number: %s' % minGidNumber)
+        # Remove first entry from the dn format to get tree base
+        self.groupBaseDn = ','.join(groupDnFormat.split(',')[1:])
+        self.getLogger().debug('Group base DN: %s' % self.groupBaseDn)
+        self.getLogger().debug('Using LDAP Admin password file: %s' % adminPasswordFile)
+        self.adminPassword = open(adminPasswordFile, 'r').readline().strip()
+        if not self.adminPassword:
+            raise ConfigurationError('LDAP password could not be found in %s file' % adminPasswordFile) 
+        self.ldapClient = None
+        self.__configure()
+
+    def __configure(self):
+        cm = ConfigurationManager.getInstance()
+        configItems = cm.getConfigItems(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME)
+        self.getLogger().debug('Got config items: %s' % configItems)
+        self.refreshAuthFilesCommand = cm.getConfigOption(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME, LdapLinuxPlatformUtility.REFRESH_AUTH_FILES_COMMAND_KEY)
+        self.getLogger().debug('Refresh auth files command: %s' % self.refreshAuthFilesCommand)
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    @classmethod
+    def executeSudoCommand(cls, cmd):
+        p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
+        p.run()
+
+    @classmethod
+    def executeCommand(cls, cmd):
+        p = DmSubprocess('%s' % (cmd))
+        p.run()
+
+    def getLdapClient(self):
+        if self.ldapClient is not None:
+            try:
+                self.ldapClient.simple_bind_s(self.adminDn, self.adminPassword)
+            except Exception, ex:
+                self.getLogger().error('Invalidating LDAP client due to error: %s' % ex)
+                self.unbind(self.ldapClient)
+                self.ldapClient = None
+
+        if not self.ldapClient:
+            self.ldapClient = self.bind(self.serverUrl, self.adminDn, self.adminPassword)
+        return self.ldapClient
+
+    @classmethod 
+    def bind(cls, serverUrl, adminDn, adminPassword):
+        try:
+            ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+            ldapClient = ldap.initialize(serverUrl)
+            ldapClient.set_option(ldap.OPT_REFERRALS,0)
+            ldapClient.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
+            ldapClient.simple_bind_s(adminDn, adminPassword)
+            cls.getLogger().debug('Successful binding with LDAP DN: %s' % adminDn)
+            return ldapClient
+        except ldap.INVALID_CREDENTIALS, ex:
+            ldapClient.unbind()
+            raise AuthenticationError('Invalid LDAP credentials for admin user %s' % adminDn)
+        except ldap.SERVER_DOWN, ex:
+            raise CommunicationError('Cannot reach LDAP server %s' % serverUrl)
+        except Exception, ex:
+            raise InternalError('Unrecognized exception while binding to LDAP server %s: %s' % (serverUrl, ex))
+
+    @classmethod 
+    def unbind(cls, ldapClient):
+        try:
+            ldapClient.unbind()
+        except Exception, ex:
+            cls.getLogger().error('Could not unbind LDAP client: %s' % ex)
+
+    def createGroup(self, name):
+        """ Create group if it does not exist. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        name = str(name)
+        try:
+            groupDn = self.groupDnFormat % name
+            logger.debug('Looking for group DN: %s' % groupDn)
+            # this method will throw exception if group is not found
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            logger.debug('Group %s already exists' % groupTuple[0])
+            return
+        except ldap.NO_SUCH_OBJECT, ex:
+            logger.debug('Group DN %s must be created' % groupDn)
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+        # determine gidNumber: look through all entries to get max value,
+        # then increment it
+        # ldap should really be configured to handle gid's automatically,
+        # and should prevent invalid entries
+        try:
+            logger.debug('Looking for max group id')
+            resultList = ldapClient.search_s(self.groupBaseDn, ldap.SCOPE_ONELEVEL, attrlist=['gidNumber'])
+            maxGid = 0
+            if self.minGidNumber:
+                maxGid = self.minGidNumber 
+            for result in resultList:
+                gidList = result[1].get('gidNumber', [])
+                gid = 0
+                if gidList:
+                    gid = int(gidList[0])
+
+                if gid > maxGid:
+                    maxGid = gid
+            gidNumber = str(maxGid + 1)
+            logger.debug('Max GID is %s, new group id will be %s' % (maxGid, gidNumber)) 
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        
+        attrs = {}
+        attrs['objectclass'] = ['posixGroup','top']
+        attrs['cn'] = name
+        attrs['gidNumber'] = [gidNumber]
+        attrs['memberUid'] = []
+        try:
+            groupLdif = ldap.modlist.addModlist(attrs)
+            ldapClient.add_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add group %s: %s' % (groupDn, ex))
+            raise InternalError(exception=ex)
+
+    def addUserToGroup(self, username, groupName):
+        """ Add user to group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username in memberUidList:
+                logger.debug('Group %s already contains user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Adding user %s to group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.append(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+        # Refresh auth files
+        self.refreshAuthFiles()
+
+    def deleteUserFromGroup(self, username, groupName):
+        """ Remove user from group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username not in memberUidList:
+                logger.debug('Group %s does not contain user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Removing user %s from group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.remove(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+        # Refresh auth files
+        self.refreshAuthFiles()
+
+    @classmethod
+    def createLocalGroup(cls, name):
+        """ Create local group if it does not exist. """
+        logger = cls.getLogger()
+        try:
+            group = grp.getgrnam(name)
+            logger.debug('Group %s already exists' % name)
+            return
+        except KeyError, ex:
+            # ok, we need to create group
+            pass
+        logger.debug('Creating group %s' % name)
+        cmd = '%s %s' % (cls.GROUPADD_CMD, name)
+        cls.executeSudoCommand(cmd)
+
+    def addLocalUserToGroup(self, username, groupName):
+        """ Add local user to group. """
+        self.addUserToGroup(username, groupName)
+
+    def deleteLocalUserFromGroup(self, username, groupName):
+        """ Remove local user from group. """
+        self.deleteUserFromGroup(username, groupName)
+
+    def getGroupInfo(self, groupName):
+        """ Get given group info. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            return groupTuple
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+    def setGroupUsers(self, groupName, usernameList):
+        """ Set list of users for a given group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Setting users %s for group %s' % (usernameList, groupName))
+        memberUidList = []
+        for username in usernameList:
+            memberUidList.append(str(username))
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not set users %s for group %s: %s' % (usernameList, groupName, ex))
+            raise InternalError(exception=ex)
+
+        # Refresh auth files 
+        self.refreshAuthFiles()
+
+    @classmethod
+    def setPathReadExecutePermissionsForGroup(cls, path, groupName):
+        """ Set path permissions for the given group. """
+        logger = cls.getLogger()
+        logger.debug('Allowing group %s to read/execute path %s' % (groupName, path))
+        cmd = '%s -m group\:%s\:rx "%s"' % (cls.SETFACL_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def changePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def recursivelyChangePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Recursively changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s -R \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def refreshNscdGroupCache(cls):
+        logger = cls.getLogger()
+        try:
+            logger.debug('Refreshing NCSD secondary group membership cache')
+            cmd = '%s -i group' % (cls.NSCD_CMD)
+            cls.executeSudoCommand(cmd)
+        except Exception, ex:
+            logger.warn('Failed to refresh NCSD group cache: %s' % (str(ex)))
+
+    # Refresh auth files in a timer, to avoid running command too often
+    def refreshAuthFiles(self):
+        if not self.refreshAuthFilesCommand:
+            return
+        if self.refreshAuthFilesTimer and self.refreshAuthFilesTimer.is_alive():
+            return
+        self.refreshAuthFilesTimer = threading.Timer(self.TIMER_DELAY_IN_SECONDS, self.__refreshAuthFiles)
+        self.refreshAuthFilesTimer.start()
+
+    def __refreshAuthFiles(self):
+        logger = self.getLogger()
+        try:
+            logger.debug('Refreshing auth files')
+            self.executeCommand(self.refreshAuthFilesCommand)
+        except Exception, ex:
+            logger.warn('Failed to refresh auth files: %s' % (str(ex)))
+
+    @classmethod
+    def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+        logger = cls.getLogger()
+        logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+        cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+        cls.executeCommand(cmd)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    utility = LdapLinuxPlatformUtility('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=dmadmin,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', '/tmp/ldapPassword', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
+    print utility.getGroupInfo(u's1id-test03')
+    #utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+    #utility.deleteLocalUserFromGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+

src/python/dm/common/utility/ldapUserManager.py

+#!/usr/bin/env python
+
+import ldap
+import ldap.modlist
+import copy
+import re
+import types
+from base64 import b16encode
+from base64 import b16decode
+from base64 import b64encode
+from base64 import b64decode
+
+from dm.common.objects.ldapUserInfo import LdapUserInfo
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.exceptions.internalError import InternalError
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.dmException import DmException
+from ldapClient import LdapClient
+
+class LdapUserManager(LdapClient):
+
+    CONFIG_SECTION_NAME = 'LdapUserManager'
+    SERVER_URL_KEY = 'serverurl'
+    ADMIN_DN_KEY = 'admindn'
+    ADMIN_PASSWORD_FILE_KEY = 'adminpasswordfile'
+    USER_DN_FORMAT_KEY = 'userdnformat'
+    GROUP_DN_FORMAT_KEY = 'groupdnformat'
+    MIN_GID_NUMBER_KEY = 'mingidnumber'
+
+    def __init__(self, serverUrl, adminDn, adminPasswordFile, userDnFormat, groupDnFormat, minGidNumber=None):
+        LdapClient.__init__(self, serverUrl, adminDn, adminPasswordFile)
+        self.userDnFormat = userDnFormat
+        self.groupDnFormat = groupDnFormat
+        self.minGidNumber = minGidNumber
+        #self.getLogger().debug('Min GID number: %s' % minGidNumber)
+        # Remove first entry from the dn format to get tree base
+        self.groupBaseDn = ','.join(groupDnFormat.split(',')[1:])
+        #self.getLogger().debug('Group base DN: %s' % self.groupBaseDn)
+
+    @classmethod
+    def createInstance(cls):
+        cm = ConfigurationManager.getInstance()
+        logger = cls.getLogger()
+        logger.debug('Creating LDAP User Manager instance')
+        serverUrl = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.SERVER_URL_KEY)
+        #logger.debug('Using server URL: %s' % serverUrl)
+        adminDn = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.ADMIN_DN_KEY)
+        #logger.debug('Using admin DN: %s' % adminDn)
+        adminPasswordFile = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.ADMIN_PASSWORD_FILE_KEY)
+        #logger.debug('Using admin password file: %s' % adminPasswordFile)
+        userDnFormat = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.USER_DN_FORMAT_KEY)
+        #logger.debug('Using user DN format: %s' % userDnFormat)
+        groupDnFormat = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.GROUP_DN_FORMAT_KEY)
+        #logger.debug('Using group DN format: %s' % groupDnFormat)
+        minGidNumber = cm.getConfigOption(LdapUserManager.CONFIG_SECTION_NAME, LdapUserManager.MIN_GID_NUMBER_KEY)
+        #logger.debug('Using min GID number: %s' % minGidNumber)
+        return LdapUserManager(serverUrl, adminDn, adminPasswordFile, userDnFormat, groupDnFormat, minGidNumber)
+
+    @classmethod
+    def decodePasswordHash(cls, b64EncodedString):
+        beginEncoding = b64EncodedString.find('}')
+        decodedString = b64EncodedString[beginEncoding+1:]
+        decodedString = b16encode(b64decode(decodedString)).upper()
+        return decodedString
+        
+    @classmethod
+    def encodePasswordHash(cls, passwordHash):
+        encodedString = '{SHA}'+b64encode(b16decode(passwordHash))
+        return encodedString
+        
+    @LdapClient.executeLdapCall
+    def getUserInfo(self, username):
+        """ Get user info. """
+        userDn = self.userDnFormat % str(username)
+        ldapClient = self.getLdapClient()
+        resultList = ldapClient.search_s(userDn, ldap.SCOPE_BASE, attrlist=['*','createTimeStamp','modifyTimestamp'])
+        userTuple = resultList[0]
+        return LdapUserInfo({'userDn' : userTuple[0], 'userAttrs' : userTuple[1]})
+
+    @LdapClient.executeLdapCall
+    def getUserInfos(self):
+        """ Get user infos. """
+        baseDnPos = self.userDnFormat.find(',')
+        baseDn = self.userDnFormat[baseDnPos+1:]
+        searchFilter = self.userDnFormat[0:baseDnPos] % '*'
+        ldapClient = self.getLdapClient()
+        resultList = ldapClient.search_s(baseDn, ldap.SCOPE_SUBTREE, searchFilter, attrlist=['*','createTimeStamp','modifyTimestamp'])
+        userList = []
+        for result in resultList:
+            userDn = result[0]
+            userAttrs = result[1]
+            userList.append(LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs}))
+        return userList
+
+    @LdapClient.executeLdapCall
+    def getUserInfoMapByUid(self):
+        """ Get user infos. """
+        baseDnPos = self.userDnFormat.find(',')
+        baseDn = self.userDnFormat[baseDnPos+1:]
+        searchFilter = self.userDnFormat[0:baseDnPos] % '*'
+        ldapClient = self.getLdapClient()
+        resultList = ldapClient.search_s(baseDn, ldap.SCOPE_SUBTREE, searchFilter)
+        userMap = {}
+        for result in resultList:
+            userDn = result[0]
+            userAttrs = result[1]
+            uid = userAttrs['uid'][0]
+            userMap[uid] = LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs})
+        return userMap
+
+    @LdapClient.executeLdapCall
+    def modifyUserInfo(self, username, attrDict):
+        """ Modify user. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        ldapUserInfo = self.getUserInfo(username)
+        userDn = ldapUserInfo.get('userDn')
+        userAttrs = ldapUserInfo.get('userAttrs')
+
+        # Remove internal LDAP attributes before creating copy for modifications
+        for key in [ 'modifyTimestamp', 'createTimeStamp' ]:
+            if userAttrs.has_key(key):
+                del userAttrs[key]
+        userAttrs2 = copy.copy(userAttrs)
+
+        middleName = attrDict.get('middleName', '')
+        fullName = '%s, %s' % (attrDict.get('lastName', ''), attrDict.get('firstName', ''))
+        if middleName:
+            fullName = '%s %s' % (fullName, middleName)
+
+        if attrDict.has_key('name'):
+            userAttrs2['cn'] = [attrDict.get('name')]
+            userAttrs2['revcn'] = ['%s %s' % (attrDict.get('lastName'), attrDict.get('firstName'))]
+            userAttrs2['sn'] = [attrDict.get('lastName')]
+            userAttrs2['givenName'] = [attrDict.get('firstName')]
+            userAttrs2['gecos'] = [attrDict.get('name')]
+        for key in ['loginShell', 'allowed-host', 'homeDirectory', 'home7Directory', 'gidNumber' ]:
+            if attrDict.has_key(key):
+                userAttrs2[key] = [attrDict.get(key)]
+        if attrDict.has_key('grp'):
+            userAttrs2['o'] = [attrDict.get('grp')]
+        if attrDict.has_key('inactive'):
+            if attrDict.get('inactive', 'N') == 'Y':
+                userAttrs2['inetUserStatus'] = ['inactive']
+            else:
+                userAttrs2['inetUserStatus'] = ['active']
+        passwordHash = attrDict.get('userPassword')
+        if not passwordHash:
+            passwordHash = attrDict.get('passwordHashValue')
+        if passwordHash:
+            userAttrs2['userPassword'] = [self.encodePasswordHash(passwordHash)]
+            #logger.debug('Encoded password entry: %s' % passwordHash)
+        logger.debug('Old user %s attrs: %s' % (username, userAttrs))
+        logger.debug('Modified user %s attrs: %s' % (username, userAttrs2))
+        userLdif = ldap.modlist.modifyModlist(userAttrs, userAttrs2)
+        ldapClient.modify_s(userDn, userLdif)
+        return LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs2})
+
+    @LdapClient.executeLdapCall
+    def createUserInfo(self, username, attrDict):
+        """ Create user. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        try:
+            ldapUserInfo = self.getUserInfo(username)
+            raise ObjectAlreadyExists('User %s already exists.' % username)
+        except ObjectNotFound, ex:
+            pass
+        middleName = attrDict.get('middleName') 
+        fullName = '%s, %s' % (attrDict.get('lastName', ''), attrDict.get('firstName', ''))
+        if middleName:
+            fullName = '%s %s' % (fullName, middleName)
+        badgeNumber = attrDict.get('badgeNumber')
+        uid = 'd%s' % badgeNumber
+        if username != uid:
+            raise InvalidArgument('Username %s and badge number %s are not consistent.' % (username, badgeNumber))
+
+        userAttrs = {}
+        userAttrs['objectclass'] = ['top', 'person', 'organizationalPerson', 'inetOrgPerson', 'inetUser', 'shadowAccount', 'posixAccount', 'passwordObject', 'APSexten', 'apple-user']
+        userAttrs['cn'] = [attrDict.get('name', fullName)]
+        userAttrs['revcn'] = ['%s %s' % (attrDict.get('lastName', ''), attrDict.get('firstName', ''))]
+        userAttrs['loginShell'] = ['/sbin/nologin']
+        userAttrs['allowed-host'] = ['dmid-vm']
+        userAttrs['uidNumber'] = ['10%s' % badgeNumber]
+        userAttrs['uid'] = [uid]
+        userAttrs['homeDirectory'] = ['/data']
+        userAttrs['home7Directory'] = ['/data']
+        userAttrs['o'] = [attrDict.get('grp')]
+        if attrDict.get('inactive', 'N') == 'Y':
+            userAttrs['inetUserStatus'] = ['inactive']
+        else:
+            userAttrs['inetUserStatus'] = ['active']
+        userAttrs['gidNumber'] = ['66001']
+        userAttrs['gecos'] = [attrDict.get('name', fullName)]
+        userAttrs['sn'] = [attrDict.get('lastName', '')]
+        userAttrs['givenName'] = [attrDict.get('firstName', '')]
+        passwordHash = attrDict.get('userPassword')
+        if not passwordHash:
+            passwordHash = attrDict.get('passwordHashValue')
+        if passwordHash:
+            userAttrs['userPassword'] = [self.encodePasswordHash(passwordHash)]
+        logger.debug('Creating user %s with attrs %s' % (username, userAttrs))
+
+        userDn = self.userDnFormat % str(username)
+        userLdif = ldap.modlist.addModlist(userAttrs)
+        ldapClient.add_s(userDn, userLdif)
+        return LdapUserInfo({'userDn' : userDn, 'userAttrs' : userAttrs})
+
+    def createGroup(self, name):
+        """ Create group if it does not exist. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        name = str(name)
+        try:
+            groupDn = self.groupDnFormat % name
+            logger.debug('Looking for group DN: %s' % groupDn)
+            # this method will throw exception if group is not found
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            logger.debug('Group %s already exists' % groupTuple[0])
+            return
+        except ldap.NO_SUCH_OBJECT, ex:
+            logger.debug('Group DN %s must be created' % groupDn)
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+        # determine gidNumber: look through all entries to get max value,
+        # then increment it
+        # ldap should really be configured to handle gid's automatically,
+        # and should prevent invalid entries
+        try:
+            logger.debug('Looking for max group id')
+            resultList = ldapClient.search_s(self.groupBaseDn, ldap.SCOPE_ONELEVEL, attrlist=['gidNumber'])
+            maxGid = 0
+            if self.minGidNumber:
+                maxGid = self.minGidNumber 
+            for result in resultList:
+                gidList = result[1].get('gidNumber', [])
+                gid = 0
+                if gidList:
+                    gid = int(gidList[0])
+
+                if gid > maxGid:
+                    maxGid = gid
+            gidNumber = str(maxGid + 1)
+            logger.debug('Max GID is %s, new group id will be %s' % (maxGid, gidNumber)) 
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        
+        attrs = {}
+        attrs['objectclass'] = ['posixGroup','top']
+        attrs['cn'] = name
+        attrs['gidNumber'] = [gidNumber]
+        attrs['memberUid'] = []
+        try:
+            groupLdif = ldap.modlist.addModlist(attrs)
+            ldapClient.add_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add group %s: %s' % (groupDn, ex))
+            raise InternalError(exception=ex)
+
+    def addUserToGroup(self, username, groupName):
+        """ Add user to group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username in memberUidList:
+                logger.debug('Group %s already contains user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Adding user %s to group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.append(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+    def deleteUserFromGroup(self, username, groupName):
+        """ Remove user from group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username not in memberUidList:
+                logger.debug('Group %s does not contain user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Removing user %s from group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.remove(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
+
+    def getGroupInfo(self, groupName):
+        """ Get given group info. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            return groupTuple
+        except Exception, ex:
+            raise InternalError(exception=ex)
+
+    def setGroupUsers(self, groupName, usernameList):
+        """ Set list of users for a given group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Setting users %s for group %s' % (usernameList, groupName))
+        memberUidList = []
+        for username in usernameList:
+            memberUidList.append(str(username))
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not set users %s for group %s: %s' % (usernameList, groupName, ex))
+            raise InternalError(exception=ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    utility = LdapUserManager('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=dmadmin,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', '/tmp/ldapPassword', userDnFormat='uid=%s,ou=DM,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
+    #print utility.getGroupInfo(u's1id-test03')
+    user = utility.getUserInfo(u'd225159')
+    print user
+    print user.getUserPassword()
+    print utility.decodePasswordHash(user.getUserPassword())
+    userMap = utility.getUserInfoMapByUid()
+    print 'N entries: ', len(userMap)
+    print userMap['d225159']
+    #for user in userList:
+    #    print user
+
+    utility.modifyUserInfo(u'd225159', {'homeDirectory' : '/data/'})
+    user = utility.getUserInfo(u'd225159')
+    print user
+    #user = utility.getUserInfo(u'd65114')
+    #print user
+
+    #passwordHash = LdapUserManager.decodePasswordHash(user['userAttrs']['userPassword'][0])
+    #print passwordHash
+
+    #print 'Creating user d000001'
+    #user = utility.getUserInfo(u'd000001')
+    #print user
+    #attrDict = {'badgeNumber' : '000001', 'name' : 'Test, User 1.', 'firstName' : 'User', 'middleName' : '1.', 'lastName' : 'Test', 'passwordHashValue' : '84673F4A8774846B1C096511C7F6B1329CEE5CCC', 'inactive' : 'N', 'grp' : 'XSD-TEST', 'lastChangeDate' : '2015-02-03 15:52:15', 'email' : 'UTEST1@APS.ANL.GOV', 'isUserNotAnlEmployee' : 'N' }
+    #try:
+    #    user = utility.createUserInfo(u'd000001', attrDict)
+    #except ObjectAlreadyExists, ex:
+    #    print 'User d000001 already exists, will modify it'
+    #    user = utility.updateUserInfo(u'd000001', attrDict)
+
+    #print user
+
+    #print LdapUserManager.encodePasswordHash(passwordHash)
+
+    #utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+    #utility.deleteLocalUserFromGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
+

src/python/dm/common/utility/ldapUtility.py

+#!/usr/bin/env python
+
+import ldap
+
+from dm.common.exceptions.authenticationError import AuthenticationError
+from dm.common.exceptions.communicationError import CommunicationError
+
+class LdapUtility:
+
+    def __init__(self, serverUrl, dnFormat):
+        self.serverUrl = serverUrl
+        self.dnFormat = dnFormat
+         
+    def checkCredentials(self, username, password):
+        """Verifies credentials for username and password. """
+        ldapUsername = self.dnFormat % username
+        try:
+            # build client
+            ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+            ldapClient = ldap.initialize(self.serverUrl)
+            ldapClient.set_option(ldap.OPT_REFERRALS,0)
+            ldapClient.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
+
+            #ldapClient.ststart_tls_s()
+            #ldapClient.set_option(ldap.OPT_X_TLS,ldap.OPT_X_TLS_DEMAND)
+            #ldapClient.set_option( ldap.OPT_X_TLS_DEMAND, True)
+            #ldapClient.set_option( ldap.OPT_DEBUG_LEVEL, 255)
+
+            # perform a synchronous bind
+            ldapClient.simple_bind_s(ldapUsername, password)
+            #ldapClient.whoami_s()
+        except ldap.INVALID_CREDENTIALS, ex:
+            ldapClient.unbind()
+            raise AuthenticationError('Invalid LDAP credentials for user %s' % username)
+        except ldap.SERVER_DOWN, ex:
+            raise CommunicationError('Cannot reach LDAP server %s' % self.serverUrl)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    ldapUtility = LdapUtility(serverUrl='ldaps://ldap.anl.gov:636', dnFormat='uid=%s,dc=anl,dc=gov')
+    ldapUtility.checkCredentials('sveseli', 'sv')

src/python/dm/common/utility/linuxUtility.py

+#!/usr/bin/env python
+
+import grp
+from dm.common.utility.loggingManager import LoggingManager
+from dmSubprocess import DmSubprocess
+
+class LinuxUtility:
+
+    SUDO_CMD = '/usr/bin/sudo'
+    GROUPADD_CMD = '/usr/sbin/groupadd'
+    USERMOD_CMD = '/usr/sbin/usermod'
+    SETFACL_CMD = '/usr/bin/setfacl'
+    CHOWN_CMD = '/bin/chown'
+    GPASSWD_CMD = '/usr/bin/gpasswd'
+    FIND_CMD = '/bin/find'
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    @classmethod
+    def executeSudoCommand(cls, cmd):
+        p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
+        p.run()
+
+    @classmethod
+    def executeCommand(cls, cmd):
+        p = DmSubprocess('%s' % (cmd))
+        p.run()
+
+    @classmethod
+    def createGroup(cls, name):
+        """ Create group if it does not exist. """
+        logger = cls.getLogger()
+        try:
+            group = grp.getgrnam(name)
+            logger.debug('Group %s already exists' % name)
+            return 
+        except KeyError, ex:
+            # ok, we need to create group 
+            pass
+        logger.debug('Creating group %s' % name)
+        cmd = '%s %s' % (cls.GROUPADD_CMD, name)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def addUserToGroup(cls, username, groupName):
+        """ Add user to group. """
+        logger = cls.getLogger()
+        logger.debug('Adding user %s to group %s' % (username, groupName))
+        cmd = '%s -a -G %s %s' % (cls.USERMOD_CMD, groupName, username)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def addLocalUserToGroup(cls, username, groupName):
+        """ Add local user to group. """
+        logger = cls.getLogger()
+        logger.debug('Adding local user %s to group %s' % (username, groupName))
+        cmd = '%s -a %s %s' % (cls.GPASSWD_CMD, username, groupName)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def deleteLocalUserFromGroup(cls, username, groupName):
+        """ Remove local user from group. """
+        logger = cls.getLogger()
+        logger.debug('Removing local user %s from group %s' % (username, groupName))
+        cmd = '%s -d %s %s' % (cls.GPASSWD_CMD, username, groupName)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def setGroupUsers(cls, groupName, usernameList):
+        """ Set list of users for a given group. """
+        logger = cls.getLogger()
+        logger.debug('Setting group %s users to: %s' % (groupName, usernameList))
+        cmd = '%s -M "%s" %s' % (cls.GPASSWD_CMD, ','.join(usernameList), groupName)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def setPathReadExecutePermissionsForGroup(cls, path, groupName):
+        """ Set path permissions for the given group. """
+        logger = cls.getLogger()
+        logger.debug('Allowing group %s to read/execute path %s' % (groupName, path))
+        cmd = '%s -m group\:%s\:rx "%s"' % (cls.SETFACL_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def changePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def recursivelyChangePathGroupOwner(cls, path, groupName):
+        logger = cls.getLogger()
+        logger.debug('Recursively changing group owner to %s for path %s' % (groupName, path))
+        cmd = '%s -R \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
+        cls.executeSudoCommand(cmd)
+
+    @classmethod
+    def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+        logger = cls.getLogger()
+        logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+        cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+        cls.executeCommand(cmd)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    LinuxUtility.createGroup('exp3')
+    LinuxUtility.addUserToGroup('hpc', 'exp3')
+    LinuxUtility.setPathReadExecutePermissionsForGroup('/home/sveseli/Work/DM/data/ESAF/exp3', 'exp3')
+

src/python/dm/common/utility/loggingManager.py

+#!/usr/bin/env python
+
+#
+# Logging manager singleton.
+#
+
+import re
+import sys
+import os.path
+import logging
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.exceptions.configurationError import ConfigurationError
+
+class LoggingManager:
+    """ 
+    The log manager class is initialized via a configuration file 
+    that may have the following sections:
+
+    ConsoleLogging      # Used for output on the screen
+    FileLogging         # Used for logging into a file
+
+    Each section in the configuration file should have the following 
+    keys:
+        handler     # Indicates which handler class to use 
+        level       # Indicates logging level
+        format      # Indicates format for log messages
+        dateFormat  # Indicates date format used for log messages
+
+    Given below is an example of a valid configuration file:
+
+    [ConsoleLogging]
+        handler=ConsoleLoggingHandler(sys.stdout,)
+        level=info
+        format=[%(levelname)s] %(message)s 
+        dateFormat=%m/%d/%y %H:%M:%S
+    [FileLogging]
+        handler=TimedRotatingFileLoggingHandler('/tmp/dm.log')
+        level=debug
+        format=%(asctime)s,%(msecs)d [%(levelname)s] %(module)s:%(lineno)d %(user)s@%(host)s %(name)s (%(process)d): %(message)s
+        dateFormat=%m/%d/%y %H:%M:%S
+    """
+
+    # Get singleton instance.
+    @classmethod
+    def getInstance(cls):
+        from dm.common.utility.loggingManager import LoggingManager
+        try:
+            lm = LoggingManager()
+        except LoggingManager, ex:
+            lm = ex
+        return lm
+
+    # Singleton.
+    __instance = None
+
+    def __init__(self):
+        if LoggingManager.__instance:
+             raise LoggingManager.__instance
+        LoggingManager.__instance = self
+        self.consoleHandler = None
+        self.fileHandlerList = []
+        self.maxIntLevel = logging.CRITICAL
+        self.minIntLevel = logging.NOTSET
+        self.levelRegExList = []
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.initFlag = False
+
+    def setMinLogLevel(self, minLogLevel=logging.INFO):
+        self.minIntLevel = minLogLevel
+
+    def parseLevelRegEx(self, levelRegExList):
+        """ Parse expressions of the form <regex>=<log level>. """
+        lines = levelRegExList.split('\n')
+        for line in lines:
+            try:
+                 (regex, level) = line.rsplit('=', 1)
+                 pattern = re.compile(regex)
+                 tuple = (pattern, logging.getLevelName(level.upper()))
+                 self.levelRegExList.append(tuple)
+            except Exception, ex:
+                 self.logger.error('Parser error in log configuration file: %s' % line)
+                 self.logger.exception(ex)
+
+    # Get Log Level based on a string representation
+    def getIntLogLevel(self, levelStr):
+        level = logging.getLevelName(levelStr)
+        # Level should be an integer
+        try:
+            return int(level) 
+        except ValueError, ex:
+            raise ConfigurationError('"%s" is not valid log level' % levelStr)
+
+    # Configure log handlers.
+    def configure(self):
+        """ Configure log handlers from the config file. """
+        cm = ConfigurationManager.getInstance()
+        configFile = cm.getConfigFile()
+        configParser = cm.getConfigParserFromConfigFile(configFile)
+        configSections = cm.getConfigSectionsFromConfigParser(configParser)
+
+        # Console handler.
+        defaults = { 
+           'level' : cm.getConsoleLogLevel(),
+           'format' : cm.getLogRecordFormat(),
+           'dateFormat' : cm.getLogDateFormat(),
+           'handler' : 'ConsoleLoggingHandler(sys.stdout,)'
+        }
+        if not self.consoleHandler:
+            consoleHandler = self.__configureHandler(configParser, 'ConsoleLogging', defaults)
+
+            if consoleHandler != None:
+                self.consoleHandler = consoleHandler
+
+        # File logging. 
+        # Do not configure if log directory does not exist.
+        defaults['handler'] = None
+        defaults['level'] = cm.getFileLogLevel() 
+        if not os.path.exists(configFile):
+            # No config file, we'll configure default.
+            defaultLogFile = cm.getLogFile()
+            defaultLogDir = os.path.dirname(defaultLogFile)
+            if os.path.exists(defaultLogDir):
+                handler = 'TimedRotatingFileLoggingHandler("%s")' % defaultLogFile 
+                defaults['handler'] = handler
+                fileHandler = self.__configureHandler(configParser, 'FileLogging', defaults)
+                if fileHandler != None:
+                    self.fileHandlerList.append(fileHandler)
+
+        else:
+            # Parse all file loggers present in the config file
+            for configSection in configSections:
+                if configSection.startswith('FileLogging'):
+                     fileHandler = self.__configureHandler(configParser, configSection, defaults)
+                     if fileHandler != None:
+                         self.fileHandlerList.append(fileHandler)
+
+        # Add handlers to the root logger.  Use logging class here
+        # to make sure we can have a logger when we parse the 
+        # logger expressions
+        rootLogger = logging.getLogger('')
+        for handler in [self.consoleHandler] + self.fileHandlerList:
+            rootLogger.addHandler(handler)
+
+        # Get a logger factory based on our current config 
+        self.configureLoggers(configParser, defaultLevel='debug')
+
+    def configureLoggers(self, configParser, defaultLevel='error'):
+        rootLogLevel = 'error'
+        levelRegEx = '^.*$=%s' % (defaultLevel)
+        if configParser is not None and configParser.has_section('LoggerLevels'):
+            rootLogLevel = ConfigurationManager.getOptionFromConfigParser(configParser, 'LoggerLevels', 'root', rootLogLevel)
+            levelRegEx = ConfigurationManager.getOptionFromConfigParser(configParser, 'LoggerLevels', 'levelregex', levelRegEx)
+
+        rootLevelInt = logging.getLevelName(rootLogLevel.upper())
+        logging.getLogger('').root.setLevel(rootLevelInt)
+        logging.getLogger('').debug('Set root logger to %s' % rootLevelInt)
+
+        if not levelRegEx:
+            return
+
+        # Parse expressions of the form <regex>=<log-level>. """
+        lines = levelRegEx.split('\n')
+        for line in lines:
+            try:
+                # Use the right split so we can have '='s in the regex
+                (regex, level) = line.rsplit('=', 1)
+                pattern = re.compile(regex)
+                tuple = (pattern, logging.getLevelName(level.upper()))
+                self.levelRegExList.append(tuple)
+            except Exception, ex:
+                # Do not fail
+                self.logger.error('Parser error in log configuration file: %s' % line)
+                self.logger.exception(ex)
+
+    # Configure particular handler with given defaults.
+    def __configureHandler(self, configParser, configSection, defaults):
+        """ Configure specified handler with a given defaults. """
+        handlerOption = defaults['handler']
+        try:
+            if configParser is not None:
+                handlerOption = configParser.get(configSection, 'handler')
+        except Exception, ex:
+            pass
+
+        # If handlerOption is empty, handler cannot be instantiated.
+        handler = None
+        if handlerOption != None:
+            # Handler argument format: MyHandler(arg1, arg2, ...)
+            # Module will be in lowercase letters, but the class
+            # should be capitalized.
+            handlerName = re.sub('\(.*', '', handlerOption)
+            moduleName = handlerName[0].lower() + handlerName[1:]
+            try:
+                exec 'from dm.common.utility import %s' % (moduleName)
+                exec 'handler = %s.%s' % (moduleName, handlerOption)
+            except IOError, ex:
+                errNo, errMsg = ex
+                import errno
+
+                # If the exception raised is an I/O permissions error, ignore
+                # it and disable this log handler.  This allows non-root users
+                # to use the (system-wide) default log configuration
+                if errNo != errno.EACCES:
+                    raise
+                handler = None 
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+        # Only request setting from the config file if it was
+        # not set via environment variable, or programmatically.
+        cm = ConfigurationManager.getInstance()
+        if handler != None:
+            try:
+                level = cm.getOptionFromConfigParser(configParser, configSection, 'level', defaults['level'])
+                intLevel = self.getIntLogLevel(level.upper())
+                handler.setLevel(intLevel)
+
+                format = cm.getOptionFromConfigParser(configParser, configSection, 'format', defaults['format'])
+                dateFormat = cm.getOptionFromConfigParser(configParser, configSection, 'dateFormat', defaults['dateFormat'])
+
+                handler.setFormatter(logging.Formatter(format, dateFormat))
+            except Exception, ex:
+                raise ConfigurationError(exception=ex)
+
+            # Apply filters to handler
+            filter = None
+            try:
+                filter = configParser.get(configSection, 'filter')
+                if filter:
+                    handler.addFilter(logging.Filter(filter))
+            except Exception, ex:
+                pass
+        return handler
+
+    def getLogger(self, name='defaultLogger'):
+        if not self.initFlag:
+            self.initFlag = True
+            self.configure()
+        logger = logging.getLogger(name)
+        logger.setLevel(self.getLevel(name))
+        return logger
+
+    def getLevel(self, name):
+        # Match from the known regex list.
+        level = logging.NOTSET
+
+        # The last regex is most important.
+        for e in reversed(self.levelRegExList):
+            (pattern, level) = e
+
+            # If we return not None it is a match
+            if not None == pattern.match(name):
+                break
+
+        if level > self.maxIntLevel:
+            level = self.maxIntLevel
+        if level < self.minIntLevel:
+            level = self.minIntLevel
+        return level
+
+    def setConsoleLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            self.consoleHandler.setLevel(intLevel)
+            self.maxIntLevel = intLevel
+            self.logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+    def setFileLogLevel(self, level):
+        try:
+            # We need to override the logger levels and the handler
+            intLevel = self.getIntLogLevel(level.upper())
+            for handler in self.fileHandlerList:
+                handler.setLevel(intLevel)
+            self.maxIntLevel = intLevel
+            self.logger.setLevel(intLevel)
+        except Exception, ex:
+            raise ConfigurationError(exception=ex)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    lm = LoggingManager.getInstance()
+    logger = lm.getLogger('Main')
+    logger.error('Error In Main')
+    logger.debug('Debug In Main')
+    logger.warn('Warn In Main')
+    logger.info('Info In Main')
+    logger = lm.getLogger('Main')
+    logger.info('Info In Main')
+    logger = lm.getLogger('')
+    logger.info('Info using root logger')
+    logger = lm.getLogger('Main.2')
+    logger.info('Info in Main.2')
+    lm.setConsoleLogLevel('info')
+    logger.debug('You should not see this message')
+    lm.setConsoleLogLevel('debug')
+    logger.debug('Debug in Main.2')

src/python/dm/common/utility/noopPlatformUtility.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+
+class NoopPlatformUtility:
+
+    def __init__(self):
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+
+    def createGroup(self, name):
+        self.logger.debug('createGroup called for %s' % name)
+
+    def addUserToGroup(self, username, groupName):
+        self.logger.debug('addUserToGroup called for %s, %s' % (username, groupName))
+
+    def deleteUserFromGroup(self, username, groupName):
+        self.logger.debug('deleteUserFromGroup called for %s, %s' % (username, groupName))
+
+    def createLocalGroup(self, name):
+        self.logger.debug('createLocalGroup called for %s' % name)
+
+    def addLocalUserToGroup(self, username, groupName):
+        self.logger.debug('addLocalUserToGroup called for %s, %s' % (username, groupName))
+
+    def deleteLocalUserFromGroup(self, username, groupName):
+        self.logger.debug('adddeleteLocalUserFromGroup called for %s, %s' % (username, groupName))
+
+    def getGroupInfo(self, groupName):
+        self.logger.debug('getGroupInfo called for %s' % name)
+
+    def setGroupUsers(self, groupName, usernameList):
+        self.logger.debug('setGroupUsers called for %s, %s' % (groupName, usernameList))
+
+    def setPathReadExecutePermissionsForGroup(self, path, groupName):
+        self.logger.debug('setPathReadExecutePermissionsForGroup called for %s, %s' % (path, groupName))
+
+    def changePathGroupOwner(self, path, groupName):
+        self.logger.debug('changePathGroupOwner called for %s, %s' % (path, groupName))
+
+    def recursivelyChangePathGroupOwner(self, path, groupName):
+        self.logger.debug('recursivelyChangePathGroupOwner called for %s, %s' % (path, groupName))
+
+    def refreshNscdGroupCache(self):
+        self.logger.debug('refreshNscdGroupCache called')
+
+    def refreshAuthFiles(self):
+        self.logger.debug('refreshAuthFiles called')
+
+    def chmodPathForFilesInDirectory(self, directoryPath, fileMode):
+        self.logger.debug('chmodPathForFilesInDirectory called for %s, %s' % (directoryPath, fileMode))
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/utility/objectCache.py

+#!/usr/bin/env python
+
+import threading
+import time
+from collections import deque
+
+class ObjectCache:
+    """
+    Cache objects identified by id. Objects are removed from cache
+    based on the last accessed algorithm.
+    """
+
+    # How much larger than object cache should time stamp deq be
+    # allowed to grow.
+    DEFAULT_TIME_STAMP_DEQ_SIZE_FACTOR = 2
+    # Cache info expiration time.
+    DEFAULT_OBJECT_LIFETIME = 60 # seconds
+
+    def __init__(self, cacheSize, objectLifetime=DEFAULT_OBJECT_LIFETIME, objectClass=None):
+        self.lock = threading.RLock()
+        self.objectMap = {} # id/object map
+        self.timestampDeq = deque() # timestamp deq
+        self.cacheSize = cacheSize
+        self.objectLifetime = objectLifetime
+        self.deqSize = ObjectCache.DEFAULT_TIME_STAMP_DEQ_SIZE_FACTOR*cacheSize
+        self.objectClass = objectClass
+
+    def setCacheSize(self, cacheSize):
+        self.cacheSize = cacheSize
+
+    def setObjectLifetime(self, objectLifetime):
+        self.objectLifetime = objectLifetime
+
+    def __purgeOne(self):
+        # Get rid of one cached item based on the last accessed algorithm.
+        while True:
+            deqEntry = self.timestampDeq.popleft()
+            oldId = deqEntry[0]
+            cachedEntry = self.objectMap.get(oldId)
+            if cachedEntry is not None:
+                # Timestamp entry is valid.
+                if cachedEntry == deqEntry:
+                    # Found an old item, get rid of it from the cache.
+                    del self.objectMap[oldId]
+                    break
+        # Done.
+        return
+
+    def __purgeTimestampDeq(self):
+        # Get rid of stale entries.
+        timestampDeq = deque()
+        while len(self.timestampDeq):
+            deqEntry = self.timestampDeq.popleft()
+            id = deqEntry[0]
+            cachedEntry = self.objectMap.get(id)
+            if cachedEntry is not None:
+                # Timestamp entry is valid.
+                if cachedEntry == deqEntry:
+                    # Found current item, keep it.
+                    timestampDeq.append(deqEntry)
+        # Done.
+        self.timestampDeq = timestampDeq 
+        return
+
+    def put(self, id, item, objectLifetime=None):
+        updateTime = time.time()
+        expirationTime = updateTime + self.objectLifetime
+        if objectLifetime is not None:
+            expirationTime = updateTime + objectLifetime
+        entry = (id, item, updateTime, expirationTime)
+        self.lock.acquire()
+        try:
+            self.objectMap[id] = entry
+            self.timestampDeq.append(entry)
+            if len(self.objectMap) > self.cacheSize:
+                self.__purgeOne()
+            if len(self.timestampDeq) > self.deqSize:
+                self.__purgeTimestampDeq()
+        
+        finally:
+            self.lock.release()
+
+    def get(self, id):
+        item = None
+        itemTuple = self.objectMap.get(id)
+        if itemTuple is not None:
+            id, item, updateTime, expirationTime = itemTuple
+        return item
+
+    def getAll(self):
+        # Item tuple: id, item, updateTime, expirationTime = itemTuple
+        return map(lambda itemTuple:itemTuple[1], self.objectMap.values())
+
+    def getItemTuple(self, id):
+        itemTuple = self.objectMap.get(id)
+        if itemTuple is None:
+            itemTuple = (id, None, None, None)
+        return itemTuple
+
+    def remove(self, id):
+        self.lock.acquire()
+        try:
+            item = self.objectMap.get(id)
+            if item is not None:
+                del self.objectMap[id]
+            return item
+        finally:
+            self.lock.release()
+
+    def isEmpty(self):
+        return len(self.objectMap) == 0
+
+    def size(self):
+        return len(self.objectMap) 
+
+    def __str__(self):
+        return '%s' % self.timestampDeq
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    c = ObjectCache(3)
+
+    class Item:
+        def __init__(self, id):
+            self.id = id
+        def getId(self):
+            return self.id
+        def __str__(self):
+            return '%s' % self.id
+
+    class Item2:
+        def __init__(self, name):
+            self.name = name 
+        def getName(self):
+            return self.name
+        def __str__(self):
+            return '%s' % self.name
+
+    for i in range(0,5):
+        item = Item(i)
+        c.put(i, item)
+        print 'Added item: ', item
+        print 'Cache: ', c
+        time.sleep(1)
+
+    for j in range(0,3):
+        item = Item(2)
+        c.put(2, item)
+        print 'Updated item: ', item
+        print 'Cache: ', c
+        time.sleep(1)
+
+    item = c.remove(2)
+    print 'Deleted item 2: ', item
+    print 'Cache: ', c
+    time.sleep(1)
+    item = c.get(2)
+    print 'Got item 2: ', item
+    print 'Cache: ', c
+    print
+    time.sleep(1)
+
+    print
+    c = ObjectCache(3)
+    c.put('sv', Item2('sv')) 
+    print c
+    i = c.get('sv')
+    print i
+    print 'Done'
+

src/python/dm/common/utility/objectTracker.py

+#!/usr/bin/env python
+
+import threading
+import time
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.utility.objectCache import ObjectCache
+from dm.common.utility.singleton import Singleton
+
+class ObjectTracker(Singleton):
+
+    DEFAULT_CACHE_SIZE = 1000
+    DEFAULT_OBJECT_LIFETIME_IN_SECONDS = 86400*30  # 30 days
+
+    # Singleton.
+    __instanceLock = threading.RLock()
+
+    # Cache configuration
+    objectClass = None
+    objectLifetime = DEFAULT_OBJECT_LIFETIME_IN_SECONDS 
+    cacheSize = DEFAULT_CACHE_SIZE
+
+    def __init__(self, *args, **kwargs):
+        ObjectTracker.__instanceLock.acquire()
+        try:
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+            self.logger.debug('Initializing')
+            self.objectCache = ObjectCache(self.cacheSize, self.objectLifetime, self.objectClass)
+            self.logger.debug('Initialization complete')
+        finally:
+            ObjectTracker.__instanceLock.release()
+
+    def put(self, id, item, objectLifetime=None):
+        return self.objectCache.put(id, item, objectLifetime)
+
+    def get(self, id):
+        return self.objectCache.get(id)
+
+    def getAll(self):
+        return self.objectCache.getAll()
+
+    def remove(self, id):
+        return self.objectCache.remove(id)
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    ot = ObjectTracker.getInstance()
+    print ot
+    ot2 = ObjectTracker()
+    print ot2
+
+

src/python/dm/common/utility/objectUtility.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+
+class ObjectUtility:
+
+    @classmethod
+    def createObjectInstance(cls, moduleName, className, constructor, importPath=None):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        logger.debug('Creating object: %s, %s, %s' % (moduleName, className, constructor))
+        if importPath is not None:
+            # Add import path if it was provided
+            sys.path.append[importPath]
+
+        cmd = 'from %s import %s' % (moduleName, className)
+        exec cmd
+        cmd = 'objectInstance = %s' % (constructor)
+        exec cmd
+
+        if importPath is not None:
+            # Remove import path that was added
+            del sys.path[-1]
+        return objectInstance
+

src/python/dm/common/utility/osUtility.py

+#!/usr/bin/env python
+
+import os
+import stat
+
+class OsUtility:
+
+    @classmethod
+    def createDir(cls, path, mode=None):
+        """ Create directory if it does not exist already. """
+        if not os.path.isdir(path):
+            os.makedirs(path)
+            if mode is not None:
+                os.chmod(path, mode)
+
+    @classmethod
+    def removeLink(cls, path):
+        """ Remove link on a given path. """
+        if not os.path.islink(path):
+            return
+        os.remove(path)
+
+    @classmethod
+    def removeFile(cls, path):
+        """ Remove file on a given path. """
+        if not os.path.isfile(path):
+            return
+        os.remove(path)
+
+    @classmethod
+    def removeAndIgnoreErrors(cls, path):
+        """ Remove file on a given path and ignore any errors. """
+        try:
+            os.remove(path)
+        except Exception, ex:
+            pass
+
+    @classmethod
+    def removeDir(cls, path):
+        """ Remove dir on a given path, even if it is not empty. """
+        if not os.path.isdir(path):
+            return
+        files=os.listdir(path)
+        for f in files:
+            fullPath=os.path.join(path, f)
+            if os.path.islink(fullPath) or not os.path.isdir(fullPath):
+                os.remove(fullPath)
+            else:
+                removeDir(fullPath)
+        os.rmdir(path)
+        
+    @classmethod
+    def chmodPath(cls, path, fileMode=None, dirMode=None):
+        if os.path.isfile(path):
+            if fileMode is not None:
+                os.chmod(path, fileMode)
+            return 
+        elif os.path.isdir(path):
+            files = os.listdir(path)
+            for f in files:
+                fullPath = os.path.join(path, f)
+                cls.chmodPath(fullPath, fileMode, dirMode)
+            if dirMode is not None:
+                os.chmod(path, dirMode)
+
+    @classmethod
+    def chownPath(cls, path, uid, gid):
+        """ Change owner on a given path recursively.  """
+        if os.path.isfile(path):
+            os.chown(path, uid, gid)
+            return 
+        elif os.path.islink(path):
+            os.lchown(path, uid, gid)
+            return 
+        elif os.path.isdir(path):
+            files = os.listdir(path)
+            for f in files:
+                fullPath = os.path.join(path, f)
+                cls.chownPath(fullPath, uid, gid)
+            os.chown(path, uid, gid)
+
+    @classmethod
+    def chownPathByUserName(cls, path, userName):
+        """ Change owner on a given path recursively.  """
+        import pwd
+        user = pwd.getpwnam(userName)
+        chownPath(path, user.pw_uid, user.pw_gid)
+
+    @classmethod
+    def findFiles(cls, dirPath, fileList=None):
+        """ List files in a given directory. Return list of absolute paths.
+            Do not follow symbolic links.
+        """
+        fList = fileList
+        if not fList:
+            fList = []
+        if os.path.isdir(dirPath):
+            files = os.listdir(dirPath)
+            for f in files:
+                fullPath = os.path.join(dirPath, f)
+                if os.path.isfile(fullPath):
+                    fList.append(fullPath)
+                elif os.path.isdir(fullPath):
+                    fList = cls.findFiles(fullPath, fList)
+        return fList
+
+    @classmethod
+    def findFilesAsDict(cls, dirPath, fileDict=None):
+        """ Find files in a given directory. Return dictionary of 
+            absolute paths.
+            Do not follow symbolic links.
+        """
+        fDict = fileDict
+        if not fDict:
+            fDict = {}
+        if os.path.isdir(dirPath):
+            files = os.listdir(dirPath)
+            for f in files:
+                fullPath = os.path.join(dirPath, f)
+                if os.path.isfile(fullPath):
+                    statResult = os.stat(fullPath)
+                    fileInfo = {}
+                    fileInfo['fileSize'] = statResult[stat.ST_SIZE]
+                    fileInfo['fileCreationTime'] = statResult[stat.ST_CTIME]
+                    fileInfo['fileModificationTime'] = statResult[stat.ST_MTIME]
+                    fDict[fullPath] = fileInfo
+                elif os.path.isdir(fullPath):
+                    fDict = cls.findFilesAsDict(fullPath, fDict)
+        return fDict
+
+    @classmethod
+    def importNameFromFile(cls, name, filePath):
+        """ Import specified name from file. """
+        import sys
+        import os.path
+        dirName = os.path.dirname(filePath)
+        moduleName = os.path.basename(filePath).replace('.py', '')
+        sys.path = [dirName] + sys.path
+        cmd = 'from %s import %s as tmpObject' % (moduleName, name)
+        exec cmd
+        del sys.path[0]
+        return tmpObject
+
+    @classmethod
+    def getUserHomeDir(cls):
+        """ Get current user home directory. """
+        from os.path import expanduser
+        home = expanduser('~')
+        return home
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    OsUtility.chmodPath('/home/sveseli/Work/DM/data/ESAF/exp3', 0700)
+

src/python/dm/common/utility/rsyncFileTransfer.py

+#!/usr/bin/env python
+
+from dm.common.utility.dmSubprocess import DmSubprocess
+
+class RsyncFileTransfer:
+
+    COMMAND = 'rsync'
+
+    def __init__(self, src, dest, flags='-arvlP'):
+        self.src = src
+        self.dest = dest
+        self.flags = flags
+        self.command = '%s %s %s %s' % (self.COMMAND, self.flags, self.src, self.dest)
+        self.subprocess = DmSubprocess.getSubprocess(self.command)
+
+    def execute(self):
+        return self.subprocess.run()
+
+    def getStdOut(self):
+        return self.subprocess.getStdOut()
+
+    def getStdErr(self):
+        return self.subprocess.getStdErr()
+
+    def getExitStatus(self):
+        return self.subprocess.getExitStatus()
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    ft = RsyncFileTransfer('/tmp/abc', '/tmp/abc2')
+    ft.execute()
+    print 'StdOut: ', ft.getStdOut()
+    print 'StdErr: ', ft.getStdErr()
+    print 'Exit Status: ', ft.getExitStatus()

src/python/dm/common/utility/sftpUtility.py

+#!/usr/bin/env python
+
+import threading
+import copy
+import stat
+import pysftp
+from dm.common.utility.timeUtility import TimeUtility
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.commandFailed import CommandFailed
+import urlparse
+
+class SftpUtility:
+
+    DEFAULT_PORT = 22
+
+    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.privateKey = privateKey
+        self.sftpClient = None
+        self.lock = threading.RLock()
+
+    @classmethod
+    def parseFtpUrl(cls, url, defaultHost=None, defaultPort=None):
+        host = defaultHost
+        port = defaultPort
+        scheme = None
+        dirPath = url
+        if url.startswith('ftp://'):
+            parseResult = urlparse.urlparse(url)
+            scheme = parseResult.scheme
+            netlocTokens = parseResult.netloc.split(':')
+            host = netlocTokens[0]
+            if len(netlocTokens) > 1:
+                port = int(netlocTokens[1])
+            dirPath = parseResult.path
+        return (scheme, host, port, dirPath)
+
+    @classmethod
+    def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None):
+        sftp = pysftp.Connection(host, username=username, password=password, port=port, private_key=privateKey)
+        return sftp
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+
+    def __parseKeyValue(cls, keyValue, outputDict={}):
+        key,value = keyValue.split('=')
+        value = value.strip()
+        outputDict[key] = value
+        return outputDict
+        
+    def getFiles(self, dirPath, fileDict={}, replacementDirPath=None):
+        self.lock.acquire()
+        try:
+            if not self.sftpClient:
+                self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
+            if not replacementDirPath:
+                replacementDirPath = dirPath
+            try:
+                attrs = self.sftpClient.listdir_attr(dirPath)
+            except Exception, ex:
+                self.getLogger().error('Could not retrieve files from %s: %s' % (dirPath,ex))
+                self.closeConnection()
+                raise
+        finally:
+            self.lock.release()
+
+        for attr in attrs:
+            fileName = attr.filename
+            mode = attr.st_mode
+            if stat.S_ISDIR(mode):
+                dirPath2 = '%s/%s' % (dirPath, fileName)
+                replacementDirPath2 = '%s/%s' % (replacementDirPath, fileName)
+                self.getFiles(dirPath2, fileDict, replacementDirPath2)
+            elif stat.S_ISREG(mode):
+                filePath = '%s/%s' % (replacementDirPath, fileName)
+                fileInfo = {'fileSize' : attr.st_size, 
+                    'fileModificationTime' : attr.st_mtime }
+                fileDict[filePath] = fileInfo
+        return fileDict
+
+    def getMd5Sum(self, filePath, fileInfo={}):
+        self.lock.acquire()
+        try:
+            if not self.sftpClient:
+                self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
+            try:
+                #md5Sum = self.sftpClient.execute('md5sum "%s"' % filePath)[0].split()[0]
+                output = self.sftpClient.execute('md5sum "%s"' % filePath)[0].strip()
+                if not output.endswith(filePath):
+                    raise CommandFailed(output)
+                md5Sum = output.split()[0]
+                fileInfo['md5Sum'] = md5Sum
+            except CommandFailed, ex:
+                self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex))
+                raise
+            except Exception, ex:
+                self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex))
+                self.closeConnection()
+                raise
+            return md5Sum
+        finally:
+            self.lock.release()
+
+    def statFile(self, filePath, fileInfo={}):
+        self.lock.acquire()
+        try:
+            if not self.sftpClient:
+                self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
+            try:
+                attr = self.sftpClient.stat(filePath)
+                fileInfo['fileSize'] = attr.st_size 
+                fileInfo['fileModificationTime'] = attr.st_mtime
+            except Exception, ex:
+                self.getLogger().error('Could not get stat file %s: %s' % (filePath,ex))
+                self.closeConnection()
+                raise
+            return fileInfo
+        finally:
+            self.lock.release()
+
+    def closeConnection(self):
+        logger = self.getLogger()
+        self.lock.acquire()
+        try:
+            try:
+                if self.sftpClient:
+                    logger.warn('Closing SFTP connection to host %s' % self.host)
+                    self.sftpClient.close()
+            except Exception, ex:
+                logger.error('Could not close SFTP connection to host %s: %s' % (self.host, ex))
+            self.sftpClient = None
+        finally:
+            self.lock.release()
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    #sftpUtility = SftpUtility('s1dserv', username='dmadmin', password='theKey12')
+    #sftpUtility = SftpUtility('s1dserv',privateKey='/home/beams/DMADMIN/.ssh/id_dsa')
+    sftpUtility = SftpUtility('xstor-devel')
+    files = sftpUtility.getFiles('/data/testing/id7-test02')
+    #files = sftpUtility.getFiles('/export/dm/test')
+    print files
+    #print sftpUtility.getMd5Sum('/export/dm/test/testfile03(2nd).txt')
+    #print sftpUtility.getMd5Sum('/export/dm/test/testfile 04.txt')
+    #print 'Closing connection'
+    #sftpUtility.closeConnection()
+    #print sftpUtility.statFile('/export/dm/test/testfile01')

src/python/dm/common/utility/singleton.py

+#!/usr/bin/env python
+
+class Singleton(object):
+
+    __instance = None
+
+    # This class will behave properly as long as getInstance() is called.
+    # If object is constructed using constructor, __init__() will be called
+    # multiple times in the derived class (i.e., one needs protection
+    # against multiple initializations in the derived class)
+    def __new__(cls, *args, **kwargs):
+        # Allow subclasses to create their own instances.
+        if cls.__instance is None or cls != type(cls.__instance):
+            instance = object.__new__(cls, *args, **kwargs)
+            instance.__init__(*args, **kwargs)
+            cls.__instance = instance
+        return cls.__instance
+
+    @classmethod
+    def getInstance(cls, *args, **kwargs):
+        return cls.__new__(cls, *args, **kwargs)
+
+    def __init__(self, *args, **kwargs):
+        # Only initialize once.
+        if self.__class__.__instance is not None:
+            return
+
+####################################################################
+# Testing
+if __name__ == '__main__':
+    print 'Testing Singleton class'
+    s1 = Singleton.getInstance()
+    s2 = Singleton()
+    s3 = Singleton.getInstance()
+    s4 = Singleton()
+    print 'S1: ', s1
+    print 'S2: ', s2
+    print 'S3: ', s3
+    print 'S4: ', s4
+    class A(Singleton):
+        __instance = None
+        def __init__(self, x):
+            if self.__class__.__instance is None:
+                print 'in A.__init__()'
+                self.x = x
+    class B(Singleton):
+        def __init__(self, x):
+            self.x = x
+    class C(Singleton):
+        def __init__(self):
+            self.x = 14
+
+    print
+    print 'Testing Class A'
+    print 'Init A(3)'
+    a1 = A(3)
+    print 'Init A(4)'
+    a2 = A(4)
+    print 'A.getInstance()'
+    a3 = A.getInstance()
+    print 'A.getInstance()'
+    a4 = A.getInstance()
+    print a1
+    print a2
+    print a3
+    print a3.x, a2.x, a1.x
+
+    print
+    print 'Testing Class B'
+    b1 = B(6)
+    b2 = B(5)
+    print b1
+    print b2
+    print b2.x, b1.x
+
+    print
+    print 'Testing Class C'
+    c1 = C()
+    c2 = C()
+    print c1
+    print c2
+    print c2.x, c1.x