src/python/dm/common/api/dmApi.py

+#!/usr/bin/env python
+
+import json
+from functools import wraps
+from decorator import decorator
+from dm.common.exceptions.dmException import DmException
+from dm.common.utility.loggingManager import LoggingManager
+
+class DmApi(object):
+    """ Base dm api class. """
+    def __init__(self, username = None, password = None):
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    @classmethod
+    def toDmObjectList(cls, dictList, dmObjectClass):
+        dmObjectList = []
+        for dict in dictList:
+            dmObjectList.append(dmObjectClass(dict))
+        return dmObjectList 
+
+    # Exception decorator for all api calls
+    @classmethod
+    def execute(cls, func):
+        def decorate(*args, **kwargs):
+            try:
+                response = func(*args, **kwargs)
+                return response
+            except DmException, ex:
+                raise
+            except Exception, ex:
+                cls.getLogger().exception('%s' % ex)
+                raise DmException(exception=ex)
+        return decorate
+
+    # Exception decorator for api calls
+    # Use two decorators for the sake of documentation
+    @classmethod
+    def execute2(cls, *dargs, **dkwargs):
+        def internalCall(func):
+            @wraps(func)
+            def wrappedCall(func, *args, **kwargs):
+                try:
+                    response = func(*args, **kwargs)
+                    return response
+                except DmException, ex:
+                    raise
+                except Exception, ex:
+                    cls.getLogger().exception('%s' % ex)
+                    raise DmException(exception=ex)
+            return decorator(wrappedCall, func)
+        if len(dargs) == 1 and callable(dargs[0]):
+            return internalCall(dargs[0])
+        else:
+            return internalCall
+

src/python/dm/common/api/dmRestApi.py

+#!/usr/bin/env python
+
+import socket
+import json
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.client.sessionManager import SessionManager
+from dm.common.utility.configurationManager import ConfigurationManager 
+from dm.common.exceptions.authorizationError import AuthorizationError
+from dm.common.api.dmApi import DmApi
+
+class DmRestApi(DmApi):
+    """ Base dm REST api class. """
+
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        DmApi.__init__(self)
+        self.configurationManager = ConfigurationManager.getInstance()
+        if protocol == None:
+            protocol = self.configurationManager.getServiceProtocol()
+        self.username = username
+        self.password = password
+        self.host = host
+        self.port = port
+        self.protocol = protocol
+        self.sessionManager = None
+
+    @classmethod
+    def toJson(cls, o):
+        return json.dumps(o)
+
+    @classmethod
+    def fromJson(cls, s):
+        return json.loads(s)
+
+    def __getWebServiceUrl(self, url):
+        if url.find('://') < 0:
+            return '%s://%s:%s' % (self.protocol, socket.gethostbyname(self.host), self.port)
+
+        # Break the URL down into component pieces
+        from urlparse import urlparse
+        o = urlparse(url)
+        wsUrl = '%s://%s' % (o[0], o[1])
+        return wsUrl
+
+    def getContextRoot(self):
+        return self.configurationManager.getContextRoot()
+
+    def setUsername(self, username):
+        self.username = username
+
+    def getUsername(self):
+        return self.username
+
+    def setPassword(self, password):
+        self.password = password
+
+    def getPassword(self):
+        return self.password
+
+    def setHost(self, host):
+        self.host = host
+
+    def getHost(self):
+        return self.host
+
+    def setPort(self, port):
+        self.port = port
+
+    def getPort(self):
+        return self.port
+
+    def setProtocol(self, protocol):
+        self.protocol = protocol
+
+    def getProtocol(self):
+        return self.protocol
+
+    def getSessionManager(self):
+        if not self.sessionManager:
+            self.sessionManager = SessionManager()
+        return self.sessionManager
+
+    def getConfigManager(self):
+        return self.configurationManager
+
+    def sendSessionRequest(self, url, method, contentType='html', data={}):
+        """ Send authorized session request. """
+        sm = self.getSessionManager()
+        if not sm.hasSession():
+            #if self.username == None:
+            #    raise AuthorizationError('Username not supplied.')
+            #if self.password == None:
+            #    raise AuthorizationError('Password not supplied.')
+            wsUrl = self.__getWebServiceUrl(url)
+            # establishSession() sets the 'wsUrl' so the explicit call
+            # to setHost() is not required
+            sm.establishSession(wsUrl, self.username, self.password)
+        (response, responseData) = sm.sendSessionRequest(url, method, contentType, data)
+        return json.loads(responseData)
+
+    def sendRequest(self, url, method, contentType='html', data={}):
+        """ Send non-authorized request. """
+        sm = self.getSessionManager()
+        # Because there's no call to establishSession(), explicitly call
+        # setHost()
+        sm.setHost(self.__getWebServiceUrl(url))
+        (response, responseData) = self.getSessionManager().sendRequest(url, method, contentType, data)
+        return json.loads(responseData)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = DmRestApi('sveseli', 'sveseli')
+    #api.sendRequest('https://zagreb.svdev.net:10232/dm/directory/list', 'GET', data='')
+    import urllib
+    from dm.common.utility.configurationManager import ConfigurationManager
+    cm = ConfigurationManager.getInstance()
+    cm.setSessionCacheFile('/tmp/session')
+    #print 'Non-session request'
+    #print api.sendRequest('https://zagreb.svdev.net:10232/dm/directory/list?path=/tmp', 'GET')
+    print 'Session request'
+    data = { 'path' : '/tmp/xyz' }
+    #print api.sendSessionRequest('https://zagreb.svdev.net:10232/dm/file/write?path=/tmp/xyz&content=xyz', 'POST', contentType='application/x-www-form-urlencoded', data=urllib.urlencode(data))
+    #print api.sendSessionRequest('https://zagreb.svdev.net:10232/dm/file/write', 'POST', data=data)
+    postdata='path=/tmp/xyz'
+    postdata+='&content=%s' % urllib.quote_plus('Hey there')
+    print api.sendSessionRequest('https://zagreb.svdev.net:10232/dm/file/write', 'POST', contentType='application/x-www-form-urlencoded', data=postdata)
+
+
+

src/python/dm/common/cli/dmCli.py

+#!/usr/bin/env python
+
+import sys
+import os
+import os.path
+import stat
+import urlparse
+from optparse import OptionGroup
+
+import dm
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.utility.osUtility import OsUtility
+from dm.common.objects.dmObject import DmObject
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.internalError import InternalError
+from dm.common.cli.dmOptionParser import DmOptionParser
+from dm.common.constants import dmStatus
+
+class DmCli(object):
+    """ Base dm command line interface class. """
+    DEFAULT_SESSION_CACHE_FILE = OsUtility.getUserHomeDir() + '/.dm/.session.cache'
+    ANY_NUMBER_OF_POSITIONAL_ARGS = 10000000
+
+    def __init__(self, validArgCount=0):
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        # Do not log into a file for CLIs
+        LoggingManager.getInstance().setFileLogLevel('CRITICAL') 
+        LoggingManager.getInstance().setConsoleLogLevel('CRITICAL') 
+        self.parser = DmOptionParser()
+        self.options = {}
+        self.args = []
+        self.validArgCount = validArgCount
+        self.optionGroupDict = {}
+
+        commonGroup = 'Common Options'
+        self.addOptionGroup(commonGroup, None)
+        self.addOptionToGroup(commonGroup, '-h', '--help', action='help', help='Show this help message and exit.')
+        self.addOptionToGroup(commonGroup, '-?', '',       action='help', help='Show this help message and exit.')
+        self.addOptionToGroup(commonGroup, '-v', '--version', action='store_true', dest='version', default=False, help='Print version and exit.')
+        self.addOptionToGroup(commonGroup, '-d', '--debug', dest='consoleLogLevel', help='Set debug level (valid values: CRITICAL, ERROR, WARNING, INFO, DEBUG). Console log level can also be set via DM_CONSOLE_LOG_LEVEL environment variable,')
+        self.addOptionToGroup(commonGroup, '', '--display-format', dest='displayFormat', default=DmObject.TEXT_DISPLAY_FORMAT, help='Display format for output objects. Possible options are: %s, %s, and %s (default: %s).' % (DmObject.TEXT_DISPLAY_FORMAT, DmObject.DICT_DISPLAY_FORMAT, DmObject.JSON_DISPLAY_FORMAT, DmObject.TEXT_DISPLAY_FORMAT)) 
+        self.addOptionToGroup(commonGroup, '', '--display-keys', dest='displayKeys', default=DmObject.DEFAULT_KEYS, help='List of output object keys to display. Possible options are: %s, %s, and string containing comma-separated keys (default: %s, represents class default keys).' % (DmObject.DEFAULT_KEYS, DmObject.ALL_KEYS, DmObject.DEFAULT_KEYS)) 
+
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getServicePort()
+
+    def getDefaultServiceProtocol(self):
+        return ConfigurationManager.getInstance().getServiceProtocol()
+
+    def getUsername(self):
+        return None
+
+    def getPassword(self):
+        return None
+
+    def getDisplayFormat(self):
+        return self.options.displayFormat
+
+    def getDisplayKeys(self):
+        return self.options.displayKeys
+
+    def getLogger(self):
+        return self.logger
+
+    def getParser(self):
+        return self.parser
+
+    def addOption(self, *args, **kwargs):
+        self.parser.add_option(*args, **kwargs)
+
+    def addOptionToGroup(self, groupName, *args, **kwargs):
+        """ Add group option. Group must be created using addOptionGroup(). """
+        group = self.optionGroupDict.get(groupName)
+        group.add_option(*args, **kwargs)
+
+    def addOptionGroup(self, groupName, desc=None, prepend=False):
+        group = OptionGroup(self.parser, groupName, desc)
+	if not prepend:
+            self.parser.add_option_group(group)
+	else:
+            self.parser.option_groups.insert(0,group)
+        self.optionGroupDict[groupName] = group
+
+    def processArgs(self):
+        pass
+
+    def parseArgs(self, usage=None):
+        if usage:
+            self.parser.usage = usage
+        try:
+            (self.options, self.args) = self.parser.parse_args()
+            self.processArgs()
+        except SystemExit, rc:
+            sys.stdout.flush()
+            sys.stderr.flush()
+            sys.exit(int(str(rc)))
+
+        if self.validArgCount < len(self.args):
+            # Positional args are not enabled and we have some
+            msg = 'Invalid positional argument(s):'
+            for arg in self.args[self.validArgCount:]:
+                msg += ' ' + arg
+            msg += ' (This command allows %s positional arguments.)' % self.validArgCount
+            raise InvalidArgument(msg)
+
+        optDict = self.options.__dict__
+        if optDict.get('version'):
+            print 'DM Software Version: %s' % (dm.__version__)
+            sys.exit(0)
+
+        # Logging level. First try from command line, then from env variable.
+        consoleLogLevel = optDict.get('consoleLogLevel', None)
+        if consoleLogLevel:
+            LoggingManager.getInstance().setConsoleLogLevel(consoleLogLevel)
+        else:
+            consoleLogLevel = ConfigurationManager.getInstance().getConsoleLogLevelFromEnvVar()
+            if consoleLogLevel:
+                LoggingManager.getInstance().setConsoleLogLevel(consoleLogLevel)
+
+
+        # Check session cache.
+        configManager = ConfigurationManager.getInstance()
+        try:
+            self.checkSessionCache()
+        except Exception, ex:
+            self.logger.warn('Disabling session cache: %s' % ex)
+            configManager.setSessionCacheFile(None)
+
+        return (self.options, self.args)
+
+    def checkSessionCache(self):
+        configManager = ConfigurationManager.getInstance()
+        sessionCacheFile = configManager.getSessionCacheFile()
+        if sessionCacheFile is None:
+            sessionCacheFile = DmCli.DEFAULT_SESSION_CACHE_FILE 
+        sessionCacheFile = sessionCacheFile.strip()
+        if len(sessionCacheFile):
+            sessionCacheDir = os.path.dirname(sessionCacheFile)
+            OsUtility.createDir(sessionCacheDir, stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR)
+            configManager.setSessionCacheFile(sessionCacheFile)
+            configManager.setRequireSessionCredentials(True)
+
+    def usage(self, s=None):
+        """ Print help provided by optparse. """
+        if s:
+            print >>sys.stderr, 'Error: ', s, '\n'
+        self.parser.print_help()
+        sys.exit(dmStatus.DM_ERROR)
+
+    def getOptions(self):
+        return self.options
+
+    def getNArgs(self):
+        """ Returns the number of command line arguments. """
+        return len(self.args)
+
+    def getArgs(self):
+        """ Returns the command line argument list. """
+        return self.args
+
+    def splitArgsIntoDict(self, keyValueDelimiter=':'):
+        """ Returns the command line argument list as dictionary of key/value
+        pairs. Each argument is split using specified delimiter. """
+        argDict = {}
+        for a in self.args:
+            sList = a.split(keyValueDelimiter)
+            key = sList[0]
+            value = ''
+            if len(sList) > 1:
+                value = keyValueDelimiter.join(sList[1:])
+            argDict[key] = value
+        return argDict
+
+    def getArg(self, i):
+        """ Returns the i-th command line argument. """
+        return self.args[i]
+
+    def getServiceHost(self):
+        #return self.serviceHost
+        return ConfigurationManager.getInstance().getServiceHost()
+
+    def getServicePort(self):
+        #return self.servicePort
+        return ConfigurationManager.getInstance().getServicePort()
+
+    def getServiceProtocol(self):
+        #return self.serviceProtocol
+        return ConfigurationManager.getInstance().getServiceProtocol()
+
+    def getSslCaCertFile(self):
+        return self.sslCaCertFile
+
+    def getSslCertFile(self):
+        return self.sslCertFile
+
+    def getSslKeyFile(self):
+        return self.sslKeyFile
+
+    def displayDmObject(self, dmObject):
+        if isinstance(dmObject, dmObject):
+            return '%s' % dmObject.getJsonRep()
+        else:
+            return '%s' % dmObject
+
+    def runCommand(self):
+        """ This method must be implemented by the derived class. """
+        raise InternalError('Method runCommand() must be overriden in the derived class.')
+
+    def run(self):
+        """ This method invokes runCommand() and handles any exceptions. """
+        try:
+            self.runCommand()
+        except DmException, ex:
+            self.logger.exception('%s' % ex)
+            print '%s' % ex.getErrorMessage()
+            raise SystemExit(ex.getErrorCode())
+        except SystemExit, ex:
+            raise
+        except Exception, ex:
+            self.logger.exception('%s' % ex)
+            print >>sys.stderr, '%s' % ex
+            raise SystemExit(dmStatus.DM_ERROR)
+
+    def getId(self):
+        id = self.options.id
+        if id == None:
+            raise InvalidRequest('Missing id.')
+        return id
+
+    def displayDmObject(self, dmObject):
+        optDict = self._options.__dict__
+        if isinstance(dmObject, DmObject):
+            if optDict.get('dict'):
+                 return '%s' % dmObject
+            else:
+                 return dmObject.display()
+        else:
+            return '%s' % dmObject
+
+    def getDataDirectory(self):
+        dataDirectory = self.options.dataDirectory
+        replacementMap = os.environ.get('DM_DATA_DIRECTORY_MAP', '')
+        (scheme, host, port, dirPath) = self.parseUrl(dataDirectory)
+        if dirPath and replacementMap:
+            # Map entries are expected to be in the form
+            # <original>:<replacement> <original>:<replacement>;...
+            for entry in replacementMap.split(' '):
+                original = entry.split(':')[0]
+                replacement = entry.split(':')[1]
+                dirPath = dirPath.replace(original,replacement)
+        return self.assembleUrl(scheme, host, port, dirPath)
+
+    @classmethod
+    def parseUrl(cls, url, defaultHost=None, defaultPort=None):
+        host = defaultHost
+        port = defaultPort
+        parseResult = urlparse.urlparse(url)
+        scheme = parseResult.scheme
+        netlocTokens = parseResult.netloc
+        if netlocTokens:
+            netlocTokens = netlocTokens.split(':')
+            host = netlocTokens[0]
+            if len(netlocTokens) > 1:
+                port = int(netlocTokens[1])
+        dirPath = parseResult.path
+        dirPath = os.path.normpath(dirPath)
+        return (scheme, host, port, dirPath)
+
+    @classmethod
+    def assembleUrl(cls, scheme, host, port, dirPath):
+        url = ''
+        if scheme:
+            url += '%s://' % scheme
+        if host:
+            url += '%s' % (host)
+            if port:
+                url += ':%s' % (port)
+        if dirPath:
+            if len(url) and not dirPath.startswith('/'):
+                url += '/%s' % (os.path.normpath(dirPath))
+            else:
+                url += '%s' % (os.path.normpath(dirPath))
+        return url
+
+#######################################################################
+# Testing
+
+if __name__ == '__main__':
+    cli = DmCli(3)
+    cli.addOption("-f", "--file", dest="filename", help="write report to FILE", metavar="FILE")
+    cli.addOption("-q", "--quiet", action="store_false", dest="verbose", default=True, help="don't print log messages to stdout")
+    (options, args) = cli.parseArgs()
+    print 'After parse:'
+    print 'OPTIONS: ', options
+    print 'ARGS: ', args
+    print 'From CLI'
+    print 'OPTIONS: ', cli.getOptions()
+    print 'ARGS: ', cli.getArgs()
+    print
+    print 'FILENAME'
+    print 'options.filename', options.filename
+    print 'cli.getOptions().filename', cli.getOptions().filename
+    o = cli.getOptions()
+    print 'o.filename', o.filename
+    print 'cli.getArgs()', cli.getArgs()
+    print 'len(cli.getArgs())', len(cli.getArgs())
+
+    for a in cli.getArgs():
+        print 'arg', a
+
+    first_arg = cli.getArg(0)
+    print 'first_arg', first_arg
+
+    second_arg = cli.getArg(1)
+    print 'second_arg', second_arg
+
+    try:
+        third_arg = cli.getArg(2)
+        print 'third_arg', third_arg
+    except:
+        print 'no third arg'

src/python/dm/common/cli/dmDbCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmCli import DmCli
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DmDbCli(DmCli):
+    """ Base dm DB cli class. """
+
+    def __init__(self, validArgCount=0):
+        DmCli.__init__(self, validArgCount)
+
+

src/python/dm/common/cli/dmOptionParser.py

+#!/usr/bin/env python
+
+import sys
+from optparse import OptionParser
+from cStringIO import StringIO
+
+class DmOptionParser(OptionParser):
+    def __init__(self):
+        OptionParser.__init__(self, add_help_option=False)
+
+    def replaceKeys(self, output):
+        replacementMap = {
+            'usage:' : 'Usage:',
+            'options:' : 'Options:',
+        }
+        result = output
+        for (key, value) in replacementMap.items():
+            result = result.replace(key, value)
+        return result
+
+    def printUsage(self, file=None):
+        self.print_usage(file)
+
+    # Replaces base class method
+    def print_usage(self, file=None):
+        sysStdout = sys.stdout
+        cliStdout = StringIO()
+        sys.stdout = cliStdout
+        OptionParser.print_usage(self, file)
+        sys.stdout = sysStdout
+        print self.replaceKeys(cliStdout.getvalue())
+
+    def printHelp(self, file=None):
+        self.printHelp(file)
+
+    # Replaces base class method
+    def print_help(self, file=None):
+        sysStdout = sys.stdout
+        cliStdout = StringIO()
+        sys.stdout = cliStdout
+        OptionParser.print_help(self, file)
+        sys.stdout = sysStdout
+        print self.replaceKeys(cliStdout.getvalue())
+
+#######################################################################
+# Testing
+
+if __name__ == '__main__':
+    p = DmOptionParser()
+    p.add_option('-f', '--file', dest='filename',
+        help='write report to FILE', metavar='FILE')
+    p.add_option('-q', '--quiet',
+        action='store_false', dest='verbose', default=True,
+        help='do not print log messages to standard output')
+    p.parse_args()
+    p.print_usage()
+    p.print_help()

src/python/dm/common/cli/dmRestCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmCli import DmCli
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DmRestCli(DmCli):
+    """ Base dm REST cli class. """
+
+    def __init__(self, validArgCount=0):
+        DmCli.__init__(self, validArgCount)
+        self.serviceHost = None
+        self.servicePort = None
+
+        serviceGroup = 'Service Options'
+        self.addOptionGroup(serviceGroup, None)
+        # These will be set via env variables
+        self.addOptionToGroup(serviceGroup, '', '--service-host', dest='serviceHost', default=self.getDefaultServiceHost(), help='Service host (default: %s, can be set via DM_SERVICE_HOST environment variable).' % self.getDefaultServiceHost())
+        self.addOptionToGroup(serviceGroup, '', '--service-port', dest='servicePort', default=self.getDefaultServicePort(), help='Service port (default: %s, can be set via DM_SERVICE_PORT environment variable).' % self.getDefaultServicePort())
+        self.addOptionToGroup(serviceGroup, '', '--service-protocol', dest='serviceProtocol', default=self.getDefaultServiceProtocol(), help='Service protocol (default: %s, can be set via DM_SERVICE_PROTOCOL environment variable).' % self.getDefaultServiceProtocol())
+
+        # SSL options, disabled for now.
+        #self.addOptionToGroup(commonGroup, '', '--ssl-key', dest='sslKeyFile', help='SSL key file (needed if service requires peer verification, can be set via DM_SSL_KEY_FILE environment variable).')
+        #self.addOptionToGroup(commonGroup, '', '--ssl-cert', dest='sslCertFile', help='SSL certificate file (needed if service requires peer verification, can be set via DM_SSL_CERT_FILE environment variable).')
+        #self.addOptionToGroup(commonGroup, '', '--ssl-ca-cert', dest='sslCaCertFile', help='SSL CA certificate file (needed if client requires peer verification, can be set via DM_SSL_CA_CERT_FILE environment variable).')
+        
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getServicePort()
+
+    def getDefaultServiceProtocol(self):
+        return ConfigurationManager.getInstance().getWebServiceProtocol()
+
+    def parseArgs(self, usage=None):
+        DmCli.parseArgs(self, usage)
+        configManager = ConfigurationManager.getInstance()
+        self.serviceHost = self.options.serviceHost
+        configManager.setServiceHost(self.serviceHost)
+        self.servicePort = self.options.servicePort
+        configManager.setServicePort(self.servicePort)
+        self.serviceProtocol = self.options.serviceProtocol
+        configManager.setServiceProtocol(self.serviceProtocol)
+
+        # SSL options, comment out for now
+        #self.sslCaCertFile = self.options.sslCaCertFile
+        #if self.sslCaCertFile:
+        #    configManager.setSslCaCertFile(self.sslCaCertFile)
+        #self.sslCertFile = self.options.sslCertFile
+        #if self.sslCertFile:
+        #    configManager.setSslCertFile(self.sslCertFile)
+        #self.sslKeyFile = self.options.sslKeyFile
+        #if self.sslKeyFile:
+        #    configManager.setSslKeyFile(self._sslKeyFile)
+

src/python/dm/common/cli/dmRestSessionCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmRestCli import DmRestCli
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DmRestSessionCli(DmRestCli):
+    """ Base dm session cli class. """
+
+    def __init__(self, validArgCount=0):
+        DmRestCli.__init__(self, validArgCount)
+        self.username = None
+        self.password = None
+
+        loginGroup = 'Login Options'
+        self.addOptionGroup(loginGroup, None)
+        self.addOptionToGroup(loginGroup, '', '--login-username', dest='loginUsername', help='Login username.')
+        self.addOptionToGroup(loginGroup, '', '--login-password', dest='loginPassword', help='Login password.')
+        self.addOptionToGroup(loginGroup, '', '--login-file', dest='loginFile', help='Login file, contains "<login username>|<login password>" pair. It can be specified using DM_LOGIN_FILE environment variable. This option cannot be used with --login-username and --login-password options.')
+
+    def parseArgs(self, usage=None):
+        DmRestCli.parseArgs(self, usage)
+        self.loginUsername = self.options.loginUsername
+        self.loginPassword = self.options.loginPassword
+        self.loginFile = self.options.loginFile
+        self.parseLoginFile()
+        return (self.options, self.args)
+
+    def getLoginUsername(self):
+        return self.loginUsername
+
+    def getLoginPassword(self):
+        return self.loginPassword
+
+    def getLoginFile(self):
+        if not self.loginFile:
+            configManager = ConfigurationManager.getInstance()
+            self.loginFile = configManager.getLoginFile()
+        return self.loginFile
+
+    def parseLoginFile(self):
+        if self.getLoginFile() and not self.getLoginUsername() and not self.getLoginPassword():
+            try:
+                # Assume form <username>|<password>
+                tokenList = open(self.loginFile).readline().split('|')
+                if len(tokenList) == 2:
+                    self.loginUsername = tokenList[0].strip()
+                    self.loginPassword = tokenList[1].strip()
+            except:
+                # Ignore invalid login file
+                pass
+
+    def hasCredentials(self):
+        return (self.getLoginUsername() != None and self.getLoginPassword() != None)
+
+#######################################################################
+# Testing
+
+if __name__ == '__main__':
+    pass

src/python/dm/common/client/dmExceptionMapper.py

+#!/usr/bin/env python
+
+from dm.common.constants import dmStatus
+from dm.common.exceptions import dmExceptionMap
+from dm.common.exceptions.dmException import DmException
+
+class DmExceptionMapper:
+
+    @classmethod
+    def checkStatus(cls, httpHeaders):
+        """ Map dm status code into appropriate exception. """
+        code = httpHeaders.get('Dm-Status-Code', None)
+        msg = httpHeaders.get('Dm-Status-Message', 'Internal Error')
+        if code is None or code == str(dmStatus.DM_OK):
+            return
+        elif dmExceptionMap.DM_EXCEPTION_MAP.has_key(int(code)):
+            # Exception string is value of the form 'x.y.z'
+            # where 'x.y' is dm module, and 'z' class in that module
+            exStr = dmExceptionMap.DM_EXCEPTION_MAP.get(int(code))
+            exClass = exStr.split('.')[-1] # 'z' in 'x.y.z'
+            exModule = '.'.join(exStr.split('.')[:-1]) # 'x.y' in 'x.y.z'
+            exec 'from dm.common.exceptions.%s import %s' % (exModule, exClass)
+            exec 'ex = %s(msg)' % (exClass)
+            raise ex
+        else:
+            raise DmException(msg)
+
+# Testing.
+if __name__ == '__main__':
+    pass

src/python/dm/common/client/dmHttpsConnection.py

+#!/usr/bin/env python
+
+import socket
+import httplib
+import ssl
+
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DmHttpsConnection(httplib.HTTPSConnection):
+
+    def __init__(self, hostPort, timeout):
+        cm = ConfigurationManager.getInstance()
+        args = hostPort.split(':')
+        host = args[0]
+        if len(args) > 1:
+            port = int(args[1])
+        else:
+            port = cm.getServicePort()
+        keyFile = cm.getSslKeyFile()
+        certFile = cm.getSslCertFile()
+        caCertFile = cm.getSslCaCertFile()
+        certChain = None
+        strict = True
+        httplib.HTTPSConnection.__init__(self, host, port, keyFile, certFile, strict, timeout)
+        context = self.getContext(keyFile, certFile, caCertFile, certChain)
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.sock = context.wrap_socket(sock)
+        self.connect()
+
+    def connect(self):
+        self.sock.connect((self.host,self.port))
+
+    def getContext(self, keyFile, certFile, caCertFile=None, certChain=None):
+        """Return SSL Context from self attributes."""
+        #context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+        if caCertFile is not None:
+            context.verify_mode = ssl.CERT_REQUIRED
+            context.load_verify_locations(caCertFile)
+        else:
+            context.verify_mode = ssl.CERT_NONE
+        if certFile is not None and keyFile is not None:
+            context.load_cert_chain(certFile, keyFile)
+        if certChain:
+            context.load_verify_locations(certChain)
+        return context
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass
+

src/python/dm/common/client/dmHttpsHandler.py

+#!/usr/bin/env python
+
+import urllib2
+import ssl
+from dm.common.client.dmHttpsConnection import DmHttpsConnection
+
+class DmHttpsHandler(urllib2.HTTPSHandler):
+
+    def https_open(self, req):
+        return self.do_open(DmHttpsConnection,req)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    from dm.common.utility.configurationManager import ConfigurationManager 
+    cm = ConfigurationManager.getInstance()
+    cm.setSslCaCertFile("/home/sveseli/Work/DM/etc/ssl/cacert.pem")
+
+    print "Installing opener"
+    opener = urllib2.build_opener(DmHttpsHandler)
+    urllib2.install_opener(opener)
+
+    url = "https://zagreb.svdev.net:10232/dm/directory/list?path=/tmp"
+    print "Opening URL: ", url
+    #context = ssl.create_default_context(cafile="/home/sveseli/Work/DM/etc/ssl/cacert.pem")
+    #ssl._create_default_https_context = ssl._create_unverified_context
+    #f = urllib2.urlopen(url, context=context)
+    f = urllib2.urlopen(url)
+    print f.code
+    print f.read()
+

src/python/dm/common/client/sessionManager.py

+#!/usr/bin/env python
+
+import urllib
+import urllib2
+import urlparse
+import time
+import getpass
+import os
+import stat
+import ssl
+import types
+
+from dm.common.constants import dmServiceConstants
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.exceptions.authorizationError import AuthorizationError
+from dm.common.exceptions.invalidSession import InvalidSession
+from dm.common.exceptions.urlError import UrlError
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.utility.osUtility import OsUtility
+from dm.common.client.dmExceptionMapper import DmExceptionMapper
+from dm.common.client.dmHttpsHandler import DmHttpsHandler
+
+class SessionManager:
+    """ Class for session management. """
+
+    def __init__(self):
+        self.sessionCookie = None
+        self.host = None
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        self.username = ''
+        self.password = ''
+        self.urlOpener = None
+        cm = ConfigurationManager.getInstance()
+        self.sessionCacheFile = cm.getSessionCacheFile()
+        self.requireSessionCredentials = cm.getRequireSessionCredentials()
+
+    def setHost(self, host):
+        self.host = host
+
+    def hasSession(self):
+        """ Return true if we have session established. """
+        if self.sessionCookie is not None:
+            return True
+        return False
+
+    def establishSession(self, url, username, password, selector='/dm/login'):
+        self.host = url
+        self.username = username
+        self.password = password
+        self.sessionCookie = self.loadSession()
+        if self.sessionCookie is not None:
+            return
+
+        # Could not load session.
+        try:
+            # User will be asked for username/password if they are not
+            # provided.
+            self.username = self.getUsername(username)
+            self.password = self.getPassword(password)
+            data = { 'username' : self.username, 'password' : self.password }
+            self.logger.debug('Establishing session for user %s @ %s)' % (username, url))
+            (response,responseData) = self.sendRequest(url='%s%s' % (url, selector), method='POST', contentType='application/x-www-form-urlencoded', data=urllib.urlencode(data))
+        except urllib2.URLError, ex:
+            self.logger.exception('%s' % ex)
+            raise UrlError(exception=ex)
+
+        #self.logger.debug('Got headers: %s' % response.headers)
+        # This will save session cookie.
+        self.sessionCookie = self.checkResponseHeadersForErrorsAndSaveSession(response.headers)
+        self.logger.debug('User %s session cookie: %s' % (self.username, self.sessionCookie))
+
+    def getUsername(self, username):
+        if username is None and self.requireSessionCredentials:
+            return self.askForUsername()
+        return username
+
+    def askForUsername(self):
+        defaultUsername = getpass.getuser()
+        username = raw_input('Username [%s]: ' % defaultUsername)
+        username = username.strip()
+        if not len(username):
+            username = defaultUsername
+        return username
+
+    def getPassword(self, password):
+        if password is None and self.requireSessionCredentials:
+            return self.askForPassword()
+        return password
+
+    def askForPassword(self):
+        password = getpass.getpass()
+        password = password.strip()
+        if not len(password):
+            raise AuthorizationError('Empty password provided.')
+        return password
+
+    def clearSessionFile(self):
+        if self.sessionCacheFile is None:
+            return
+        try:
+            self.logger.debug('Clearing session cache: %s' % (self.sessionCacheFile))
+            OsUtility.removeFile(self.sessionCacheFile)
+        except Exception, ex:
+            # ignore errors.
+            self.logger.warn('Could not clear session cache: %s' % (ex))
+            pass
+
+    def saveSession(self, sessionCookie):
+        if self.sessionCacheFile is None:
+            return
+        if sessionCookie is None:
+            return
+        try:
+            f = open(self.sessionCacheFile, 'w')
+            f.write('%s' % sessionCookie)
+            f.close()
+            os.chmod(self.sessionCacheFile, stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR)
+        except Exception, ex:
+            self.logger.warn('Could not save session: %s' % (ex))
+
+    def loadSession(self):
+        if self.sessionCacheFile is None:
+            return None
+        try:
+            f = open(self.sessionCacheFile, 'r')
+            session = f.read()
+            expires = session.split(';')[1].split('=')[-1]
+            t = time.mktime(time.strptime(expires, '%a, %d %b %Y %H:%M:%S %Z'))
+            now = time.time()
+            if t < now:
+                return None
+            else:
+                self.logger.debug('Loaded session from %s: %s' % (self.sessionCacheFile, session))
+                return session
+        except Exception, ex:
+            self.logger.warn('Could not load session: %s' % (ex))
+        return None
+
+    def getUrlOpener(self, protocol):
+        if not self.urlOpener:
+            if protocol == dmServiceConstants.DM_SERVICE_PROTOCOL_HTTPS:
+                # HTTPS, use custom https handler, which 
+                # should work even if any of cert/key/cacert files is None
+                cm = ConfigurationManager.getInstance()
+                keyFile = cm.getSslKeyFile()
+                certFile = cm.getSslCertFile()
+                self.urlOpener = urllib2.build_opener(DmHttpsHandler)
+                #self.logger.debug('Using Dm HTTPS Handler')
+            else:
+                # HTTP, use standard http handler
+                self.urlOpener = urllib2.build_opener(urllib2.HTTPHandler)
+        # Install opener before returning it.
+        urllib2.install_opener(self.urlOpener)
+        return self.urlOpener
+
+    def sendRequest(self, url, method, contentType='html', data={}):
+        """ Send http request without cookies. """
+        if url.find('://') < 0:
+            url = '%s%s' % (self.host, url)
+        parsedUrl = urlparse.urlparse(url)
+        protocol = parsedUrl[0]
+        path = parsedUrl[2]
+        #self.logger.debug('Sending request: %s' % url)
+        encodedData = ''
+        if data is not None:
+            if type(data) == types.DictType and len(data):
+                encodedData=urllib.urlencode(data)
+                contentType='application/x-www-form-urlencoded'
+            elif type(data) == types.StringType:
+                encodedData = data
+        request = urllib2.Request(url, data=encodedData)
+        request.get_method = lambda: method
+        request.add_header('Content-Type', contentType)
+        request.add_header('Content-Length', str(len(data)))
+        if self.sessionCookie != None:
+            request.add_header('Cookie', self.sessionCookie)
+        try:
+            opener = self.getUrlOpener(protocol)
+            response = opener.open(request)
+        except urllib2.HTTPError, ex:
+            # If we see dm headers, dm exception will be thrown,
+            # otherwise we'll throw UrlError
+            self.checkResponseHeadersForErrors(ex.hdrs)
+            self.logger.exception('%s' % ex)
+            raise UrlError(exception=ex)
+        except urllib2.URLError, ex:
+            self.logger.exception('%s' % ex)
+            raise UrlError(exception=ex)
+
+        # Check headers for errors and update session cookie
+        sessionCookie = self.checkResponseHeadersForErrorsAndSaveSession(response.headers)
+        if sessionCookie != None:
+            self.sessionCookie = sessionCookie
+        responseData = response.read()
+        return (response, responseData)
+
+    def sendSessionRequest(self, url, method, contentType='html', data={}):
+        """ Send session request. """
+        if self.sessionCookie is None:
+            self.establishSession(self.host, self.username, self.password)
+        try:
+            return self.sendRequest(url, method, contentType, data)
+        except InvalidSession, ex:
+            self.clearSessionFile()
+            self.establishSession(self.host, self.username, self.password)
+            return self.sendRequest(url, method, contentType, data)
+
+    def checkResponseHeadersForErrorsAndSaveSession(self, responseHeaders):
+        try:
+            DmExceptionMapper.checkStatus(responseHeaders)
+            sessionCookie = responseHeaders.get('Set-Cookie')
+            self.saveSession(sessionCookie)
+            return sessionCookie
+        except (AuthorizationError, InvalidSession), ex:
+            self.clearSessionFile()
+            raise
+
+    def checkResponseHeadersForErrors(self, responseHeaders):
+        try:
+            DmExceptionMapper.checkStatus(responseHeaders)
+        except AuthorizationError, ex:
+            self.clearSessionFile()
+            raise
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    sm = SessionManager.createSession()

src/python/dm/common/constants/dmHttpHeaders.py

+#!/usr/bin/env python
+
+#######################################################################
+
+DM_SESSION_ROLE_HTTP_HEADER = 'Dm-Session-Role'
+DM_STATUS_CODE_HTTP_HEADER = 'Dm-Status-Code'
+DM_STATUS_MESSAGE_HTTP_HEADER = 'Dm-Status-Message'
+DM_EXCEPTION_TYPE_HTTP_HEADER = 'Dm-Exception-Type'
+

src/python/dm/common/constants/dmHttpStatus.py

+#!/usr/bin/env python
+
+
+DM_HTTP_OK = 200 
+DM_HTTP_UNAUTHORIZED = 401 
+DM_HTTP_INTERNAL_ERROR = 500 

src/python/dm/common/constants/dmProcessingMode.py

+#!/usr/bin/env python
+
+#######################################################################
+
+DM_PROCESSING_MODE_FILES = 'files'
+DM_PROCESSING_MODE_DIRECTORY = 'directory'
+
+DM_ALLOWED_PROCESSING_MODE_LIST = [DM_PROCESSING_MODE_FILES, DM_PROCESSING_MODE_DIRECTORY]

src/python/dm/common/constants/dmProcessingStatus.py

+#!/usr/bin/env python
+
+#######################################################################
+
+DM_PROCESSING_STATUS_ANY = 'any'
+DM_PROCESSING_STATUS_PENDING = 'pending'
+DM_PROCESSING_STATUS_RUNNING = 'running'
+DM_PROCESSING_STATUS_FINALIZING = 'finalizing'
+DM_PROCESSING_STATUS_DONE = 'done'
+DM_PROCESSING_STATUS_FAILED = 'failed'
+DM_PROCESSING_STATUS_SKIPPED = 'skipped'
+DM_PROCESSING_STATUS_ABORTING = 'aborting'
+DM_PROCESSING_STATUS_ABORTED = 'aborted'
+
+DM_ALLOWED_PROCESSING_STATUS_LIST = [
+    DM_PROCESSING_STATUS_ANY,
+    DM_PROCESSING_STATUS_PENDING,
+    DM_PROCESSING_STATUS_RUNNING,
+    DM_PROCESSING_STATUS_FINALIZING,
+    DM_PROCESSING_STATUS_DONE,
+    DM_PROCESSING_STATUS_FAILED,
+    DM_PROCESSING_STATUS_SKIPPED,
+    DM_PROCESSING_STATUS_ABORTING,
+    DM_PROCESSING_STATUS_ABORTED
+]
+
+DM_INACTIVE_PROCESSING_STATUS_LIST = [
+    DM_PROCESSING_STATUS_DONE,
+    DM_PROCESSING_STATUS_FAILED,
+    DM_PROCESSING_STATUS_SKIPPED,
+    DM_PROCESSING_STATUS_ABORTED
+]
+
+DM_ACTIVE_PROCESSING_STATUS_LIST = [
+    DM_PROCESSING_STATUS_PENDING,
+    DM_PROCESSING_STATUS_RUNNING,
+    DM_PROCESSING_STATUS_FINALIZING,
+    DM_PROCESSING_STATUS_ABORTING
+]

src/python/dm/common/constants/dmRole.py

+#!/usr/bin/env python
+
+#######################################################################
+
+# Sessions can have either admin or user role
+DM_ADMIN_SESSION_ROLE = 'Admin'
+DM_USER_SESSION_ROLE = 'User'
+
+# System and experiment roles are used for fine grained authorization
+#    - "DM ADMIN" system role corresponds to "DM ADMIN" session role
+#    - all other system/experiment roles correspond to "DM USER" session role
+DM_ADMIN_SYSTEM_ROLE = 'Administrator'
+DM_ADMIN_SYSTEM_ROLE_ID = 1
+
+DM_MANAGER_SYSTEM_ROLE = 'Manager'
+DM_MANAGER_SYSTEM_ROLE_ID = 2
+
+DM_PI_EXPERIMENT_ROLE = 'PI'
+DM_PI_EXPERIMENT_ROLE_ID = 1
+
+DM_USER_EXPERIMENT_ROLE = 'User'
+DM_USER_EXPERIMENT_ROLE_ID = 2
+
+

src/python/dm/common/constants/dmServiceConstants.py

+#!/usr/bin/env python
+
+#######################################################################
+
+DM_SERVICE_PROTOCOL_HTTP = 'http'
+DM_SERVICE_PROTOCOL_HTTPS = 'https'
+
+DM_DAQ_MAX_RUN_TIME_IN_HOURS_KEY = 'maxRunTimeInHours'
+DM_DAQ_DEST_DIRECTORY_KEY = 'destDirectory'
+DM_DAQ_UPLOAD_DATA_DIRECTORY_ON_EXIT_KEY = 'uploadDataDirectoryOnExit'
+DM_DAQ_UPLOAD_DEST_DIRECTORY_ON_EXIT_KEY = 'uploadDestDirectoryOnExit'