src/python/dm/common/objects/authorizationPrincipal.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class AuthorizationPrincipal(DmObject):
+
+    def __init__(self, dict={}, name=None, token=None, userInfo={}):
+        DmObject.__init__(self, dict)
+        if name is not None:
+            self['name'] = name
+        if token is not None:
+            self['token'] = token
+        if userInfo is not None and len(userInfo):
+            self['userInfo'] = userInfo
+
+    def getName(self):
+        return self.get('name')
+
+    def getAuthenticationToken(self):
+        return self.get('token')
+
+    def getToken(self):
+        return self.get('token')
+
+    def setRole(self, role):
+        self['role'] = role
+
+    def getRole(self):
+        return self.get('role')
+
+    def setUserInfo(self, userInfo):
+        self['userInfo'] = userInfo
+
+    def getUserInfo(self):
+        return self.get('userInfo')

src/python/dm/common/objects/daqInfo.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class DaqInfo(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/datasetMetadata.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class DatasetMetadata(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'datasetName', 'experimentName' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/dmObject.py

+#!/usr/bin/env python
+
+#
+# DM Object class.
+#
+
+#######################################################################
+
+import UserDict
+import UserList
+import types
+import json
+import datetime
+
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.utility import loggingManager
+
+class DmObject(UserDict.UserDict):
+    """ Base dm object class. """
+    ALL_KEYS = '__all__'
+    DEFAULT_KEYS = '__default__'
+
+    DICT_DISPLAY_FORMAT = 'dict'
+    TEXT_DISPLAY_FORMAT = 'text'
+    JSON_DISPLAY_FORMAT = 'json'
+
+    DEFAULT_KEY_LIST = [ 'id', 'name' ]
+
+    def __init__(self, dict={}):
+        if isinstance(dict, types.DictType): 
+            UserDict.UserDict.__init__(self, dict)
+        elif isinstance(dict, UserDict.UserDict):
+            UserDict.UserDict.__init__(self, dict.data)
+        else:
+            raise InvalidArgument('DmObject instance must be initialized using dictionary.')
+        self.logger = None
+
+    def getLogger(self):
+        if not self.logger:
+            self.logger = loggingManager.getLogger(self._class__.__name__)
+        return self.logger
+
+    def getRequiredKeyValue(self, key):
+        value = self.get(key)
+        if value is None:
+            errorMsg = 'Required dictionary key %s is missing.' % key
+            raise ObjectNotFound(errorMsg)
+
+    @classmethod
+    def getFromDict(cls, dict):
+        inst = cls()
+        for key in dict.keys():
+            inst[key] = dict[key]
+        return inst
+
+    def getRepKeyList(self, keyList):
+        if keyList is None:
+            return self.DEFAULT_KEY_LIST
+        elif type(keyList) == types.ListType:
+            if not len(keyList):
+                return self.DEFAULT_KEY_LIST
+            else:
+                return keyList
+        elif type(keyList) == types.StringType:
+            if keyList == DmObject.ALL_KEYS:
+                return self.data.keys()
+            elif keyList == DmObject.DEFAULT_KEYS:
+                return self.DEFAULT_KEY_LIST
+            else:
+                # Assume keys are separated by comma
+                return keyList.split(',')
+        else: 
+            # Unknown key list parameter.
+            raise InvalidArgument('Key list parameter must be one of: None, string "%s", string "%s", string containing comma-separated keys, or list of strings.' (DmObject.ALL_KEYS, DmObject.DEFAULT_KEYS))
+            
+
+    def getDictRep(self, keyList=None):
+        # Dict representation is dict
+        dictRep = {}
+        displayKeyList = self.getRepKeyList(keyList)
+        for key in displayKeyList:
+            value = self.get(key)
+            if isinstance(value, DmObject):
+                dictRep[key] = value.getDictRep('__all__')
+            elif type(value) == types.ListType:
+                itemList = []
+                for item in value:
+                    if isinstance(item, DmObject):
+                        itemList.append(item.getDictRep('__all__'))
+                    else:
+                        itemList.append(item)
+                dictRep[key] = itemList
+            else:
+                if value is not None:
+                    if isinstance(value, datetime.datetime):
+                        dictRep[key] = str(value)
+                    else:
+                        dictRep[key] = value
+        return dictRep
+
+    def getTextRep(self, keyList=None):
+        display = ''
+        displayKeyList = self.getRepKeyList(keyList)
+        for key in displayKeyList:
+            value = self.get(key)
+            if isinstance(value, DmObject):
+                display = display + '%s={ %s} ' % (key, value.getTextRep())
+            elif isinstance(value, types.ListType):
+                display = display + '%s=[ ' % key
+                for item in value:
+                    if isinstance(item, DmObject):
+                        display = display + '{ %s}, ' % (item)
+                    else:
+                        display = display + ' %s, ' % (item)
+                display = display + '] '
+            else:
+                if value is not None:
+                    display = display + '%s=%s ' % (key, value)
+        return display
+
+    def getJsonRep(self, keyList=None):
+        dictRep = self.getDictRep(keyList)
+        return json.dumps(dictRep)
+
+    def getFullJsonRep(self):
+        dictRep = self.getDictRep(DmObject.ALL_KEYS)
+        return json.dumps(dictRep)
+
+    @classmethod 
+    def fromJsonString(cls, jsonString):
+        return cls.getFromDict(json.loads(jsonString))
+
+    def getDisplayString(self, displayKeyList=[], displayFormat=TEXT_DISPLAY_FORMAT):
+        """ Get display string. """
+        if displayFormat == DmObject.DICT_DISPLAY_FORMAT:
+            return self.getDictRep(displayKeyList)
+        elif displayFormat == DmObject.TEXT_DISPLAY_FORMAT:
+            return self.getTextRep(displayKeyList)
+        elif displayFormat == DmObject.JSON_DISPLAY_FORMAT:
+            return self.getJsonRep(displayKeyList)
+        raise InvalidArgument('Unrecognized display displayFormat: %s.' (displayFormat))
+
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    x = {'name' : 'XYZ', 'one':1, 'two':2 }
+    o = DmObject(x)
+    print 'DM Object: ', o
+    print 'Type of DM object: ', type(o)
+    print 'JSON Rep:  ', o.getJsonRep()
+    print 'Type of JSON rep: ', type(o.getJsonRep())
+    j = '{"name" : "XYZ", "one":1, "two":2 }'
+    print 'String: ', j
+    x2 = DmObject.fromJsonString(j)
+    print 'DM Object 2: ', x2
+    print 'Type of DM object 2: ', type(x2)
+    print x2.getDisplayString(displayKeyList='__all__')
+
+

src/python/dm/common/objects/dmObjectManager.py

+#!/usr/bin/env python
+
+#
+# Base object manager class.
+#
+
+#######################################################################
+
+import threading
+from dm.common.utility.loggingManager import LoggingManager
+
+#######################################################################
+
+class DmObjectManager:
+    """ Base object manager class. """
+
+    def __init__(self):
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        self.lock = threading.RLock()
+
+    def getLogger(self):
+        return self.logger
+
+    def acquireLock(self):
+        self.lock.acquire()
+
+    def releaseLock(self):
+        self.lock.release()
+

src/python/dm/common/objects/dmSession.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class DmSession(DmObject):
+
+    def __init__(self, dict={}, sessionId=None, updateTime=None):
+        DmObject.__init__(self, dict)
+        if sessionId is not None:
+            self['sessionId'] = sessionId
+        if updateTime is not None:
+            self['updateTime '] = updateTime 
+
+    def setSessionId(self, sessionId):
+        self['sessionId'] = sessionId
+
+    def getSessionId(self):
+        return self.get('sessionId')
+
+    def setUpdateTime(self, updateTime):
+        self['updateTime'] = updateTime
+
+    def getUpdateTime(self):
+        return self.get('updateTime')

src/python/dm/common/objects/experiment.py

+#!/usr/bin/env python
+
+import time
+from dmObject import DmObject
+
+class Experiment(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'name', 'dataDirectory', 'startDate', 'endDate', 'daqStartTime', 'daqEndTime' ]
+
+    def __init__(self, dict={}):
+        DmObject.__init__(self, dict)
+
+####################################################################
+# Testing
+if __name__ == '__main__':
+    pass
+

src/python/dm/common/objects/experimentType.py

+#!/usr/bin/env python
+
+import time
+from dmObject import DmObject
+
+class ExperimentType(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'name', 'description`', 'rootDataPath' ]
+
+    def __init__(self, dict={}):
+        DmObject.__init__(self, dict)
+
+####################################################################
+# Testing
+if __name__ == '__main__':
+    pass
+

src/python/dm/common/objects/fileMetadata.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class FileMetadata(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'fileName', 'experimentName', 'locationList', 'filePath', 'fileSize', 'md5Sum', 'experimentFilePath' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/observedFile.py

+#!/usr/bin/env python
+
+import time
+import os
+from dmObject import DmObject
+import urlparse
+
+class ObservedFile(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'filePath', 'lastUpdatedTimestamp' ]
+
+    def __init__(self, dict={}, filePath=None, dataDirectory=None, experiment=None):
+        DmObject.__init__(self, dict)
+        if filePath:
+            self['filePath'] = filePath
+        if dataDirectory:
+            self['dataDirectory'] = dataDirectory
+            if filePath:
+                parseResult = urlparse.urlparse(dataDirectory)
+                self['experimentFilePath'] = os.path.relpath(filePath, parseResult.path)
+        if experiment:
+            self['experiment'] = experiment
+
+    def setLastUpdatedTimestampToNow(self):
+        self['lastUpdateTimestamp'] = time.time()
+
+    def getLastUpdatedTimestamp(self):
+        self.get('lastUpdateTimestamp')
+
+    def getFilePath(self):
+        return self.get('filePath')
+
+    def getDataDirectory(self):
+        return self.get('dataDirectory')
+
+    def getExperiment(self):
+        return self.get('experiment')
+
+####################################################################
+# Testing
+if __name__ == '__main__':
+    of = ObservedFile(filePath='tmp/xyz')
+    print of
+    of.setLastUpdatedTimestampToNow()
+    print of
+

src/python/dm/common/objects/roleType.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class RoleType(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'name', 'description' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/uploadInfo.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class UploadInfo(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'percentageComplete' ]
+
+    def __init__(self, dict={}):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/userExperimentRole.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class UserExperimentRole(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'user_id', 'experiment_id', 'role_type_id' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/userInfo.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class UserInfo(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'username', 'firstName', 'lastName', 'middleName', 'email', 'description' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/userSystemRole.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class UserSystemRole(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'user_id', 'role_type_id' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/plugins/fileProcessorInterface.py

+#!/usr/bin/env python
+
+import abc
+class FileProcessorInterface:
+
+    @abc.abstractmethod
+    def processFile(self, filePath, daqPath, experiment):
+        return NotImplemented
+        

src/python/dm/common/plugins/fileTransferPlugin.py

+#!/usr/bin/env python
+
+import os
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.dmSubprocess import DmSubprocess
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from fileProcessorInterface import FileProcessorInterface
+
+class FileTransferPlugin(FileProcessorInterface):
+
+    def __init__(self, command, src=None, dest=None):
+        self.src = src
+        self.dest = dest
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        if command is None or not len(command):
+            raise InvalidArgument('File transfer command must be non-empty string.')
+        self.command = command
+        self.subprocess = None
+
+    def processFile(self, filePath, daqPath, experiment):
+        storageHost = experiment.get('storageHost')
+        storageDirectory = experiment.get('storageDirectory')
+        dest = '%s:%s' % (storageHost, storageDirectory)
+        # Use relative path with respect to daq directory as a source
+        os.chdir(daqPath)
+        src = os.path.relpath(filePath, daqPath)
+        self.start(src, dest)
+
+    def getFullCommand(self, src, dest):
+        return '%s %s %s' % (self.command, src, dest)
+
+    def setSrc(self, src):
+        self.src = src
+
+    def setDest(self, dest):
+        self.dest = dest
+
+    def start(self, src=None, dest=None):
+        # Use preconfigured source if provided source is None
+        fileSrc = src
+        if src is None:
+            fileSrc = self.src
+        # Use provided destination only if preconfigured destination is None
+        # Plugins may have desired destination preconfigured for all files
+        fileDest = self.dest
+        if self.dest is None:
+            fileDest = dest
+
+        if not fileSrc or not fileDest:
+            raise InvalidRequest('Both source and destination must be non-empty strings.')
+        self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand(fileSrc, fileDest))
+        return self.subprocess.run()
+
+    def wait(self):
+        if self.subprocess:
+            return self.subprocess.wait()
+        return None
+
+    def poll(self):
+        if self.subprocess:
+            return self.subprocess.poll()
+        return None
+
+    def getStdOut(self):
+        if self.subprocess:
+            return self.subprocess.getStdOut()
+        return None
+
+
+    def getStdErr(self):
+        if self.subprocess:
+            return self.subprocess.getStdErr()
+        return None
+
+    def getExitStatus(self):
+        if self.subprocess:
+            return self.subprocess.getExitStatus()
+        return None
+
+    def reset(self):
+        self.subprocess = None
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    ft = FileTransfer('rsync -arv', '/tmp/xyz', '/tmp/xyz2')
+    ft.start()
+    print 'StdOut: ', ft.getStdOut()
+    print 'StdErr: ', ft.getStdErr()
+    print 'Exit Status: ', ft.getExitStatus()

src/python/dm/common/plugins/rsyncFileTransferPlugin.py

+#!/usr/bin/env python
+
+from fileTransferPlugin import FileTransferPlugin
+class RsyncFileTransferPlugin(FileTransferPlugin):
+
+    COMMAND = 'rsync -arvlPR'
+
+    def __init__(self, src=None, dest=None):
+        FileTransferPlugin.__init__(self, self.COMMAND, src, dest)
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    ft = RsyncFileTransferPlugin('/tmp/xyz', '/tmp/xyz2')
+    ft.start()
+    print 'StdOut: ', ft.getStdOut()
+    print 'StdErr: ', ft.getStdErr()
+    print 'Exit Status: ', ft.getExitStatus()