object classes modifications related to daq and upload tracking

src/python/dm/common/objects/daqInfo.py

 #!/usr/bin/env python
 
+import copy
 from dmObject import DmObject
+from dm.common.utility.dictUtility import DictUtility
 
 class DaqInfo(DmObject):
 
-    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory' ]
+    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'nFiles', 'startTimestamp', 'endTimestamp' ]
 
-    def __init__(self, dict):
+    def __init__(self, dict={}):
         DmObject.__init__(self, dict)
+        self.originalKeys = dict.keys()
+        self['fileDict'] = self.get('fileDict', {})
 
+    def updateStatus(self):
+        daqStatus = self.get('status', 'running')
+        if daqStatus == 'done':          
+            return 
+        fileDict = self.get('fileDict')
+        nFiles = len(fileDict)
+        nProcessedFiles = 0
+        for (filePath,uploadFileInfo) in fileDict.items():
+            if uploadFileInfo.get('processed'):
+                 nProcessedFiles += 1                
+        self['nProcessedFiles'] = '%s' % (nProcessedFiles)
+        self['nFiles'] = '%s' % (nFiles)
+
+        # need to handle 'failed' uploads
+        if self.get('endTime'):
+            daqStatus = 'done'
+        self['status'] = daqStatus
+
+    def toDictWithOriginalKeys(self):
+        dict = copy.deepcopy(self.data)
+        for key in dict.keys():
+            if key not in self.originalKeys:
+                del dict[key]
+        return dict
+
+    def scrub(self):
+        # Remove redundant information
+        daqInfo2 = DictUtility.deepCopy(self.data, excludeKeys='fileDict')
+        fileDict = self.get('fileDict', {})
+        fileDict2 = {}
+        for (filePath,fileInfo) in fileDict.items():
+            fileInfo2 = {}
+            for key in ['processed', 'lastUpdateTime']:
+                if fileInfo.has_key(key):
+                    fileInfo2[key] = fileInfo[key]
+            fileDict2[filePath] = fileInfo2
+        daqInfo2['fileDict'] = fileDict2
+        return DaqInfo(daqInfo2)

src/python/dm/common/objects/observedFile.py

@@ -7,7 +7,7 @@ import urlparse
 
 class ObservedFile(DmObject):
 
-    DEFAULT_KEY_LIST = [ 'filePath', 'lastUpdatedTimestamp' ]
+    DEFAULT_KEY_LIST = [ 'filePath', 'lastUpdateTime' ]
 
     def __init__(self, dict={}, filePath=None, dataDirectory=None, experiment=None):
         DmObject.__init__(self, dict)
@@ -19,13 +19,15 @@ class ObservedFile(DmObject):
                 parseResult = urlparse.urlparse(dataDirectory)
                 self['experimentFilePath'] = os.path.relpath(filePath, parseResult.path)
         if experiment:
-            self['experiment'] = experiment
+            self['experimentName'] = experiment.get('name')
+            self['storageHost'] = experiment.get('storageHost')
+            self['storageDirectory'] = experiment.get('storageDirectory')
 
-    def setLastUpdatedTimestampToNow(self):
-        self['lastUpdateTimestamp'] = time.time()
+    def setLastUpdateTimeToNow(self):
+        self['lastUpdateTime'] = time.time()
 
-    def getLastUpdatedTimestamp(self):
-        self.get('lastUpdateTimestamp')
+    def getLastUpdateTime(self):
+        self.get('lastUpdateTime')
 
     def getFilePath(self):
         return self.get('filePath')
@@ -33,14 +35,11 @@ class ObservedFile(DmObject):
     def getDataDirectory(self):
         return self.get('dataDirectory')
 
-    def getExperiment(self):
-        return self.get('experiment')
-
 ####################################################################
 # Testing
 if __name__ == '__main__':
     of = ObservedFile(filePath='tmp/xyz')
     print of
-    of.setLastUpdatedTimestampToNow()
+    of.setLastUpdateTimeToNow()
     print of
 

src/python/dm/common/objects/uploadInfo.py

 #!/usr/bin/env python
 
 from dmObject import DmObject
+from dm.common.utility.dictUtility import DictUtility
 
 class UploadInfo(DmObject):
 
-    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'percentageComplete' ]
+    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'nFiles', 'percentageComplete', 'startTimestamp', 'endTimestamp' ]
 
     def __init__(self, dict={}):
         DmObject.__init__(self, dict)
+        self['fileDict'] = self.get('fileDict', {})
+
+    def updateStatus(self):
+        uploadStatus = self.get('status', 'running')
+        if uploadStatus == 'done':
+            return 
+        fileDict = self.get('fileDict')
+        nFiles = len(fileDict)
+        nProcessedFiles = 0
+        for (filePath,uploadFileInfo) in fileDict.items():
+            if uploadFileInfo.get('processed'):
+                nProcessedFiles += 1
+
+        # need to handle 'failed' uploads
+        if nProcessedFiles == nFiles:
+            uploadStatus = 'done'
+        self['status'] = uploadStatus
+        self['nProcessedFiles'] = '%s' % (nProcessedFiles)
+        self['nFiles'] = '%s' % (nFiles)
+
+        percentageComplete = 100.0
+        if nFiles > 0:
+             percentageComplete = float(nProcessedFiles)/float(nFiles)*100.0
+        self['percentageComplete'] = '%.2f' % percentageComplete
+
+    def scrub(self):
+        # Remove redundant information
+        uploadInfo2 = DictUtility.deepCopy(self.data, excludeKeys='fileDict')
+        fileDict = self.get('fileDict', {})
+        fileDict2 = {}
+        for (filePath,fileInfo) in fileDict.items():
+            fileInfo2 = {}
+            for key in ['processed', 'lastUpdateTime']:
+                if fileInfo.has_key(key):
+                    fileInfo2[key] = fileInfo[key]
+            fileDict2[filePath] = fileInfo2
+        uploadInfo2['fileDict'] = fileDict2
+        return UploadInfo(uploadInfo2)