From 9d8333ef2b4196ca7e278cf9ff4d7bdfe9e1f64d Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Wed, 2 Dec 2015 17:21:27 +0000
Subject: [PATCH] object classes modifications related to daq and upload
tracking
---
src/python/dm/common/objects/daqInfo.py | 46 +++++++++++++++++++-
src/python/dm/common/objects/observedFile.py | 19 ++++----
src/python/dm/common/objects/uploadInfo.py | 41 ++++++++++++++++-
3 files changed, 93 insertions(+), 13 deletions(-)
diff --git a/src/python/dm/common/objects/daqInfo.py b/src/python/dm/common/objects/daqInfo.py
index e28bc5e9..c5d32214 100755
--- a/src/python/dm/common/objects/daqInfo.py
+++ b/src/python/dm/common/objects/daqInfo.py
@@ -1,11 +1,53 @@
#!/usr/bin/env python
+import copy
from dmObject import DmObject
+from dm.common.utility.dictUtility import DictUtility
class DaqInfo(DmObject):
- DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory' ]
+ DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'nFiles', 'startTimestamp', 'endTimestamp' ]
- def __init__(self, dict):
+ def __init__(self, dict={}):
DmObject.__init__(self, dict)
+ self.originalKeys = dict.keys()
+ self['fileDict'] = self.get('fileDict', {})
+ def updateStatus(self):
+ daqStatus = self.get('status', 'running')
+ if daqStatus == 'done':
+ return
+ fileDict = self.get('fileDict')
+ nFiles = len(fileDict)
+ nProcessedFiles = 0
+ for (filePath,uploadFileInfo) in fileDict.items():
+ if uploadFileInfo.get('processed'):
+ nProcessedFiles += 1
+ self['nProcessedFiles'] = '%s' % (nProcessedFiles)
+ self['nFiles'] = '%s' % (nFiles)
+
+ # need to handle 'failed' uploads
+ if self.get('endTime'):
+ daqStatus = 'done'
+ self['status'] = daqStatus
+
+ def toDictWithOriginalKeys(self):
+ dict = copy.deepcopy(self.data)
+ for key in dict.keys():
+ if key not in self.originalKeys:
+ del dict[key]
+ return dict
+
+ def scrub(self):
+ # Remove redundant information
+ daqInfo2 = DictUtility.deepCopy(self.data, excludeKeys='fileDict')
+ fileDict = self.get('fileDict', {})
+ fileDict2 = {}
+ for (filePath,fileInfo) in fileDict.items():
+ fileInfo2 = {}
+ for key in ['processed', 'lastUpdateTime']:
+ if fileInfo.has_key(key):
+ fileInfo2[key] = fileInfo[key]
+ fileDict2[filePath] = fileInfo2
+ daqInfo2['fileDict'] = fileDict2
+ return DaqInfo(daqInfo2)
diff --git a/src/python/dm/common/objects/observedFile.py b/src/python/dm/common/objects/observedFile.py
index 02de1c75..2648d958 100755
--- a/src/python/dm/common/objects/observedFile.py
+++ b/src/python/dm/common/objects/observedFile.py
@@ -7,7 +7,7 @@ import urlparse
class ObservedFile(DmObject):
- DEFAULT_KEY_LIST = [ 'filePath', 'lastUpdatedTimestamp' ]
+ DEFAULT_KEY_LIST = [ 'filePath', 'lastUpdateTime' ]
def __init__(self, dict={}, filePath=None, dataDirectory=None, experiment=None):
DmObject.__init__(self, dict)
@@ -19,13 +19,15 @@ class ObservedFile(DmObject):
parseResult = urlparse.urlparse(dataDirectory)
self['experimentFilePath'] = os.path.relpath(filePath, parseResult.path)
if experiment:
- self['experiment'] = experiment
+ self['experimentName'] = experiment.get('name')
+ self['storageHost'] = experiment.get('storageHost')
+ self['storageDirectory'] = experiment.get('storageDirectory')
- def setLastUpdatedTimestampToNow(self):
- self['lastUpdateTimestamp'] = time.time()
+ def setLastUpdateTimeToNow(self):
+ self['lastUpdateTime'] = time.time()
- def getLastUpdatedTimestamp(self):
- self.get('lastUpdateTimestamp')
+ def getLastUpdateTime(self):
+ self.get('lastUpdateTime')
def getFilePath(self):
return self.get('filePath')
@@ -33,14 +35,11 @@ class ObservedFile(DmObject):
def getDataDirectory(self):
return self.get('dataDirectory')
- def getExperiment(self):
- return self.get('experiment')
-
####################################################################
# Testing
if __name__ == '__main__':
of = ObservedFile(filePath='tmp/xyz')
print of
- of.setLastUpdatedTimestampToNow()
+ of.setLastUpdateTimeToNow()
print of
diff --git a/src/python/dm/common/objects/uploadInfo.py b/src/python/dm/common/objects/uploadInfo.py
index ffae07d5..d992e3c7 100755
--- a/src/python/dm/common/objects/uploadInfo.py
+++ b/src/python/dm/common/objects/uploadInfo.py
@@ -1,11 +1,50 @@
#!/usr/bin/env python
from dmObject import DmObject
+from dm.common.utility.dictUtility import DictUtility
class UploadInfo(DmObject):
- DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'percentageComplete' ]
+ DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'nFiles', 'percentageComplete', 'startTimestamp', 'endTimestamp' ]
def __init__(self, dict={}):
DmObject.__init__(self, dict)
+ self['fileDict'] = self.get('fileDict', {})
+
+ def updateStatus(self):
+ uploadStatus = self.get('status', 'running')
+ if uploadStatus == 'done':
+ return
+ fileDict = self.get('fileDict')
+ nFiles = len(fileDict)
+ nProcessedFiles = 0
+ for (filePath,uploadFileInfo) in fileDict.items():
+ if uploadFileInfo.get('processed'):
+ nProcessedFiles += 1
+
+ # need to handle 'failed' uploads
+ if nProcessedFiles == nFiles:
+ uploadStatus = 'done'
+ self['status'] = uploadStatus
+ self['nProcessedFiles'] = '%s' % (nProcessedFiles)
+ self['nFiles'] = '%s' % (nFiles)
+
+ percentageComplete = 100.0
+ if nFiles > 0:
+ percentageComplete = float(nProcessedFiles)/float(nFiles)*100.0
+ self['percentageComplete'] = '%.2f' % percentageComplete
+
+ def scrub(self):
+ # Remove redundant information
+ uploadInfo2 = DictUtility.deepCopy(self.data, excludeKeys='fileDict')
+ fileDict = self.get('fileDict', {})
+ fileDict2 = {}
+ for (filePath,fileInfo) in fileDict.items():
+ fileInfo2 = {}
+ for key in ['processed', 'lastUpdateTime']:
+ if fileInfo.has_key(key):
+ fileInfo2[key] = fileInfo[key]
+ fileDict2[filePath] = fileInfo2
+ uploadInfo2['fileDict'] = fileDict2
+ return UploadInfo(uploadInfo2)
--
GitLab