#!/usr/bin/env python import time from dmObject import DmObject from dm.common.utility.dictUtility import DictUtility from dm.common.utility.timeUtility import TimeUtility class UploadInfo(DmObject): DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'nProcessingErrors', 'nFiles', 'percentageComplete', 'startTime', 'endTime', 'runTime', 'startTimestamp', 'endTimestamp' ] def __init__(self, dict={}): DmObject.__init__(self, dict) self['fileDict'] = self.get('fileDict', {}) def updateStatus(self): now = time.time() uploadStatus = self.get('status', 'running') if uploadStatus == 'done': return fileDict = self.get('fileDict') nFiles = self.get('nFiles', len(fileDict)) nProcessedFiles = 0 nProcessingErrors = 0 processingErrors = {} endTime = 0 for (filePath,fileProcessingInfo) in fileDict.items(): if fileProcessingInfo.get('processed'): nProcessedFiles += 1 elif fileProcessingInfo.get('processingError'): nProcessingErrors += 1 processingErrors[filePath] = fileProcessingInfo.get('processingError') endProcessingTime = fileProcessingInfo.get('endProcessingTime') if endProcessingTime is not None and endProcessingTime > endTime: endTime = endProcessingTime if len(processingErrors): self['processingErrors'] = processingErrors startTime = self.get('startTime') if startTime: runTime = now - startTime self['runTime'] = runTime # need to handle 'failed' uploads nCompletedFiles = nProcessedFiles+nProcessingErrors if nCompletedFiles == nFiles: uploadStatus = 'done' if not endTime: endTime = now self['endTime'] = endTime self['endTimestamp'] = TimeUtility.formatLocalTimeStamp(endTime) if startTime: runTime = endTime - startTime self['runTime'] = runTime self['status'] = uploadStatus self['nProcessedFiles'] = '%s' % (nProcessedFiles) self['nProcessingErrors'] = '%s' % (nProcessingErrors) #self['nFiles'] = '%s' % (nFiles) percentageComplete = 100.0 percentageProcessed = 100.0 percentageProcessingErrors = 0.0 if nFiles > 0: percentageComplete = float(nCompletedFiles)/float(nFiles)*100.0 percentageProcessed = float(nProcessedFiles)/float(nFiles)*100.0 percentageProcessingErrors = float(nProcessingErrors)/float(nFiles)*100.0 self['percentageComplete'] = '%.2f' % percentageComplete self['percentageProcessed'] = '%.2f' % percentageProcessed self['percentageProcessingErrors'] = '%.2f' % percentageProcessingErrors def scrub(self, includeFileDetails=False): # Remove redundant information uploadInfo2 = DictUtility.deepCopy(self.data, excludeKeys=['fileDict']) if not includeFileDetails: return UploadInfo(uploadInfo2) fileDict = self.get('fileDict', {}) fileDict2 = {} for (filePath,fileInfo) in fileDict.items(): fileInfo2 = {} for key in ['processed', 'lastUpdateTime', 'processingError']: if fileInfo.has_key(key): fileInfo2[key] = fileInfo[key] fileDict2[filePath] = fileInfo2 uploadInfo2['fileDict'] = fileDict2 return UploadInfo(uploadInfo2)