uploadInfo.py

#!/usr/bin/env python

import time
from dmObject import DmObject
from dm.common.utility.dictUtility import DictUtility
from dm.common.utility.timeUtility import TimeUtility

class UploadInfo(DmObject):

    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'nProcessingErrors', 'nFiles', 'percentageComplete', 'startTime', 'endTime', 'runTime', 'startTimestamp', 'endTimestamp' ]

    def __init__(self, dict={}):
        DmObject.__init__(self, dict)
        self['fileDict'] = self.get('fileDict', {})

    def updateStatus(self):
        now = time.time()
        uploadStatus = self.get('status', 'running')
        if uploadStatus == 'done':
            return 
        fileDict = self.get('fileDict')
        nFiles = self.get('nFiles', len(fileDict))
        nProcessedFiles = 0
        nProcessingErrors = 0
        processingErrors = {}
        endTime = 0
        for (filePath,fileProcessingInfo) in fileDict.items():
            if fileProcessingInfo.get('processed'):
                nProcessedFiles += 1
            elif fileProcessingInfo.get('processingError'):
                nProcessingErrors += 1
                processingErrors[filePath] = fileProcessingInfo.get('processingError')

            endProcessingTime = fileProcessingInfo.get('endProcessingTime')
            if endProcessingTime is not None and endProcessingTime > endTime:
                endTime = endProcessingTime
        if len(processingErrors):
            self['processingErrors'] = processingErrors

        startTime = self.get('startTime')
        if startTime:
            runTime = now - startTime
            self['runTime'] = runTime

        # need to handle 'failed' uploads
        nCompletedFiles = nProcessedFiles+nProcessingErrors
        if nCompletedFiles == nFiles:
            uploadStatus = 'done'
            if not endTime:
                endTime = now
            self['endTime'] = endTime
            self['endTimestamp'] = TimeUtility.formatLocalTimeStamp(endTime)
            if startTime:
                runTime = endTime - startTime
                self['runTime'] = runTime
        self['status'] = uploadStatus
        self['nProcessedFiles'] = '%s' % (nProcessedFiles)
        self['nProcessingErrors'] = '%s' % (nProcessingErrors)
        #self['nFiles'] = '%s' % (nFiles)

        percentageComplete = 100.0
        percentageProcessed = 100.0
        percentageProcessingErrors = 0.0
        if nFiles > 0:
             percentageComplete = float(nCompletedFiles)/float(nFiles)*100.0
             percentageProcessed = float(nProcessedFiles)/float(nFiles)*100.0
             percentageProcessingErrors = float(nProcessingErrors)/float(nFiles)*100.0
        self['percentageComplete'] = '%.2f' % percentageComplete
        self['percentageProcessed'] = '%.2f' % percentageProcessed
        self['percentageProcessingErrors'] = '%.2f' % percentageProcessingErrors

    def scrub(self, includeFileDetails=False):
        # Remove redundant information
        uploadInfo2 = DictUtility.deepCopy(self.data, excludeKeys=['fileDict'])
        if not includeFileDetails:
            return UploadInfo(uploadInfo2)
        fileDict = self.get('fileDict', {})
        fileDict2 = {}
        for (filePath,fileInfo) in fileDict.items():
            fileInfo2 = {}
            for key in ['processed', 'lastUpdateTime', 'processingError']:
                if fileInfo.has_key(key):
                    fileInfo2[key] = fileInfo[key]
            fileDict2[filePath] = fileInfo2
        uploadInfo2['fileDict'] = fileDict2
        return UploadInfo(uploadInfo2)