Skip to content
Snippets Groups Projects
Select Git revision
  • master default protected
  • 4.0.0
  • v4.0
  • 4.0.0a1
  • 3.3.2
  • 3.3.1
  • 3.3.0
  • 3.2.2
  • 3.2.1
  • 3.2.0
  • 3.1.0
  • 3.0.1
  • 3.0.0
  • 2.6.0
  • 0.3.0
  • 0.4.0
  • 0.5.0
  • 0.6.0
  • 0.1.0
  • 0.2.0
  • 0.7.0
21 results
Blame Permalink
Forked from DM / dm-docs
261 commits behind, 631 commits ahead of the upstream repository.
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
uploadInfo.py 4.69 KiB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109 






#!/usr/bin/env python

import time
import threading

from dmObject import DmObject
from dm.common.utility.dictUtility import DictUtility
from dm.common.utility.timeUtility import TimeUtility

class UploadInfo(DmObject):

    DEFAULT_KEY_LIST = [ 'id', 'experimentName', 'dataDirectory', 'status', 'nProcessedFiles', 'nProcessingErrors', 'nCancelledFiles', 'nFiles', 'percentageComplete', 'percentageProcessed', 'percentageProcessingErrors', 'percentageCancelled', 'startTime', 'endTime', 'runTime', 'startTimestamp', 'endTimestamp' ]

    def __init__(self, dict={}):
        DmObject.__init__(self, dict)
        self.lock = threading.RLock()

    def fileProcessed(self, filePath, processingEndTime):
        self.lock.acquire()
        try:
            self['nProcessedFiles'] = self.get('nProcessedFiles', 0) + 1
            lastFileProcessedTime = self.get('lastFileProcessedTime', 0)
            if processingEndTime is not None and processingEndTime > lastFileProcessedTime:
                self['lastFileProcessed'] = filePath
                self['lastFileProcessedTime'] = processingEndTime
        finally:
            self.lock.release()

    def fileProcessingError(self, filePath, processingError, processingEndTime):
        self.lock.acquire()
        try:
            processingErrors = self.get('processingErrors', {})
            processingErrors[filePath] = processingError
            self['processingErrors'] = processingErrors
            self['nProcessingErrors'] = len(processingErrors)
            lastFileProcessingErrorTime = self.get('lastFileProcessingErrorTime', 0)
            if processingEndTime is not None and processingEndTime > lastFileProcessingErrorTime:
                self['lastFileProcessingErrorTime'] = processingEndTime
        finally:
            self.lock.release()

    def fileProcessingCancelled(self, filePath, processingEndTime):
        self.lock.acquire()
        try:
            self['nCancelledFiles'] = self.get('nCancelledFiles', 0) + 1
            lastFileProcessingCancelledTime = self.get('lastFileProcessingCancelledTime', 0)
            if processingEndTime is not None and processingEndTime > lastFileProcessingCancelledTime:
                self['lastFileProcessingCancelledTime'] = processingEndTime
        finally:
            self.lock.release()

    def uploadAborted(self, nCancelledFiles):
        self.lock.acquire()
        try:
            self['nCancelledFiles'] = self.get('nCancelledFiles', 0) + nCancelledFiles
        finally:
            self.lock.release()

    def updateStatus(self):
        now = time.time()
        uploadStatus = self.get('status', 'running')
        if uploadStatus == 'done' or uploadStatus == 'aborted':
            return 
        nFiles = self.get('nFiles', 0)
        nProcessedFiles = self.get('nProcessedFiles', 0)
        nCancelledFiles = self.get('nCancelledFiles', 0)
        nProcessingErrors = self.get('nProcessingErrors', 0)
        processingErrors = self.get('processingErrors', {})
        nCompletedFiles = nProcessedFiles+nProcessingErrors

        startTime = self.get('startTime', now)
        runTime = now - startTime
        endTime = None
        if nCompletedFiles == nFiles:
            uploadStatus = 'done'
            lastFileProcessingErrorTime = self.get('lastFileProcessingErrorTime')
            lastFileProcessedTime = self.get('lastFileProcessedTime')
            endTime = lastFileProcessedTime
            if not endTime or lastFileProcessingErrorTime > endTime:
                endTime = lastFileProcessingErrorTime

        if nCancelledFiles > 0 and nCancelledFiles+nCompletedFiles == nFiles:
            uploadStatus = 'aborted'
            endTime = self.get('lastFileProcessingCancelledTime', now)

        if endTime:
            runTime = endTime - startTime
            self['endTime'] = endTime
            self['endTimestamp'] = TimeUtility.formatLocalTimestamp(endTime)
        self['runTime'] = runTime
        self['status'] = uploadStatus

        percentageComplete = 100.0
        percentageProcessed = 100.0
        percentageCancelled = 0.0
        percentageProcessingErrors = 0.0
        if nFiles > 0:
            percentageComplete = float(nCompletedFiles)/float(nFiles)*100.0
            percentageProcessed = float(nProcessedFiles)/float(nFiles)*100.0
            percentageCancelled = float(nCancelledFiles)/float(nFiles)*100.0
            percentageProcessingErrors = float(nProcessingErrors)/float(nFiles)*100.0
        self['percentageComplete'] = '%.2f' % percentageComplete
        self['percentageProcessed'] = '%.2f' % percentageProcessed
        self['percentageProcessingErrors'] = '%.2f' % percentageProcessingErrors
        if nCancelledFiles > 0:
            self['percentageCancelled'] = '%.2f' % percentageCancelled