From cda6acbcc2e2aa2af7e2a3edf473e80b5dbfd68e Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Tue, 2 Feb 2016 07:47:36 +0000
Subject: [PATCH] performance improvements for upload: introduced timer for
preparing upload files
---
src/python/dm/common/objects/uploadInfo.py | 4 ++--
.../service/impl/experimentSessionControllerImpl.py | 10 +++++++---
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/src/python/dm/common/objects/uploadInfo.py b/src/python/dm/common/objects/uploadInfo.py
index 06469ab7..98088420 100755
--- a/src/python/dm/common/objects/uploadInfo.py
+++ b/src/python/dm/common/objects/uploadInfo.py
@@ -18,7 +18,7 @@ class UploadInfo(DmObject):
if uploadStatus == 'done':
return
fileDict = self.get('fileDict')
- nFiles = len(fileDict)
+ nFiles = self.get('nFiles', len(fileDict))
nProcessedFiles = 0
nProcessingErrors = 0
processingErrors = {}
@@ -51,7 +51,7 @@ class UploadInfo(DmObject):
self['status'] = uploadStatus
self['nProcessedFiles'] = '%s' % (nProcessedFiles)
self['nProcessingErrors'] = '%s' % (nProcessingErrors)
- self['nFiles'] = '%s' % (nFiles)
+ #self['nFiles'] = '%s' % (nFiles)
percentageComplete = 100.0
percentageProcessed = 100.0
diff --git a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
index df6608ff..cddc1cec 100755
--- a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
+++ b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
@@ -103,17 +103,21 @@ class ExperimentSessionControllerImpl(DmObjectManager):
def prepareUpload(self, uploadInfo, daqInfo, experiment, filePathsDict):
uploadId = uploadInfo.get('id')
- self.logger.debug('Preparing upload id: %s', uploadId)
+ self.logger.debug('Preparing upload id: %s' % uploadId)
fileDict = uploadInfo.get('fileDict')
dataDirectory = uploadInfo.get('dataDirectory')
+ fileProcessingManager = FileProcessingManager.getInstance()
for filePath in filePathsDict.keys():
fileInfo = ObservedFile(filePath=filePath, dataDirectory=dataDirectory, experiment=experiment)
fileInfo['daqInfo'] = daqInfo
fileInfo['uploadId'] = uploadId
fileUploadInfo = { 'processed' : False }
fileDict[filePath] = fileUploadInfo
- fileProcessingManager.processFile(fileInfo)
- self.logger.debug('Done preparing upload id: %s', uploadId)
+ try:
+ fileProcessingManager.processFile(fileInfo)
+ except Exception, ex:
+ self.logger.error('Processing error: %s', ex)
+ self.logger.debug('Done preparing upload id: %s (total of %s files)' % (uploadId, len(filePathsDict)))
def getUploadInfo(self, id, includeFileDetails=False):
uploadInfo = UploadTracker.getInstance().get(id)
--
GitLab