performance improvements for upload: introduced timer for preparing upload files

src/python/dm/common/objects/uploadInfo.py

@@ -18,7 +18,7 @@ class UploadInfo(DmObject):
         if uploadStatus == 'done':
             return 
         fileDict = self.get('fileDict')
-        nFiles = len(fileDict)
+        nFiles = self.get('nFiles', len(fileDict))
         nProcessedFiles = 0
         nProcessingErrors = 0
         processingErrors = {}
@@ -51,7 +51,7 @@ class UploadInfo(DmObject):
         self['status'] = uploadStatus
         self['nProcessedFiles'] = '%s' % (nProcessedFiles)
         self['nProcessingErrors'] = '%s' % (nProcessingErrors)
-        self['nFiles'] = '%s' % (nFiles)
+        #self['nFiles'] = '%s' % (nFiles)
 
         percentageComplete = 100.0
         percentageProcessed = 100.0

src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py

@@ -103,17 +103,21 @@ class ExperimentSessionControllerImpl(DmObjectManager):
 
     def prepareUpload(self, uploadInfo, daqInfo, experiment, filePathsDict):
         uploadId = uploadInfo.get('id')
-        self.logger.debug('Preparing upload id: %s', uploadId)
+        self.logger.debug('Preparing upload id: %s' % uploadId)
         fileDict = uploadInfo.get('fileDict')
         dataDirectory = uploadInfo.get('dataDirectory')
+        fileProcessingManager = FileProcessingManager.getInstance()
         for filePath in filePathsDict.keys():
             fileInfo = ObservedFile(filePath=filePath, dataDirectory=dataDirectory, experiment=experiment)
             fileInfo['daqInfo'] = daqInfo
             fileInfo['uploadId'] = uploadId
             fileUploadInfo = { 'processed' : False }
             fileDict[filePath] = fileUploadInfo
-            fileProcessingManager.processFile(fileInfo)
-        self.logger.debug('Done preparing upload id: %s', uploadId)
+            try: 
+                fileProcessingManager.processFile(fileInfo)
+            except Exception, ex:
+                self.logger.error('Processing error: %s', ex)
+        self.logger.debug('Done preparing upload id: %s (total of %s files)' % (uploadId, len(filePathsDict)))
 
     def getUploadInfo(self, id, includeFileDetails=False):
         uploadInfo = UploadTracker.getInstance().get(id)