From a1f895d89172d63cde8e31c797c6613e05ec8b94 Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Wed, 2 Dec 2015 17:19:43 +0000
Subject: [PATCH] plugin modifications needed to support new daq and upload
tracking framework
---
.../service/impl/mongoDbFileCatalogPlugin.py | 19 ++++++++++++-------
.../service/impl/scriptProcessingPlugin.py | 7 +++----
.../impl/sddsParameterProcessingPlugin.py | 7 +++----
.../service/impl/sgeJobSubmissionPlugin.py | 7 +++----
4 files changed, 21 insertions(+), 19 deletions(-)
diff --git a/src/python/dm/ds_web_service/service/impl/mongoDbFileCatalogPlugin.py b/src/python/dm/ds_web_service/service/impl/mongoDbFileCatalogPlugin.py
index 11ea3be2..5796944a 100755
--- a/src/python/dm/ds_web_service/service/impl/mongoDbFileCatalogPlugin.py
+++ b/src/python/dm/ds_web_service/service/impl/mongoDbFileCatalogPlugin.py
@@ -16,13 +16,13 @@ class MongoDbFileCatalogPlugin(FileProcessor):
def processFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
- experiment = fileInfo.get('experiment')
- experimentName = experiment.get('name')
+ experimentName = fileInfo.get('experimentName')
self.logger.debug('Processing file %s for experiment %s' % (fileInfo, experimentName))
- experimentStorageDirectory = experiment.get('storageDirectory')
- storageHost = experiment.get('storageHost')
- storageFilePath = os.path.join(experimentStorageDirectory, experimentFilePath)
+ daqInfo = fileInfo.get('daqInfo')
+ storageDirectory = daqInfo.get('storageDirectory')
+ storageHost = daqInfo.get('storageHost')
+ storageFilePath = os.path.join(storageDirectory, experimentFilePath)
fileProcessingTime = time.time()
fileProcessingTimeStamp = TimeUtility.formatLocalTimeStamp(fileProcessingTime)
@@ -32,14 +32,19 @@ class MongoDbFileCatalogPlugin(FileProcessor):
if fileInfo.has_key(key):
fileInfo2[key] = fileInfo.get(key, '')
fileInfo2['fileName'] = os.path.basename(experimentFilePath)
- fileInfo2['experimentStorageDirectory'] = experimentStorageDirectory
+ fileInfo2['storageDirectory'] = storageDirectory
fileInfo2['storageHost'] = storageHost
fileInfo2['storageFilePath'] = storageFilePath
fileInfo2['experimentFilePath'] = experimentFilePath
fileInfo2['experimentName'] = experimentName
fileInfo2['fileProcessingTime'] = fileProcessingTime
fileInfo2['fileProcessingTimeStamp'] = fileProcessingTimeStamp
- fileInfo2.update(fileInfo.get('daqInfo', {}))
+ self.logger.debug('Daq info: %s' % (daqInfo))
+ fileInfo2.update(daqInfo)
+ if daqInfo.has_key('id'):
+ fileInfo2['daqId'] = daqInfo.get('id')
+ del fileInfo2['id']
+
self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)
diff --git a/src/python/dm/ds_web_service/service/impl/scriptProcessingPlugin.py b/src/python/dm/ds_web_service/service/impl/scriptProcessingPlugin.py
index 75be4c78..c892f6dd 100755
--- a/src/python/dm/ds_web_service/service/impl/scriptProcessingPlugin.py
+++ b/src/python/dm/ds_web_service/service/impl/scriptProcessingPlugin.py
@@ -18,8 +18,7 @@ class ScriptProcessingPlugin(FileProcessor):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
- experiment = fileInfo.get('experiment')
- experimentName = experiment.get('name')
+ experimentName = fileInfo.get('experimentName')
experimentFilePath = fileInfo.get('experimentFilePath')
daqInfo = fileInfo.get('daqInfo', {})
processingScript = daqInfo.get(self.PROCESSING_SCRIPT_KEY)
@@ -29,8 +28,8 @@ class ScriptProcessingPlugin(FileProcessor):
self.logger.debug('Processing file %s for experiment %s' % (experimentFilePath, experimentName))
- experimentStorageDirectory = experiment.get('storageDirectory')
- storageFilePath = os.path.join(experimentStorageDirectory, experimentFilePath)
+ storageDirectory = fileInfo.get('storageDirectory')
+ storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Process file
try:
diff --git a/src/python/dm/ds_web_service/service/impl/sddsParameterProcessingPlugin.py b/src/python/dm/ds_web_service/service/impl/sddsParameterProcessingPlugin.py
index ed2f8f35..54c99808 100755
--- a/src/python/dm/ds_web_service/service/impl/sddsParameterProcessingPlugin.py
+++ b/src/python/dm/ds_web_service/service/impl/sddsParameterProcessingPlugin.py
@@ -19,8 +19,7 @@ class SddsParameterProcessingPlugin(FileProcessor):
def processFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
- experiment = fileInfo.get('experiment')
- experimentName = experiment.get('name')
+ experimentName = fileInfo.get('experimentName')
daqInfo = fileInfo.get('daqInfo', {})
processSddsParameters = daqInfo.get(self.PROCESS_SDDS_PARAMETERS_KEY)
if not processSddsParameters:
@@ -29,8 +28,8 @@ class SddsParameterProcessingPlugin(FileProcessor):
self.logger.debug('Processing file %s for experiment %s' % (fileInfo, experimentName))
- experimentStorageDirectory = experiment.get('storageDirectory')
- storageFilePath = os.path.join(experimentStorageDirectory, experimentFilePath)
+ storageDirectory = fileInfo.get('storageDirectory')
+ storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Load file
try:
diff --git a/src/python/dm/ds_web_service/service/impl/sgeJobSubmissionPlugin.py b/src/python/dm/ds_web_service/service/impl/sgeJobSubmissionPlugin.py
index 42ccb7ec..a63e7134 100755
--- a/src/python/dm/ds_web_service/service/impl/sgeJobSubmissionPlugin.py
+++ b/src/python/dm/ds_web_service/service/impl/sgeJobSubmissionPlugin.py
@@ -17,8 +17,7 @@ class SgeJobSubmissionPlugin(FileProcessor):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
- experiment = fileInfo.get('experiment')
- experimentName = experiment.get('name')
+ experimentName = fileInfo.get('experimentName')
experimentFilePath = fileInfo.get('experimentFilePath')
daqInfo = fileInfo.get('daqInfo', {})
sgeJobScript = daqInfo.get(self.SGE_JOB_SCRIPT_KEY)
@@ -28,8 +27,8 @@ class SgeJobSubmissionPlugin(FileProcessor):
self.logger.debug('Processing file %s for experiment %s' % (experimentFilePath, experimentName))
- experimentStorageDirectory = experiment.get('storageDirectory')
- storageFilePath = os.path.join(experimentStorageDirectory, experimentFilePath)
+ storageDirectory = fileInfo.get('storageDirectory')
+ storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Process file
try:
--
GitLab