diff --git a/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py new file mode 100755 index 0000000000000000000000000000000000000000..fe5e35758031745408d645658a6cd86a8a4ad20b --- /dev/null +++ b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +import os +import time +from dm.common.utility.loggingManager import LoggingManager +from dm.common.utility.timeUtility import TimeUtility +from dm.common.processing.plugins.fileProcessor import FileProcessor +from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi + +class MongoDbFileCatalogPlugin(FileProcessor): + + def __init__(self): + FileProcessor.__init__(self) + self.fileMongoDbApi = FileMongoDbApi() + self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__) + + def processFile(self, fileInfo): + experimentFilePath = fileInfo.get('experimentFilePath') + experimentName = fileInfo.get('experimentName') + self.logger.debug('Processing file %s for experiment %s' % (fileInfo, experimentName)) + + daqInfo = fileInfo.get('daqInfo') + storageDirectory = daqInfo.get('storageDirectory') + storageHost = daqInfo.get('storageHost') + storageUrl = daqInfo.get('storageUrl') + storageFilePath = os.path.join(storageDirectory, experimentFilePath) + fileProcessingTime = time.time() + fileProcessingTimeStamp = TimeUtility.formatLocalTimeStamp(fileProcessingTime) + + # Prepare catalogging entry + fileInfo2 = {} + for key in ['md5Sum', 'fileSize', 'fileCreationTime', 'fileCreationTimeStamp']: + if fileInfo.has_key(key): + fileInfo2[key] = fileInfo.get(key, '') + fileLocations = fileInfo.get('fileLocations', []) + fileLocations.append('%s/%s' % (storageUrl, experimentFilePath)) + + fileInfo2['fileName'] = os.path.basename(experimentFilePath) + fileInfo2['storageDirectory'] = storageDirectory + fileInfo2['storageUrl'] = storageUrl + fileInfo2['storageHost'] = storageHost + fileInfo2['storageFilePath'] = storageFilePath + fileInfo2['experimentFilePath'] = experimentFilePath + fileInfo2['experimentName'] = experimentName + fileInfo2['fileProcessingTime'] = fileProcessingTime + fileInfo2['fileProcessingTimeStamp'] = fileProcessingTimeStamp + fileInfo2['fileLocations'] = fileLocations + self.logger.debug('Daq info: %s' % (daqInfo)) + fileInfo2.update(daqInfo) + if daqInfo.has_key('id'): + fileInfo2['daqId'] = daqInfo.get('id') + del fileInfo2['id'] + + self.logger.debug('File %s catalog entry: %s' % (experimentFilePath, str(fileInfo2))) + self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2) + + +####################################################################### +# Testing. +if __name__ == '__main__': + pass