#!/usr/bin/env python import os import time from dm.common.utility.loggingManager import LoggingManager from dm.common.utility.timeUtility import TimeUtility from dm.common.processing.plugins.fileProcessor import FileProcessor from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi class MongoDbFileCatalogPlugin(FileProcessor): def __init__(self): FileProcessor.__init__(self) self.fileMongoDbApi = FileMongoDbApi() self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__) def processFile(self, fileInfo): experimentFilePath = fileInfo.get('experimentFilePath') experimentName = fileInfo.get('experimentName') self.logger.debug('Processing file %s for experiment %s' % (fileInfo, experimentName)) daqInfo = fileInfo.get('daqInfo') storageDirectory = daqInfo.get('storageDirectory') storageHost = daqInfo.get('storageHost') storageUrl = daqInfo.get('storageUrl') storageFilePath = os.path.join(storageDirectory, experimentFilePath) fileProcessingTime = time.time() fileProcessingTimeStamp = TimeUtility.formatLocalTimeStamp(fileProcessingTime) # Prepare catalogging entry fileInfo2 = {} for key in ['md5Sum', 'fileSize', 'fileCreationTime', 'fileCreationTimeStamp', 'fileModificationTime', 'fileModificationTimeStamp']: if fileInfo.has_key(key): fileInfo2[key] = fileInfo.get(key, '') fileLocations = fileInfo.get('fileLocations', []) fileLocations.append('%s/%s' % (storageUrl, experimentFilePath)) fileInfo2['fileName'] = os.path.basename(experimentFilePath) fileInfo2['storageDirectory'] = storageDirectory fileInfo2['storageUrl'] = storageUrl fileInfo2['storageHost'] = storageHost fileInfo2['storageFilePath'] = storageFilePath fileInfo2['experimentFilePath'] = experimentFilePath fileInfo2['experimentName'] = experimentName fileInfo2['fileProcessingTime'] = fileProcessingTime fileInfo2['fileProcessingTimeStamp'] = fileProcessingTimeStamp fileInfo2['fileLocations'] = fileLocations self.logger.debug('Daq info: %s' % (daqInfo)) fileInfo2.update(daqInfo) if daqInfo.has_key('id'): fileInfo2['daqId'] = daqInfo.get('id') del fileInfo2['id'] self.logger.debug('File %s catalog entry: %s' % (experimentFilePath, str(fileInfo2))) self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2) ####################################################################### # Testing. if __name__ == '__main__': pass