#!/usr/bin/env python import os import time from dm.common.utility.loggingManager import LoggingManager from dm.common.utility.timeUtility import TimeUtility from dm.common.processing.plugins.fileProcessor import FileProcessor from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi class MongoDbFileCatalogPlugin(FileProcessor): def __init__(self): FileProcessor.__init__(self) self.fileMongoDbApi = FileMongoDbApi() self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__) def processFile(self, fileInfo): experimentFilePath = fileInfo.get('experimentFilePath') experimentName = fileInfo.get('experimentName') self.logger.debug('Processing file "%s" for experiment %s: %s' % (experimentFilePath, experimentName, fileInfo)) daqInfo = fileInfo.get('daqInfo') storageDirectory = daqInfo.get('storageDirectory') storageHost = daqInfo.get('storageHost') storageUrl = daqInfo.get('storageUrl') storageFilePath = os.path.join(storageDirectory, experimentFilePath) fileInfo['fileProcessingTime'] = time.time() # Prepare catalogging entry fileInfo2 = {} for key in ['md5Sum', 'fileSize']: if fileInfo.has_key(key): fileInfo2[key] = fileInfo.get(key, '') for key in ['fileProcessingTime', 'fileCreationTime', 'fileModificationTime']: if fileInfo.has_key(key): t = fileInfo.get(key, 0) fileInfo2[key] = t key2 = '%sstamp' % key fileInfo2[key2] = TimeUtility.formatLocalTimestamp(t) fileLocations = fileInfo.get('fileLocations', []) fileLocations.append('%s/%s' % (storageUrl, experimentFilePath)) fileInfo2['fileName'] = os.path.basename(experimentFilePath) fileInfo2['experimentFilePath'] = experimentFilePath fileInfo2['experimentName'] = experimentName fileInfo2['fileLocations'] = fileLocations self.logger.debug('Daq info: %s' % (daqInfo)) fileInfo2.update(daqInfo) if daqInfo.has_key('id'): fileInfo2['daqId'] = daqInfo.get('id') del fileInfo2['id'] for key in ['storageDirectory', 'storageUrl', 'storageHost']: if fileInfo2.has_key(key): del fileInfo2[key] self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileInfo2))) self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2) ####################################################################### # Testing. if __name__ == '__main__': pass