diff --git a/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py index 38bd600044472a4e6b8f2a4d942f4a3683cb26b0..64ea7638e351b7a8e3801a263eadf51718aff871 100755 --- a/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py +++ b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py @@ -7,15 +7,44 @@ from dm.common.objects.observedFile import ObservedFile from dm.common.utility.timeUtility import TimeUtility from dm.common.processing.plugins.fileProcessor import FileProcessor from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi +from dm.common.utility.dmSubprocess import DmSubprocess class MongoDbFileCatalogPlugin(FileProcessor): - def __init__(self, dependsOn=[]): + DEFAULT_HDF5_METADATA_COMMAND = None + + def __init__(self, hdf5MetadataCommand=DEFAULT_HDF5_METADATA_COMMAND, dependsOn=[]): FileProcessor.__init__(self, dependsOn=dependsOn) self.fileMongoDbApi = FileMongoDbApi() + self.hdf5MetadataCommand = hdf5MetadataCommand self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__) + def processHdf5Metadata(self, filePath, fileInfo={}): + if not self.hdf5MetadataCommand: + return fileInfo + experimentName = fileInfo.get('experimentName', '') + if not filePath.endswith('.h5'): + return fileInfo + command = '%s %s' % (self.hdf5MetadataCommand, filePath) + subprocess = DmSubprocess.getSubprocess(command) + subprocess.run() + stdout = subprocess.getStdOut().replace('\n', ';') + parts = stdout.split(';') + for part in parts: + keyValue = part.split('=') + key = keyValue[0] + if not len(key): + continue + value = '' + if len(keyValue) > 1: + value = '='.join(keyValue[1:]) + if not fileInfo.has_key(key): + fileInfo[key] = value + else: + self.logger.warn('Key %s already exists for file %s (experiment: %s)' % (key, filePath, experimentName)) + def processFile(self, fileInfo): + filePath = fileInfo.get('filePath') experimentFilePath = fileInfo.get('experimentFilePath') experimentName = fileInfo.get('experimentName') self.logger.debug('Processing file "%s" for experiment %s' % (experimentFilePath, experimentName)) @@ -55,6 +84,7 @@ class MongoDbFileCatalogPlugin(FileProcessor): if fileInfo2.has_key(key): del fileInfo2[key] + self.processHdf5Metadata(filePath, fileInfo2) self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileInfo2))) self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)