Commit bf405af6 authored by sveseli's avatar sveseli
Browse files

Merge branch '0.9.3' into release/0.9

parents 9848a911 5fcb036a
#!/bin/sh
# Run command
if [ -z $DM_ROOT_DIR ]; then
cd `dirname $0` && myDir=`pwd`
setupFile=$myDir/../setup.sh
if [ ! -f $setupFile ]; then
echo "Cannot find setup file: $setupFile"
exit 1
fi
source $setupFile > /dev/null
fi
source dm_command_setup.sh
eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getFilesCli.py $DM_COMMAND_ARGS"
Release 0.12 (05/06/2016)
=============================
- Developed processing for HDF5 metadata in Mongo cataloging plugin
- Modified catalog API and service interfaces to use file collections on
a per-experiment basis
Release 0.11 (04/29/2016)
=============================
......
......@@ -15,5 +15,5 @@ DM_DAQ_WEB_SERVICE_HOST=DM_HOSTNAME
DM_DAQ_WEB_SERVICE_PORT=33336
DM_CAT_WEB_SERVICE_HOST=DM_HOSTNAME
DM_CAT_WEB_SERVICE_PORT=44436
DM_SOFTWARE_VERSION="0.11 (DM_DATE)"
DM_SOFTWARE_VERSION="0.12 (DM_DATE)"
......@@ -23,7 +23,7 @@ class FileRestApi(CatRestApi):
experimentFilePath = fileInfo.get('experimentFilePath')
if not experimentFilePath:
raise InvalidRequest('File metadata must contain experimentFilePath key.')
url = '%s/filesByExperiment/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
responseData = self.sendSessionRequest(url=url, method='POST')
return FileMetadata(responseData)
......@@ -36,28 +36,24 @@ class FileRestApi(CatRestApi):
experimentFilePath = fileInfo.get('experimentFilePath')
if not experimentFilePath:
raise InvalidRequest('File metadata must contain experimentFilePath key.')
url = '%s/filesByExperiment/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
responseData = self.sendSessionRequest(url=url, method='PUT')
return FileMetadata(responseData)
@CatRestApi.execute
def updateFileById(self, fileInfo):
def updateExperimentFileById(self, fileInfo):
experimentName = fileInfo.get('experimentName')
if not experimentName:
raise InvalidRequest('File metadata must contain experimentName key.')
id = fileInfo.get('id')
if not id:
raise InvalidRequest('File metadata must contain id key.')
url = '%s/files/%s' % (self.getContextRoot(), id)
url = '%s/filesByExperimentAndId/%s/%s' % (self.getContextRoot(), experimentName, id)
url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
responseData = self.sendSessionRequest(url=url, method='PUT')
return FileMetadata(responseData)
@CatRestApi.execute
def getFiles(self, queryDict={}):
url = '%s/files' % (self.getContextRoot())
url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
responseData = self.sendSessionRequest(url=url, method='GET')
return self.toDmObjectList(responseData, FileMetadata)
@CatRestApi.execute
def getExperimentFiles(self, experimentName, queryDict={}):
if not experimentName:
......@@ -68,10 +64,12 @@ class FileRestApi(CatRestApi):
return self.toDmObjectList(responseData, FileMetadata)
@CatRestApi.execute
def getFileById(self, id):
def getExperimentFileById(self, experimentName, id):
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
if not id:
raise InvalidRequest('Invalid file id provided.')
url = '%s/files/%s' % (self.getContextRoot(), id)
url = '%s/filesByExperimentAndId/%s/%s' % (self.getContextRoot(), experimentName, id)
responseData = self.sendSessionRequest(url=url, method='GET')
return FileMetadata(responseData)
......@@ -81,7 +79,7 @@ class FileRestApi(CatRestApi):
raise InvalidRequest('Invalid experiment name provided.')
if not experimentFilePath:
raise InvalidRequest('Invalid experiment file path provided.')
url = '%s/filesByExperiment/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
responseData = self.sendSessionRequest(url=url, method='GET')
return FileMetadata(responseData)
......@@ -90,9 +88,6 @@ class FileRestApi(CatRestApi):
if __name__ == '__main__':
api = FileRestApi()
print api.getFiles()
print api.getFileById('556de0059e058b0ef4c4413b')
print api.getFileByName('xyz-001')
import time
t = long(time.time())
......
#!/usr/bin/env python
from dm.cat_web_service.api.fileRestApi import FileRestApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
class GetFilesCli(CatWebServiceSessionCli):
def __init__(self):
CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
def checkArgs(self):
pass
def runCommand(self):
self.parseArgs(usage="""
dm-get-files
[key1:value1, key2:value2, ...]
Description:
Retrieve files from the metadata catalog. Only those files that
match provided key/value metadata pairs will be returned. If no specific
metadata key/values are requested, all experiment files will be returned.
""")
self.checkArgs()
api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
queryDict = self.splitArgsIntoDict()
fileMetadataList = api.getFiles(queryDict)
for fileMetadata in fileMetadataList:
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
#######################################################################
# Run command.
if __name__ == '__main__':
cli = GetFilesCli()
cli.run()
......@@ -22,39 +22,30 @@ class FileRouteDescriptor:
# Add experiment file
{
'name' : 'addExperimentFile',
'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'controller' : fileSessionController,
'action' : 'addExperimentFile',
'method' : ['POST']
},
# Update file by id
# Update experiment file by id
{
'name' : 'updateFileById',
'path' : '%s/files/:(id)' % contextRoot,
'name' : 'updateExperimentFileById',
'path' : '%s/filesByExperimentAndId/:(experimentName)/:(id)' % contextRoot,
'controller' : fileSessionController,
'action' : 'updateFileById',
'action' : 'updateExperimentFileById',
'method' : ['PUT']
},
# Update experiment file by name
{
'name' : 'updateExperimentFile',
'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'controller' : fileSessionController,
'action' : 'updateExperimentFile',
'method' : ['PUT']
},
# Get file info list
{
'name' : 'getFiles',
'path' : '%s/files' % contextRoot,
'controller' : fileSessionController,
'action' : 'getFiles',
'method' : ['GET']
},
# Get experiment file info list
{
'name' : 'getExperimentFiles',
......@@ -64,19 +55,19 @@ class FileRouteDescriptor:
'method' : ['GET']
},
# Get file by id
# Get experiment file by id
{
'name' : 'getFileById',
'path' : '%s/files/:(id)' % contextRoot,
'name' : 'getExperimentFileById',
'path' : '%s/filesByExperimentAndId/:(experimentName)/:(id)' % contextRoot,
'controller' : fileSessionController,
'action' : 'getFileById',
'action' : 'getExperimentFileById',
'method' : ['GET']
},
# Get experiment file by name
# Get experiment file by path
{
'name' : 'getExperimentFile',
'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'controller' : fileSessionController,
'action' : 'getExperimentFile',
'method' : ['GET']
......
......@@ -53,27 +53,20 @@ class FileSessionController(DmSessionController):
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def updateFileById(self, id, **kwargs):
def updateExperimentFileById(self, expermentName, id, **kwargs):
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
if not id:
raise InvalidRequest('Invalid file id provided.')
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
response = self.fileSessionControllerImpl.updateFileById(fileInfo).getFullJsonRep()
fileInfo['experimentName'] = experimentName
response = self.fileSessionControllerImpl.updateExperimentFileById(fileInfo).getFullJsonRep()
self.logger.debug('Updated file id %s: %s' % (id,response))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getFiles(self, **kwargs):
encodedQueryDict = kwargs.get('queryDict')
queryDict = {}
if encodedQueryDict:
queryDict = json.loads(Encoder.decode(encodedQueryDict))
return self.listToJson(self.fileSessionControllerImpl.getFiles(queryDict=queryDict))
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
......@@ -89,10 +82,12 @@ class FileSessionController(DmSessionController):
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getFileById(self, id, **kwargs):
def getExperimentFileById(self, experimentName, id, **kwargs):
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
if not id:
raise InvalidRequest('Invalid id provided.')
response = self.fileSessionControllerImpl.getFileById(id).getFullJsonRep()
response = self.fileSessionControllerImpl.getExperimentFileById(experimentName, id).getFullJsonRep()
self.logger.debug('Returning file id %s: %s' % (id,response))
return response
......
......@@ -21,17 +21,14 @@ class FileSessionControllerImpl(DmObjectManager):
def updateExperimentFile(self, fileInfo):
return self.fileMongoDbApi.updateExperimentFile(fileInfo)
def updateFileById(self, fileInfo):
return self.fileMongoDbApi.updateFileById(fileInfo)
def getFiles(self, queryDict):
return self.fileMongoDbApi.getFiles(queryDict=queryDict)
def updateExperimentFileById(self, fileInfo):
return self.fileMongoDbApi.updateExperimentFileById(fileInfo)
def getExperimentFiles(self, experimentName, queryDict):
return self.fileMongoDbApi.getExperimentFiles(experimentName, queryDict=queryDict)
def getFileById(self, id):
return self.fileMongoDbApi.getFileById(id)
def getExperimentFileById(self, experimentName, id):
return self.fileMongoDbApi.getExperimentFileById(experimentName, id)
def getExperimentFile(self, experimentName, fileName):
return self.fileMongoDbApi.getExperimentFile(experimentName, fileName)
......
......@@ -9,6 +9,7 @@ from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
from dm.common.mongodb.impl.fileCollection import FileCollection
from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
from dm.common.objects.fileMetadata import FileMetadata
from dm.common.exceptions.invalidRequest import InvalidRequest
class FileMongoDbApi(DmMongoDbApi):
......@@ -16,55 +17,74 @@ class FileMongoDbApi(DmMongoDbApi):
def __init__(self):
DmMongoDbApi.__init__(self)
self.fileCollection = FileCollection(self.dbClient)
self.fileCollectionDict = {}
def getFileCollection(self, experimentName):
fileCollection = self.fileCollectionDict.get(experimentName)
if not fileCollection:
fileCollection = FileCollection(self.dbClient, experimentName)
self.fileCollectionDict[experimentName] = fileCollection
return fileCollection
@classmethod
def getAndCheckExperimentName(cls, dictObject):
experimentName = dictObject.get('experimentName')
cls.checkExperimentName(experimentName)
return experimentName
@classmethod
def checkExperimentName(cls, experimentName):
if not experimentName:
raise InvalidRequest('Experiment name has not been provided.')
@DmMongoDbApi.executeDbCall
def addExperimentFile(self, fileInfo, **kwargs):
experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
dbFileMetadata = self.fileCollection.addByUniqueKeys(fileInfo2)
dbFileMetadata = self.getFileCollection(experimentName).addByUniqueKeys(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def getFiles(self, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
self.checkExperimentName(experimentName)
queryDict2 = self.getMongoDict(queryDict)
self.getLogger().debug('Converting query dict to regex patterns')
ignoreCase = kwargs.get(self.REGEX_IGNORE_CASE_KEY, True)
queryDict2 = self.convertStringsToRegex(queryDict2, ignoreCase)
return self.listToDmObjects(self.fileCollection.findByQueryDict(queryDict2, returnFieldDict), FileMetadata)
return self.listToDmObjects(self.getFileCollection(experimentName).findByQueryDict(queryDict2, returnFieldDict), FileMetadata)
@DmMongoDbApi.executeDbCall
def getFileById(self, id, **kwargs):
dbFileMetadata = self.fileCollection.findById(id)
def getExperimentFileById(self, experimentName, id, **kwargs):
self.checkExperimentName(experimentName)
dbFileMetadata = self.getFileCollection(experimentName).findById(id)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def getExperimentFile(self, experimentName, experimentFilePath, **kwargs):
self.checkExperimentName(experimentName)
queryDict = { 'experimentFilePath' : experimentFilePath, 'experimentName' : experimentName }
dbFileMetadata = self.fileCollection.findByUniqueKeys(queryDict)
dbFileMetadata = self.getFileCollection(experimentName).findByUniqueKeys(queryDict)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
queryDict2 = copy.copy(queryDict)
queryDict2['experimentName'] = experimentName
return self.getFiles(queryDict2, returnFieldDict)
@DmMongoDbApi.executeDbCall
def updateFileById(self, fileInfo, **kwargs):
def updateExperimentFileById(self, fileInfo, **kwargs):
experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
dbFileMetadata = self.fileCollection.updateById(fileInfo2)
dbFileMetadata = self.getFileCollection(experimentName).updateById(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def updateExperimentFile(self, fileInfo, **kwargs):
experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
dbFileMetadata = self.fileCollection.updateByUniqueKeys(fileInfo2)
dbFileMetadata = self.getFileCollection(experimentName).updateByUniqueKeys(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def updateOrAddExperimentFile(self, fileInfo, **kwargs):
experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
dbFileMetadata = self.fileCollection.updateOrAddByUniqueKeys(fileInfo2)
dbFileMetadata = self.getFileCollection(experimentName).updateOrAddByUniqueKeys(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
#######################################################################
......
......@@ -11,6 +11,7 @@ class DatasetCollection(DmMongoCollection):
"""Class responsible for updating dataset collection in mongo db."""
UNIQUE_KEYS_LIST = [ '_id', 'datasetName', 'experimentName' ]
ITEM_TYPE = 'dataset'
def __init__(self, dbClient):
DmMongoCollection.__init__(self, 'datasets', dbClient)
......
......@@ -12,6 +12,7 @@ class DmMongoCollection(object):
ALL_FIELDS_DICT = {'__return_only_id__' : False}
UNIQUE_KEYS_LIST = []
ITEM_TYPE = 'item'
def __init__(self, collectionName, dbClient):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
......@@ -21,7 +22,7 @@ class DmMongoCollection(object):
self.collectionName = collectionName
# items => item
self.itemName = collectionName[:-1]
self.itemName = self.ITEM_TYPE
# item => Item
self.capitalizedItemName = self.itemName.capitalize()
......
......@@ -12,6 +12,7 @@ class FileCollection(DmMongoCollection):
UNIQUE_KEYS_LIST = [ 'experimentFilePath', 'experimentName' ]
NAME_KEY = 'experimentFilePath'
ITEM_TYPE = 'file'
def __init__(self, dbClient, experimentName=None):
collectionName = 'files'
......
......@@ -7,15 +7,44 @@ from dm.common.objects.observedFile import ObservedFile
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi
from dm.common.utility.dmSubprocess import DmSubprocess
class MongoDbFileCatalogPlugin(FileProcessor):
def __init__(self, dependsOn=[]):
DEFAULT_HDF5_METADATA_COMMAND = None
def __init__(self, hdf5MetadataCommand=DEFAULT_HDF5_METADATA_COMMAND, dependsOn=[]):
FileProcessor.__init__(self, dependsOn=dependsOn)
self.fileMongoDbApi = FileMongoDbApi()
self.hdf5MetadataCommand = hdf5MetadataCommand
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processHdf5Metadata(self, filePath, fileInfo={}):
if not self.hdf5MetadataCommand:
return fileInfo
experimentName = fileInfo.get('experimentName', '')
if not filePath.endswith('.h5'):
return fileInfo
command = '%s %s' % (self.hdf5MetadataCommand, filePath)
subprocess = DmSubprocess.getSubprocess(command)
subprocess.run()
stdout = subprocess.getStdOut().replace('\n', ';')
parts = stdout.split(';')
for part in parts:
keyValue = part.split('=')
key = keyValue[0]
if not len(key):
continue
value = ''
if len(keyValue) > 1:
value = '='.join(keyValue[1:])
if not fileInfo.has_key(key):
fileInfo[key] = value
else:
self.logger.warn('Key %s already exists for file %s (experiment: %s)' % (key, filePath, experimentName))
def processFile(self, fileInfo):
filePath = fileInfo.get('filePath')
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
self.logger.debug('Processing file "%s" for experiment %s' % (experimentFilePath, experimentName))
......@@ -55,6 +84,7 @@ class MongoDbFileCatalogPlugin(FileProcessor):
if fileInfo2.has_key(key):
del fileInfo2[key]
self.processHdf5Metadata(filePath, fileInfo2)
self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileInfo2)))
self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)
......
......@@ -39,7 +39,7 @@ class ExperimentSessionController(DmSessionController):
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getExperimentById(self, id, **kwargs):
response = self.experimentSessionControllerImpl.getExperimentByid(id).getFullJsonRep()
response = self.experimentSessionControllerImpl.getExperimentById(id).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment