From 7f9f84ed3be1b364e8860b564955a2d6b021b0d7 Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Fri, 6 May 2016 19:05:35 +0000
Subject: [PATCH] reworked catalogging to use collection on a per experiment
basis
---
.../service/fileRouteDescriptor.py | 33 +++++-------
.../service/fileSessionController.py | 23 ++++----
.../service/impl/fileSessionControllerImpl.py | 11 ++--
.../dm/common/mongodb/api/fileMongoDbApi.py | 54 +++++++++++++------
.../common/mongodb/impl/datasetCollection.py | 1 +
.../common/mongodb/impl/dmMongoCollection.py | 3 +-
.../dm/common/mongodb/impl/fileCollection.py | 1 +
7 files changed, 66 insertions(+), 60 deletions(-)
diff --git a/src/python/dm/cat_web_service/service/fileRouteDescriptor.py b/src/python/dm/cat_web_service/service/fileRouteDescriptor.py
index 07931473..2a7e70b7 100755
--- a/src/python/dm/cat_web_service/service/fileRouteDescriptor.py
+++ b/src/python/dm/cat_web_service/service/fileRouteDescriptor.py
@@ -22,39 +22,30 @@ class FileRouteDescriptor:
# Add experiment file
{
'name' : 'addExperimentFile',
- 'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
+ 'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'controller' : fileSessionController,
'action' : 'addExperimentFile',
'method' : ['POST']
},
- # Update file by id
+ # Update experiment file by id
{
- 'name' : 'updateFileById',
- 'path' : '%s/files/:(id)' % contextRoot,
+ 'name' : 'updateExperimentFileById',
+ 'path' : '%s/filesByExperimentAndId/:(experimentName)/:(id)' % contextRoot,
'controller' : fileSessionController,
- 'action' : 'updateFileById',
+ 'action' : 'updateExperimentFileById',
'method' : ['PUT']
},
# Update experiment file by name
{
'name' : 'updateExperimentFile',
- 'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
+ 'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'controller' : fileSessionController,
'action' : 'updateExperimentFile',
'method' : ['PUT']
},
- # Get file info list
- {
- 'name' : 'getFiles',
- 'path' : '%s/files' % contextRoot,
- 'controller' : fileSessionController,
- 'action' : 'getFiles',
- 'method' : ['GET']
- },
-
# Get experiment file info list
{
'name' : 'getExperimentFiles',
@@ -64,19 +55,19 @@ class FileRouteDescriptor:
'method' : ['GET']
},
- # Get file by id
+ # Get experiment file by id
{
- 'name' : 'getFileById',
- 'path' : '%s/files/:(id)' % contextRoot,
+ 'name' : 'getExperimentFileById',
+ 'path' : '%s/filesByExperimentAndId/:(experimentName)/:(id)' % contextRoot,
'controller' : fileSessionController,
- 'action' : 'getFileById',
+ 'action' : 'getExperimentFileById',
'method' : ['GET']
},
- # Get experiment file by name
+ # Get experiment file by path
{
'name' : 'getExperimentFile',
- 'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
+ 'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
'controller' : fileSessionController,
'action' : 'getExperimentFile',
'method' : ['GET']
diff --git a/src/python/dm/cat_web_service/service/fileSessionController.py b/src/python/dm/cat_web_service/service/fileSessionController.py
index d9e3ecbd..ec9db79d 100755
--- a/src/python/dm/cat_web_service/service/fileSessionController.py
+++ b/src/python/dm/cat_web_service/service/fileSessionController.py
@@ -53,27 +53,20 @@ class FileSessionController(DmSessionController):
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
- def updateFileById(self, id, **kwargs):
+ def updateExperimentFileById(self, expermentName, id, **kwargs):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
if not id:
raise InvalidRequest('Invalid file id provided.')
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
- response = self.fileSessionControllerImpl.updateFileById(fileInfo).getFullJsonRep()
+ fileInfo['experimentName'] = experimentName
+ response = self.fileSessionControllerImpl.updateExperimentFileById(fileInfo).getFullJsonRep()
self.logger.debug('Updated file id %s: %s' % (id,response))
return response
- @cherrypy.expose
- @DmSessionController.require(DmSessionController.isAdministrator())
- @DmSessionController.execute
- def getFiles(self, **kwargs):
- encodedQueryDict = kwargs.get('queryDict')
- queryDict = {}
- if encodedQueryDict:
- queryDict = json.loads(Encoder.decode(encodedQueryDict))
- return self.listToJson(self.fileSessionControllerImpl.getFiles(queryDict=queryDict))
-
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
@@ -89,10 +82,12 @@ class FileSessionController(DmSessionController):
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
- def getFileById(self, id, **kwargs):
+ def getExperimentFileById(self, experimentName, id, **kwargs):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
if not id:
raise InvalidRequest('Invalid id provided.')
- response = self.fileSessionControllerImpl.getFileById(id).getFullJsonRep()
+ response = self.fileSessionControllerImpl.getExperimentFileById(experimentName, id).getFullJsonRep()
self.logger.debug('Returning file id %s: %s' % (id,response))
return response
diff --git a/src/python/dm/cat_web_service/service/impl/fileSessionControllerImpl.py b/src/python/dm/cat_web_service/service/impl/fileSessionControllerImpl.py
index 8cae7829..8f2a124f 100755
--- a/src/python/dm/cat_web_service/service/impl/fileSessionControllerImpl.py
+++ b/src/python/dm/cat_web_service/service/impl/fileSessionControllerImpl.py
@@ -21,17 +21,14 @@ class FileSessionControllerImpl(DmObjectManager):
def updateExperimentFile(self, fileInfo):
return self.fileMongoDbApi.updateExperimentFile(fileInfo)
- def updateFileById(self, fileInfo):
- return self.fileMongoDbApi.updateFileById(fileInfo)
-
- def getFiles(self, queryDict):
- return self.fileMongoDbApi.getFiles(queryDict=queryDict)
+ def updateExperimentFileById(self, fileInfo):
+ return self.fileMongoDbApi.updateExperimentFileById(fileInfo)
def getExperimentFiles(self, experimentName, queryDict):
return self.fileMongoDbApi.getExperimentFiles(experimentName, queryDict=queryDict)
- def getFileById(self, id):
- return self.fileMongoDbApi.getFileById(id)
+ def getExperimentFileById(self, experimentName, id):
+ return self.fileMongoDbApi.getExperimentFileById(experimentName, id)
def getExperimentFile(self, experimentName, fileName):
return self.fileMongoDbApi.getExperimentFile(experimentName, fileName)
diff --git a/src/python/dm/common/mongodb/api/fileMongoDbApi.py b/src/python/dm/common/mongodb/api/fileMongoDbApi.py
index 4c5a1fd5..2585776a 100755
--- a/src/python/dm/common/mongodb/api/fileMongoDbApi.py
+++ b/src/python/dm/common/mongodb/api/fileMongoDbApi.py
@@ -9,6 +9,7 @@ from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
from dm.common.mongodb.impl.fileCollection import FileCollection
from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
from dm.common.objects.fileMetadata import FileMetadata
+from dm.common.exceptions.invalidRequest import InvalidRequest
class FileMongoDbApi(DmMongoDbApi):
@@ -16,55 +17,74 @@ class FileMongoDbApi(DmMongoDbApi):
def __init__(self):
DmMongoDbApi.__init__(self)
- self.fileCollection = FileCollection(self.dbClient)
+ self.fileCollectionDict = {}
+
+ def getFileCollection(self, experimentName):
+ fileCollection = self.fileCollectionDict.get(experimentName)
+ if not fileCollection:
+ fileCollection = FileCollection(self.dbClient, experimentName)
+ self.fileCollectionDict[experimentName] = fileCollection
+ return fileCollection
+
+ @classmethod
+ def getAndCheckExperimentName(cls, dictObject):
+ experimentName = dictObject.get('experimentName')
+ cls.checkExperimentName(experimentName)
+ return experimentName
+
+ @classmethod
+ def checkExperimentName(cls, experimentName):
+ if not experimentName:
+ raise InvalidRequest('Experiment name has not been provided.')
@DmMongoDbApi.executeDbCall
def addExperimentFile(self, fileInfo, **kwargs):
+ experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
- dbFileMetadata = self.fileCollection.addByUniqueKeys(fileInfo2)
+ dbFileMetadata = self.getFileCollection(experimentName).addByUniqueKeys(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
- def getFiles(self, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
+ def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
+ self.checkExperimentName(experimentName)
queryDict2 = self.getMongoDict(queryDict)
self.getLogger().debug('Converting query dict to regex patterns')
ignoreCase = kwargs.get(self.REGEX_IGNORE_CASE_KEY, True)
queryDict2 = self.convertStringsToRegex(queryDict2, ignoreCase)
- return self.listToDmObjects(self.fileCollection.findByQueryDict(queryDict2, returnFieldDict), FileMetadata)
+ return self.listToDmObjects(self.getFileCollection(experimentName).findByQueryDict(queryDict2, returnFieldDict), FileMetadata)
@DmMongoDbApi.executeDbCall
- def getFileById(self, id, **kwargs):
- dbFileMetadata = self.fileCollection.findById(id)
+ def getExperimentFileById(self, experimentName, id, **kwargs):
+ self.checkExperimentName(experimentName)
+ dbFileMetadata = self.getFileCollection(experimentName).findById(id)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def getExperimentFile(self, experimentName, experimentFilePath, **kwargs):
+ self.checkExperimentName(experimentName)
queryDict = { 'experimentFilePath' : experimentFilePath, 'experimentName' : experimentName }
- dbFileMetadata = self.fileCollection.findByUniqueKeys(queryDict)
+ dbFileMetadata = self.getFileCollection(experimentName).findByUniqueKeys(queryDict)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
- def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
- queryDict2 = copy.copy(queryDict)
- queryDict2['experimentName'] = experimentName
- return self.getFiles(queryDict2, returnFieldDict)
-
- @DmMongoDbApi.executeDbCall
- def updateFileById(self, fileInfo, **kwargs):
+ def updateExperimentFileById(self, fileInfo, **kwargs):
+ experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
- dbFileMetadata = self.fileCollection.updateById(fileInfo2)
+ dbFileMetadata = self.getFileCollection(experimentName).updateById(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def updateExperimentFile(self, fileInfo, **kwargs):
+ experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
- dbFileMetadata = self.fileCollection.updateByUniqueKeys(fileInfo2)
+ dbFileMetadata = self.getFileCollection(experimentName).updateByUniqueKeys(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def updateOrAddExperimentFile(self, fileInfo, **kwargs):
+ experimentName = self.getAndCheckExperimentName(fileInfo)
fileInfo2 = self.getMongoDict(fileInfo)
- dbFileMetadata = self.fileCollection.updateOrAddByUniqueKeys(fileInfo2)
+ dbFileMetadata = self.getFileCollection(experimentName).updateOrAddByUniqueKeys(fileInfo2)
return self.toDmObject(dbFileMetadata, FileMetadata)
#######################################################################
diff --git a/src/python/dm/common/mongodb/impl/datasetCollection.py b/src/python/dm/common/mongodb/impl/datasetCollection.py
index abdb997a..6a23c4b8 100755
--- a/src/python/dm/common/mongodb/impl/datasetCollection.py
+++ b/src/python/dm/common/mongodb/impl/datasetCollection.py
@@ -11,6 +11,7 @@ class DatasetCollection(DmMongoCollection):
"""Class responsible for updating dataset collection in mongo db."""
UNIQUE_KEYS_LIST = [ '_id', 'datasetName', 'experimentName' ]
+ ITEM_TYPE = 'dataset'
def __init__(self, dbClient):
DmMongoCollection.__init__(self, 'datasets', dbClient)
diff --git a/src/python/dm/common/mongodb/impl/dmMongoCollection.py b/src/python/dm/common/mongodb/impl/dmMongoCollection.py
index f2c2c49f..ace4777f 100755
--- a/src/python/dm/common/mongodb/impl/dmMongoCollection.py
+++ b/src/python/dm/common/mongodb/impl/dmMongoCollection.py
@@ -12,6 +12,7 @@ class DmMongoCollection(object):
ALL_FIELDS_DICT = {'__return_only_id__' : False}
UNIQUE_KEYS_LIST = []
+ ITEM_TYPE = 'item'
def __init__(self, collectionName, dbClient):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
@@ -21,7 +22,7 @@ class DmMongoCollection(object):
self.collectionName = collectionName
# items => item
- self.itemName = collectionName[:-1]
+ self.itemName = self.ITEM_TYPE
# item => Item
self.capitalizedItemName = self.itemName.capitalize()
diff --git a/src/python/dm/common/mongodb/impl/fileCollection.py b/src/python/dm/common/mongodb/impl/fileCollection.py
index 7658aef2..c1f534e3 100755
--- a/src/python/dm/common/mongodb/impl/fileCollection.py
+++ b/src/python/dm/common/mongodb/impl/fileCollection.py
@@ -12,6 +12,7 @@ class FileCollection(DmMongoCollection):
UNIQUE_KEYS_LIST = [ 'experimentFilePath', 'experimentName' ]
NAME_KEY = 'experimentFilePath'
+ ITEM_TYPE = 'file'
def __init__(self, dbClient, experimentName=None):
collectionName = 'files'
--
GitLab