reworked catalogging to use collection on a per experiment basis

src/python/dm/cat_web_service/service/fileRouteDescriptor.py

@@ -22,39 +22,30 @@ class FileRouteDescriptor:
             # Add experiment file
             {
                 'name' : 'addExperimentFile',
-                'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
+                'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
                 'controller' : fileSessionController,
                 'action' : 'addExperimentFile',
                 'method' : ['POST']
             },
 
-            # Update file by id
+            # Update experiment file by id
             {
-                'name' : 'updateFileById',
-                'path' : '%s/files/:(id)' % contextRoot,
+                'name' : 'updateExperimentFileById',
+                'path' : '%s/filesByExperimentAndId/:(experimentName)/:(id)' % contextRoot,
                 'controller' : fileSessionController,
-                'action' : 'updateFileById',
+                'action' : 'updateExperimentFileById',
                 'method' : ['PUT']
             },
 
             # Update experiment file by name
             {
                 'name' : 'updateExperimentFile',
-                'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
+                'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
                 'controller' : fileSessionController,
                 'action' : 'updateExperimentFile',
                 'method' : ['PUT']
             },
 
-            # Get file info list
-            {
-                'name' : 'getFiles',
-                'path' : '%s/files' % contextRoot,
-                'controller' : fileSessionController,
-                'action' : 'getFiles',
-                'method' : ['GET']
-            },
-
             # Get experiment file info list 
             {
                 'name' : 'getExperimentFiles',
@@ -64,19 +55,19 @@ class FileRouteDescriptor:
                 'method' : ['GET']
             },
 
-            # Get file by id
+            # Get experiment file by id
             {
-                'name' : 'getFileById',
-                'path' : '%s/files/:(id)' % contextRoot,
+                'name' : 'getExperimentFileById',
+                'path' : '%s/filesByExperimentAndId/:(experimentName)/:(id)' % contextRoot,
                 'controller' : fileSessionController,
-                'action' : 'getFileById',
+                'action' : 'getExperimentFileById',
                 'method' : ['GET']
             },
 
-            # Get experiment file by name
+            # Get experiment file by path
             {
                 'name' : 'getExperimentFile',
-                'path' : '%s/filesByExperiment/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
+                'path' : '%s/filesByExperimentAndPath/:(experimentName)/:(encodedExperimentFilePath)' % contextRoot,
                 'controller' : fileSessionController,
                 'action' : 'getExperimentFile',
                 'method' : ['GET']

src/python/dm/cat_web_service/service/fileSessionController.py

@@ -53,27 +53,20 @@ class FileSessionController(DmSessionController):
     @cherrypy.expose
     @DmSessionController.require(DmSessionController.isAdministrator())
     @DmSessionController.execute
-    def updateFileById(self, id, **kwargs):
+    def updateExperimentFileById(self, expermentName, id, **kwargs):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
         if not id:
             raise InvalidRequest('Invalid file id provided.')
         encodedFileInfo = kwargs.get('fileInfo')
         if not encodedFileInfo:
             raise InvalidRequest('Invalid file info provided.')
         fileInfo = json.loads(Encoder.decode(encodedFileInfo))
-        response = self.fileSessionControllerImpl.updateFileById(fileInfo).getFullJsonRep()
+        fileInfo['experimentName'] = experimentName
+        response = self.fileSessionControllerImpl.updateExperimentFileById(fileInfo).getFullJsonRep()
         self.logger.debug('Updated file id %s: %s' % (id,response))
         return response
 
-    @cherrypy.expose
-    @DmSessionController.require(DmSessionController.isAdministrator())
-    @DmSessionController.execute
-    def getFiles(self, **kwargs):
-        encodedQueryDict = kwargs.get('queryDict')
-        queryDict = {}
-        if encodedQueryDict:
-            queryDict = json.loads(Encoder.decode(encodedQueryDict))
-        return self.listToJson(self.fileSessionControllerImpl.getFiles(queryDict=queryDict))
-
     @cherrypy.expose
     @DmSessionController.require(DmSessionController.isAdministrator())
     @DmSessionController.execute
@@ -89,10 +82,12 @@ class FileSessionController(DmSessionController):
     @cherrypy.expose
     @DmSessionController.require(DmSessionController.isAdministrator())
     @DmSessionController.execute
-    def getFileById(self, id, **kwargs):
+    def getExperimentFileById(self, experimentName, id, **kwargs):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
         if not id:
             raise InvalidRequest('Invalid id provided.')
-        response = self.fileSessionControllerImpl.getFileById(id).getFullJsonRep()
+        response = self.fileSessionControllerImpl.getExperimentFileById(experimentName, id).getFullJsonRep()
         self.logger.debug('Returning file id %s: %s' % (id,response))
         return response
 

src/python/dm/cat_web_service/service/impl/fileSessionControllerImpl.py

@@ -21,17 +21,14 @@ class FileSessionControllerImpl(DmObjectManager):
     def updateExperimentFile(self, fileInfo):
         return self.fileMongoDbApi.updateExperimentFile(fileInfo)
 
-    def updateFileById(self, fileInfo):
-        return self.fileMongoDbApi.updateFileById(fileInfo)
-
-    def getFiles(self, queryDict):
-        return self.fileMongoDbApi.getFiles(queryDict=queryDict)
+    def updateExperimentFileById(self, fileInfo):
+        return self.fileMongoDbApi.updateExperimentFileById(fileInfo)
 
     def getExperimentFiles(self, experimentName, queryDict):
         return self.fileMongoDbApi.getExperimentFiles(experimentName, queryDict=queryDict)
 
-    def getFileById(self, id):
-        return self.fileMongoDbApi.getFileById(id)
+    def getExperimentFileById(self, experimentName, id):
+        return self.fileMongoDbApi.getExperimentFileById(experimentName, id)
 
     def getExperimentFile(self, experimentName, fileName):
         return self.fileMongoDbApi.getExperimentFile(experimentName, fileName)

src/python/dm/common/mongodb/api/fileMongoDbApi.py

@@ -9,6 +9,7 @@ from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
 from dm.common.mongodb.impl.fileCollection import FileCollection
 from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
 from dm.common.objects.fileMetadata import FileMetadata
+from dm.common.exceptions.invalidRequest import InvalidRequest
 
 class FileMongoDbApi(DmMongoDbApi):
 
@@ -16,55 +17,74 @@ class FileMongoDbApi(DmMongoDbApi):
 
     def __init__(self):
         DmMongoDbApi.__init__(self)
-        self.fileCollection = FileCollection(self.dbClient)
+        self.fileCollectionDict = {}
+
+    def getFileCollection(self, experimentName):
+        fileCollection = self.fileCollectionDict.get(experimentName)
+        if not fileCollection:
+            fileCollection = FileCollection(self.dbClient, experimentName)
+            self.fileCollectionDict[experimentName] = fileCollection
+        return fileCollection
+
+    @classmethod
+    def getAndCheckExperimentName(cls, dictObject):
+        experimentName = dictObject.get('experimentName')
+        cls.checkExperimentName(experimentName)
+        return experimentName
+
+    @classmethod
+    def checkExperimentName(cls, experimentName):
+        if not experimentName:
+            raise InvalidRequest('Experiment name has not been provided.')
 
     @DmMongoDbApi.executeDbCall
     def addExperimentFile(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
         fileInfo2 = self.getMongoDict(fileInfo)
-        dbFileMetadata = self.fileCollection.addByUniqueKeys(fileInfo2)
+        dbFileMetadata = self.getFileCollection(experimentName).addByUniqueKeys(fileInfo2)
         return self.toDmObject(dbFileMetadata, FileMetadata)
 
     @DmMongoDbApi.executeDbCall
-    def getFiles(self, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
+    def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
+        self.checkExperimentName(experimentName)
         queryDict2 = self.getMongoDict(queryDict)
         self.getLogger().debug('Converting query dict to regex patterns')
         ignoreCase = kwargs.get(self.REGEX_IGNORE_CASE_KEY, True)
         queryDict2 = self.convertStringsToRegex(queryDict2, ignoreCase)
-        return self.listToDmObjects(self.fileCollection.findByQueryDict(queryDict2, returnFieldDict), FileMetadata)
+        return self.listToDmObjects(self.getFileCollection(experimentName).findByQueryDict(queryDict2, returnFieldDict), FileMetadata)
 
     @DmMongoDbApi.executeDbCall
-    def getFileById(self, id, **kwargs):
-        dbFileMetadata = self.fileCollection.findById(id)
+    def getExperimentFileById(self, experimentName, id, **kwargs):
+        self.checkExperimentName(experimentName)
+        dbFileMetadata = self.getFileCollection(experimentName).findById(id)
         return self.toDmObject(dbFileMetadata, FileMetadata)
 
     @DmMongoDbApi.executeDbCall
     def getExperimentFile(self, experimentName, experimentFilePath, **kwargs):
+        self.checkExperimentName(experimentName)
         queryDict = { 'experimentFilePath' : experimentFilePath, 'experimentName' : experimentName }
-        dbFileMetadata = self.fileCollection.findByUniqueKeys(queryDict)
+        dbFileMetadata = self.getFileCollection(experimentName).findByUniqueKeys(queryDict)
         return self.toDmObject(dbFileMetadata, FileMetadata)
 
     @DmMongoDbApi.executeDbCall
-    def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
-        queryDict2 = copy.copy(queryDict)
-        queryDict2['experimentName'] = experimentName 
-        return self.getFiles(queryDict2, returnFieldDict)
-
-    @DmMongoDbApi.executeDbCall
-    def updateFileById(self, fileInfo, **kwargs):
+    def updateExperimentFileById(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
         fileInfo2 = self.getMongoDict(fileInfo)
-        dbFileMetadata = self.fileCollection.updateById(fileInfo2)
+        dbFileMetadata = self.getFileCollection(experimentName).updateById(fileInfo2)
         return self.toDmObject(dbFileMetadata, FileMetadata)
 
     @DmMongoDbApi.executeDbCall
     def updateExperimentFile(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
         fileInfo2 = self.getMongoDict(fileInfo)
-        dbFileMetadata = self.fileCollection.updateByUniqueKeys(fileInfo2)
+        dbFileMetadata = self.getFileCollection(experimentName).updateByUniqueKeys(fileInfo2)
         return self.toDmObject(dbFileMetadata, FileMetadata)
 
     @DmMongoDbApi.executeDbCall
     def updateOrAddExperimentFile(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
         fileInfo2 = self.getMongoDict(fileInfo)
-        dbFileMetadata = self.fileCollection.updateOrAddByUniqueKeys(fileInfo2)
+        dbFileMetadata = self.getFileCollection(experimentName).updateOrAddByUniqueKeys(fileInfo2)
         return self.toDmObject(dbFileMetadata, FileMetadata)
 
 #######################################################################

src/python/dm/common/mongodb/impl/datasetCollection.py

@@ -11,6 +11,7 @@ class DatasetCollection(DmMongoCollection):
     """Class responsible for updating dataset collection in mongo db."""
 
     UNIQUE_KEYS_LIST = [ '_id', 'datasetName', 'experimentName' ]
+    ITEM_TYPE = 'dataset'
 
     def __init__(self, dbClient):
         DmMongoCollection.__init__(self, 'datasets', dbClient)

src/python/dm/common/mongodb/impl/dmMongoCollection.py

@@ -12,6 +12,7 @@ class DmMongoCollection(object):
 
     ALL_FIELDS_DICT = {'__return_only_id__' : False}
     UNIQUE_KEYS_LIST = []
+    ITEM_TYPE = 'item'
 
     def __init__(self, collectionName, dbClient):
         self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
@@ -21,7 +22,7 @@ class DmMongoCollection(object):
         self.collectionName = collectionName
 
         # items => item
-        self.itemName = collectionName[:-1] 
+        self.itemName = self.ITEM_TYPE
 
         # item => Item
         self.capitalizedItemName = self.itemName.capitalize() 

src/python/dm/common/mongodb/impl/fileCollection.py

@@ -12,6 +12,7 @@ class FileCollection(DmMongoCollection):
 
     UNIQUE_KEYS_LIST = [ 'experimentFilePath', 'experimentName' ]
     NAME_KEY = 'experimentFilePath'
+    ITEM_TYPE = 'file'
 
     def __init__(self, dbClient, experimentName=None):
         collectionName = 'files'