src/python/dm/common/mongodb/api/datasetMongoDbApi.py

+#!/usr/bin/env python
+
+import copy
+import types
+import re
+
+from dm.common.objects.dmObject import DmObject
+from dm.common.exceptions.dmException import DmException
+from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
+from dm.common.mongodb.impl.datasetCollection import DatasetCollection
+from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi
+from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
+from dm.common.objects.datasetMetadata import DatasetMetadata
+
+class DatasetMongoDbApi(DmMongoDbApi):
+
+    SYSTEM_KEY_LIST = ['_id', 'datasetName', 'experimentName']
+    USE_DATASET_NAME_KEY = '_useDatasetName' 
+    USE_EXPERIMENT_NAME_KEY = '_useDatasetName' 
+
+    def __init__(self):
+        DmMongoDbApi.__init__(self)
+        self.datasetCollection = DatasetCollection(self.dbClient)
+        self.fileMongoDbApi = FileMongoDbApi()
+
+    @DmMongoDbApi.executeDbCall
+    def addExperimentDataset(self, datasetInfo, **kwargs):
+        datasetInfo2 = self.getMongoDict(datasetInfo)
+        dbDatasetMetadata = self.datasetCollection.addByUniqueKeys(datasetInfo2)
+        return self.toDmObject(dbDatasetMetadata, DatasetMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getDatasets(self, queryDict={}, returnFieldDict=DatasetCollection.ALL_FIELDS_DICT, **kwargs):
+        queryDict2 = self.getMongoDict(queryDict)
+        return self.listToDmObjects(self.datasetCollection.findByQueryDict(queryDict2, returnFieldDict), DatasetMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getDatasetById(self, id, **kwargs):
+        dbDatasetMetadata = self.datasetCollection.findById(id)
+        return self.toDmObject(dbDatasetMetadata, DatasetMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getExperimentDataset(self, experimentName, datasetName, **kwargs):
+        queryDict = { 'datasetName' : datasetName, 'experimentName' : experimentName }
+        dbDatasetMetadata = self.datasetCollection.findByUniqueKeys(queryDict)
+        return self.toDmObject(dbDatasetMetadata, DatasetMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getExperimentDatasets(self, experimentName, queryDict={}, returnFieldDict=DatasetCollection.ALL_FIELDS_DICT, **kwargs):
+        queryDict2 = copy.copy(queryDict)
+        queryDict2['experimentName'] = experimentName 
+        return self.getDatasets(queryDict2, returnFieldDict)
+
+    @DmMongoDbApi.executeDbCall
+    def updateDatasetById(self, datasetInfo, **kwargs):
+        datasetInfo2 = self.getMongoDict(datasetInfo)
+        dbDatasetMetadata = self.datasetCollection.updateById(datasetInfo2)
+        return self.toDmObject(dbDatasetMetadata, DatasetMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def updateExperimentDataset(self, datasetInfo, **kwargs):
+        datasetInfo2 = self.getMongoDict(datasetInfo)
+        dbDatasetMetadata = self.datasetCollection.updateByUniqueKeys(datasetInfo2)
+        return self.toDmObject(dbDatasetMetadata, DatasetMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def updateOrAddExperimentDataset(self, datasetInfo, **kwargs):
+        datasetInfo2 = self.getMongoDict(datasetInfo)
+        dbDatasetMetadata = self.datasetCollection.updateOrAddByUniqueKeys(datasetInfo2)
+        return self.toDmObject(dbDatasetMetadata, DatasetMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getExperimentDatasetFiles(self, experimentName, datasetName, returnFieldDict=DatasetCollection.ALL_FIELDS_DICT, **kwargs):
+        self.getLogger().debug('Looking for dataset %s (experiment %s)' % (datasetName, experimentName))
+        datasetDict = self.getExperimentDataset(experimentName, datasetName).getDictRep(DmObject.ALL_KEYS)
+        del datasetDict['id']
+        useDatasetName = kwargs.get(self.USE_DATASET_NAME_KEY, False)
+        if not useDatasetName:
+            del datasetDict['datasetName']
+        useExperimentName = kwargs.get(self.USE_EXPERIMENT_NAME_KEY, False)
+        if not useExperimentName:
+            del datasetDict['experimentName']
+        self.getLogger().debug('Converting dataset dict to regex patterns')
+        ignoreCase = kwargs.get(self.REGEX_IGNORE_CASE_KEY, True)
+        queryDict = self.convertStringsToRegex(datasetDict, ignoreCase)
+        return self.fileMongoDbApi.getExperimentFiles(experimentName, queryDict, returnFieldDict)
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    api = DatasetMongoDbApi()
+
+    import time
+    t = long(time.time())
+    datasetName = 'd-%s' % t
+    experimentName = 'exp-01'
+    datasetInfo = {'datasetName' : datasetName, 'intKey' : 1, 'doubleKey' : 2.0, 'stringKey' : 'myString' , 'experimentName' : experimentName}
+    dataset = api.updateOrAddExperimentDataset(datasetInfo)
+    print '\nADDED DATASET\n', dataset
+
+    datasets = api.getDatasets()
+    print '\nDATASETS: \n', api.getDatasets()
+    for dataset in datasets:
+        print dataset.getDictRep()
+
+    #import re
+    #regex = re.compile("file0.*", re.IGNORECASE)
+    datasetName = 'dataset1'
+    datasetInfo = {'datasetName' : datasetName, 'experimentName' : experimentName, 'fileName' : 'file0.*'}
+    dataset = api.updateOrAddExperimentDataset(datasetInfo)
+    print '\nUPDATED DATASET\n', dataset
+
+    print '\nDATASET FILES\n', api.getExperimentDatasetFiles(experimentName, datasetName)
+
+
+

src/python/dm/common/mongodb/api/dmMongoDbApi.py

+#!/usr/bin/env python
+
+import copy
+import types
+import re
+
+from dm.common.exceptions.dmException import DmException
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.mongodb.impl.mongoDbManager import MongoDbManager
+
+class DmMongoDbApi:
+    """ Base Mongo DB API class. """
+
+    SYSTEM_KEY_LIST = ['_id']
+    REGEX_IGNORE_KEY_LIST = ['_id']
+    REGEX_IGNORE_CASE_KEY = '_ignoreCase'
+
+    def __init__(self):
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        self.dbClient = MongoDbManager.getInstance().getDbClient()
+
+    # Decorator for all DB methods
+    @classmethod
+    def executeDbCall(cls, func):
+        def dbCall(*args, **kwargs):
+            try:
+                try:
+                    return func(*args, **kwargs)
+                except DmException, ex:
+                    raise
+                except Exception, ex:
+                    cls.getLogger().exception('%s' % ex)
+                    raise DmException(exception=ex)
+            finally:
+                # For now, do nothing
+                pass
+        return dbCall
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+    @classmethod
+    def listToDmObjects(cls, mongoDbObjectList, dmObjectClass):
+        dmObjectList = []
+        for o in mongoDbObjectList:
+            dmObjectList.append(cls.toDmObject(o, dmObjectClass))
+        return dmObjectList
+
+    @classmethod
+    def toDmObject(cls, mongoDbObject, dmObjectClass):
+        cls.scrubMongoDbObject(mongoDbObject)
+        return dmObjectClass(mongoDbObject)
+
+    @classmethod
+    def scrubMongoDbObject(cls, mongoDbObject):
+        for key in cls.SYSTEM_KEY_LIST:
+            if mongoDbObject.has_key(key):
+                # Remove leading underscore
+                if key.startswith('_'):
+                    newKey = key[1:]
+                    mongoDbObject[newKey] = str(mongoDbObject[key])
+                    del mongoDbObject[key]
+
+    @classmethod
+    def getMongoDict(cls, dict):
+        mongoDict = copy.copy(dict)
+        for key in cls.SYSTEM_KEY_LIST:
+            # if dictionary has system key, do not modify it
+            if mongoDict.has_key(key):
+                continue
+            # If provided dict has system key without leading underscore
+            # modify it
+            key2 = key[1:]
+            if mongoDict.has_key(key2):
+                mongoDict[key] = mongoDict[key2]
+                del mongoDict[key2]
+        return mongoDict
+
+    @classmethod
+    def convertStringsToRegex(cls, dict, ignoreCase=True):
+        dict2 = copy.copy(dict)
+        for (key,value) in dict2.items():
+            if key in cls.REGEX_IGNORE_KEY_LIST:
+                continue
+            elif type(value) == types.StringType or type(value) == types.UnicodeType:
+                cls.getLogger().debug('Converting to regex: %s for key %s' % (value,key))
+                if ignoreCase:
+                    regex = re.compile(value, re.IGNORECASE)
+                else:
+                    regex = re.compile(value)
+                dict2[key] = regex
+        return dict2
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    api = DmMongoDbApi()
+
+

src/python/dm/common/mongodb/api/fileMongoDbApi.py

+#!/usr/bin/env python
+
+import copy
+import types
+import re
+
+from dm.common.exceptions.dmException import DmException
+from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
+from dm.common.mongodb.impl.fileCollection import FileCollection
+from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
+from dm.common.objects.fileMetadata import FileMetadata
+from dm.common.exceptions.invalidRequest import InvalidRequest
+
+class FileMongoDbApi(DmMongoDbApi):
+
+    SYSTEM_KEY_LIST = ['_id', 'experimentFilePath', 'experimentName']
+
+    def __init__(self):
+        DmMongoDbApi.__init__(self)
+        self.fileCollectionDict = {}
+
+    def getFileCollection(self, experimentName):
+        fileCollection = self.fileCollectionDict.get(experimentName)
+        if not fileCollection:
+            fileCollection = FileCollection(self.dbClient, experimentName)
+            self.fileCollectionDict[experimentName] = fileCollection
+        return fileCollection
+
+    @classmethod
+    def getAndCheckExperimentName(cls, dictObject):
+        experimentName = dictObject.get('experimentName')
+        cls.checkExperimentName(experimentName)
+        return experimentName
+
+    @classmethod
+    def checkExperimentName(cls, experimentName):
+        if not experimentName:
+            raise InvalidRequest('Experiment name has not been provided.')
+
+    @DmMongoDbApi.executeDbCall
+    def addExperimentFile(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
+        fileInfo2 = self.getMongoDict(fileInfo)
+        dbFileMetadata = self.getFileCollection(experimentName).addByUniqueKeys(fileInfo2)
+        return self.toDmObject(dbFileMetadata, FileMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
+        self.checkExperimentName(experimentName)
+        queryDict2 = self.getMongoDict(queryDict)
+        self.getLogger().debug('Converting query dict to regex patterns')
+        ignoreCase = kwargs.get(self.REGEX_IGNORE_CASE_KEY, True)
+        queryDict2 = self.convertStringsToRegex(queryDict2, ignoreCase)
+        return self.listToDmObjects(self.getFileCollection(experimentName).findByQueryDict(queryDict2, returnFieldDict), FileMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getExperimentFileById(self, experimentName, id, **kwargs):
+        self.checkExperimentName(experimentName)
+        dbFileMetadata = self.getFileCollection(experimentName).findById(id)
+        return self.toDmObject(dbFileMetadata, FileMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def getExperimentFile(self, experimentName, experimentFilePath, **kwargs):
+        self.checkExperimentName(experimentName)
+        queryDict = { 'experimentFilePath' : experimentFilePath, 'experimentName' : experimentName }
+        dbFileMetadata = self.getFileCollection(experimentName).findByUniqueKeys(queryDict)
+        return self.toDmObject(dbFileMetadata, FileMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def updateExperimentFileById(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
+        fileInfo2 = self.getMongoDict(fileInfo)
+        dbFileMetadata = self.getFileCollection(experimentName).updateById(fileInfo2)
+        return self.toDmObject(dbFileMetadata, FileMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def updateExperimentFile(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
+        fileInfo2 = self.getMongoDict(fileInfo)
+        dbFileMetadata = self.getFileCollection(experimentName).updateByUniqueKeys(fileInfo2)
+        return self.toDmObject(dbFileMetadata, FileMetadata)
+
+    @DmMongoDbApi.executeDbCall
+    def updateOrAddExperimentFile(self, fileInfo, **kwargs):
+        experimentName = self.getAndCheckExperimentName(fileInfo)
+        fileInfo2 = self.getMongoDict(fileInfo)
+        dbFileMetadata = self.getFileCollection(experimentName).updateOrAddByUniqueKeys(fileInfo2)
+        return self.toDmObject(dbFileMetadata, FileMetadata)
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    from dm.common.mongodb.impl.mongoDbManager import MongoDbManager
+    mgr = MongoDbManager.getInstance()
+    mgr.dbName = 'dm'
+    api = FileMongoDbApi()
+
+    experimentFilePath = 'file02'
+    experimentName = 'exp-01'
+    fileInfo = {'experimentFilePath' : experimentFilePath, 'intKey' : 1, 'doubleKey' : 2.0, 'stringKey' : 'myString' , 'experimentName' : experimentName}
+    file = api.updateOrAddExperimentFile(fileInfo)
+    print '\nADDED FILE\n', file
+
+    import time
+    t = long(time.time())
+    experimentFilePath = 'f-%s' % t
+    experimentName = 'exp-01'
+    fileInfo = {'experimentFilePath' : experimentFilePath, 'intKey' : 1, 'doubleKey' : 2.0, 'stringKey' : 'myString' , 'dictKey' : {'a' : 'A', 'b' : 'B', 'c' : 3}, 'experimentName' : experimentName}
+    file = api.updateOrAddExperimentFile(fileInfo)
+    print '\nADDED FILE\n', file
+
+    files = api.getFiles()
+    for file in files:
+        print 'FILE: %s\n' % file.getDictRep()
+
+    fileInfo = {'experimentFilePath' : experimentFilePath, 'experimentName' : experimentName, 'intKey' : 101}
+    file = api.updateExperimentFile(fileInfo)
+    print '\nUPDATED FILE\n', file
+
+    print '\nFILES FOR EXPERIMENT exp1: \n', api.getExperimentFiles(experimentName)
+    print '\nFILES FOR EXPERIMENT exp1 with experimentFilePath=file01: \n', api.getExperimentFiles(experimentName, queryDict={'experimentFilePath':'file02'})
+
+

src/python/dm/common/mongodb/api/workflowMongoDbApi.py

+#!/usr/bin/env python
+
+import copy
+import types
+import re
+
+from dm.common.exceptions.dmException import DmException
+from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
+from dm.common.mongodb.impl.workflowCollection import WorkflowCollection
+from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
+from dm.common.objects.workflow import Workflow
+from dm.common.exceptions.invalidRequest import InvalidRequest
+
+class WorkflowMongoDbApi(DmMongoDbApi):
+
+    SYSTEM_KEY_LIST = ['_id', 'name', 'owner']
+
+    def __init__(self):
+        DmMongoDbApi.__init__(self)
+        self.workflowCollectionDict = {}
+
+    def getWorkflowCollection(self, owner):
+        workflowCollection = self.workflowCollectionDict.get(owner)
+        if not workflowCollection:
+            workflowCollection = WorkflowCollection(self.dbClient, owner)
+            self.workflowCollectionDict[owner] = workflowCollection
+        return workflowCollection
+
+    @classmethod
+    def getAndCheckOwner(cls, dictObject):
+        owner = dictObject.get('owner')
+        cls.checkOwner(owner)
+        return owner
+
+    @classmethod
+    def checkOwner(cls, owner):
+        if not owner:
+            raise InvalidRequest('Owner name has not been provided.')
+
+    @DmMongoDbApi.executeDbCall
+    def addWorkflow(self, workflow, **kwargs):
+        owner = self.getAndCheckOwner(owner)
+        workflow2 = self.getMongoDict(workflow)
+        dbWorkflow = self.getWorkflowCollection(owner).addByUniqueKeys(workflow2)
+        return self.toDmObject(dbWorkflow, Workflow)
+
+    @DmMongoDbApi.executeDbCall
+    def getWorkflows(self, owner, queryDict={}, returnFieldDict=WorkflowCollection.ALL_FIELDS_DICT, **kwargs):
+        self.checkOwner(owner)
+        queryDict2 = self.getMongoDict(queryDict)
+        ignoreCase = kwargs.get(self.REGEX_IGNORE_CASE_KEY, True)
+        queryDict2 = self.convertStringsToRegex(queryDict2, ignoreCase)
+        return self.listToDmObjects(self.getWorkflowCollection(owner).findByQueryDict(queryDict2, returnFieldDict), Workflow)
+
+    @DmMongoDbApi.executeDbCall
+    def getWorkflowById(self, owner, id, **kwargs):
+        self.checkOwner(owner)
+        dbWorkflow = self.getWorkflowCollection(owner).findById(id)
+        return self.toDmObject(dbWorkflow, Workflow)
+
+    @DmMongoDbApi.executeDbCall
+    def getWorkflowByName(self, owner, name, **kwargs):
+        self.checkOwner(owner)
+        queryDict = { 'name' : name, 'owner' : owner }
+        dbWorkflow = self.getWorkflowCollection(owner).findByUniqueKeys(queryDict)
+        return self.toDmObject(dbWorkflow, Workflow)
+
+    @DmMongoDbApi.executeDbCall
+    def updateWorkflowById(self, workflow, **kwargs):
+        owner = self.getAndCheckOwner(workflow)
+        workflow2 = self.getMongoDict(workflow)
+        dbWorkflow = self.getWorkflowCollection(owner).updateById(workflow2)
+        return self.toDmObject(dbWorkflow, Workflow)
+
+    @DmMongoDbApi.executeDbCall
+    def updateWorkflow(self, workflow, **kwargs):
+        owner = self.getAndCheckOwner(workflow)
+        workflow2 = self.getMongoDict(workflow)
+        dbWorkflow = self.getWorkflowCollection(owner).updateByUniqueKeys(workflow2)
+        return self.toDmObject(dbWorkflow, Workflow)
+
+    @DmMongoDbApi.executeDbCall
+    def updateOrAddWorkflow(self, workflow, **kwargs):
+        owner = self.getAndCheckOwner(workflow)
+        workflow2 = self.getMongoDict(workflow)
+        dbWorkflow = self.getWorkflowCollection(owner).updateOrAddByUniqueKeys(workflow2)
+        return self.toDmObject(dbWorkflow, Workflow)
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    from dm.common.mongodb.impl.mongoDbManager import MongoDbManager
+    mgr = MongoDbManager.getInstance()
+    mgr.dbName = 'dm'
+    api = WorkflowMongoDbApi()
+
+    workflows = api.getWorkflows('sveseli')
+    for workflow in workflows:
+        print 'WORKFLOW: %s\n' % workflow.getDictRep()
+
+    workflow = api.getWorkflowByName('sveseli', 'workflow-01')
+    print 'WORKFLOW: %s\n' % workflow.getDictRep()
+
+

src/python/dm/common/mongodb/impl/datasetCollection.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.dbError import DbError
+from dmMongoCollection import DmMongoCollection
+
+class DatasetCollection(DmMongoCollection):
+    """Class responsible for updating dataset collection in mongo db."""
+
+    UNIQUE_KEYS_LIST = [ '_id', 'datasetName', 'experimentName' ]
+    ITEM_TYPE = 'dataset'
+
+    def __init__(self, dbClient):
+        DmMongoCollection.__init__(self, 'datasets', dbClient)
+
+#######################################################################
+# Testing
+if __name__ == '__main__':
+    from dmMongoClient import DmMongoClient
+    mongo = DmMongoClient('dm')
+    datasetCollection = DatasetCollection(mongo)
+    datasetInfo = {'name' : 'ds-001', 
+        'owner' : 'sv',
+        'experimentName' : 'exp-001',
+        'voltage' : { 'gt' : 400},
+        'current' : { 'lt' : 100},
+    }
+    #print datasetCollection.add(datasetInfo)
+    print datasetCollection.updateByName(datasetInfo)
+    print datasetCollection.findByName('ds-001')
+    print datasetCollection.findByQueryDict({'dataset' : 'ds-001'}, {'owner' : 1})
+
+
+

src/python/dm/common/mongodb/impl/dmMongoClient.py

+#!/usr/bin/env python
+
+from pymongo import MongoClient
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.dbError import DbError
+
+class DmMongoClient(object):
+    """ DM MongoDB client."""
+
+    def __init__(self, dbName, dbUri='mongodb://localhost:27017/'):
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        try:
+            self.client = MongoClient(dbUri)
+            self.db = self.client[dbName]
+            self.dbName = dbName
+        except Exception, ex:
+            self.logger.error('Cannot connect to Mongo DB: %s' % ex)
+            raise DbError(exception=ex)
+
+    def drop(self, collection):
+        return self.db[collection].drop()
+
+    def insert(self, collection, dict, **kwargs):
+        return self.db[collection].insert(dict, **kwargs)
+
+    def update(self, collection, query, update, **kwargs):
+        key = '.'.join((collection, str(update['$set'].keys()[0])))
+        return self.db[collection].update(query, update, **kwargs)
+
+    def findOne(self, collection, criteria={}):
+        return self.db[collection].find_one(criteria)
+
+    def find(self, collection, criteria={}, projections={}):
+        return self.db[collection].find(criteria, projections)
+
+    def findAsList(self, collection, criteria={}, projections={}):
+        return list(self.db[collection].find(criteria, projections))
+
+    def getCollectionNames(self):
+        return self.db.collection_names()
+
+#######################################################################
+# Testing
+if __name__ == '__main__':
+    mongo = DmMongoClient('dm')
+    mongo.drop('students')
+    id = mongo.insert('students', { '_id' : 1, 'semester' : 1, 'grades' : [ 70, 87, 90 ]   })
+    print 'Student #1 id: ', id
+    id = mongo.insert('students', { '_id' : 2, 'semester' : 1, 'grades' : [ 90, 88, 92 ] } )
+    print 'Student #2 id: ', id
+    print mongo.findAsList('students', criteria={ 'semester' : 1, 'grades': { '$gte' : 85 } }, projections={ 'grades.$' : 1 } )
+    result = mongo.update('files', {'_id' : 2}, {'$set' : {'experimentId' : 2}}, upsert=True)
+    print result
+    doc = mongo.findOne('files', {'experimentId' : 1})
+    print doc
+    docs = mongo.find('files', projections={'experimentId' : 1})
+    print docs.count()
+    docs = mongo.findAsList('files', projections={'experimentId' : 1})
+    print docs
+    print 'Collection names: ', mongo.getCollectionNames()
+    mongo.drop('students')
+    print 'Collection names (after drop): ', mongo.getCollectionNames()

src/python/dm/common/mongodb/impl/dmMongoCollection.py

+#!/usr/bin/env python
+
+from bson.objectid import ObjectId
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.dbError import DbError
+
+class DmMongoCollection(object):
+    """Base collection in mongo db."""
+
+    ALL_FIELDS_DICT = {'__return_only_id__' : False}
+    UNIQUE_KEYS_LIST = []
+    ITEM_TYPE = 'item'
+
+    def __init__(self, collectionName, dbClient):
+        self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+        self.dbClient = dbClient
+
+        # Collection consists of items: [item]
+        self.collectionName = collectionName
+
+        # items => item
+        self.itemName = self.ITEM_TYPE
+
+        # item => Item
+        self.capitalizedItemName = self.itemName.capitalize() 
+
+    @classmethod
+    def getUniqueKeys(cls):
+        return cls.UNIQUE_KEYS_LIST
+
+    def findByKey(self, key, value):
+        dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
+        if dbObjectDict is None:
+            raise ObjectNotFound('%s with %s=%s not found.' % (self.capitalizedItemName, key, value))
+        return dbObjectDict
+
+    def findByKeys(self, keyList, queryDict):
+        for key in keyList:
+            if not queryDict.has_key(key):
+                raise InvalidArgument('%s query dictionary does not specify key %s.' % (self.capitalizedItemName, key))
+
+        dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
+        if dbObjectDict is None:
+            raise ObjectNotFound('%s with properties %s not found.' % (self.capitalizedItemName, queryDict))
+        return dbObjectDict
+
+    def findByUniqueKeys(self, queryDict):
+        return self.findByKeys(self.UNIQUE_KEYS_LIST, queryDict)
+
+    def findById(self, id):
+        return self.findByKey('_id', ObjectId(id))
+
+    def findByName(self, name):
+        return self.findByKey(self.NAME_KEY, name)
+
+    def findByQueryDict(self, queryDict, returnFieldDict=ALL_FIELDS_DICT):
+        return self.dbClient.findAsList(self.collectionName, queryDict, returnFieldDict)
+
+    def listByKey(self, key):
+        return self.dbClient.findAsList(self.collectionName, {}, {key : True})
+
+    def listById(self):
+        return self.dbClient.findAsList(self.collectionName, {}, {})
+
+    def listByName(self):
+        return self.findByName('_name')
+
+    def listByKeys(self, keyList):
+        returnFieldDict = {}
+        for key in keyList:
+            returnFieldDict[key] = True
+        return self.dbClient.findAsList(self.collectionName, {}, returnFieldDict)
+    def listAll(self):
+        return self.dbClient.findAsList(self.collectionName, {}, DmMongoCollection.ALL_FIELDS_DICT)
+
+    def __addDbObject(self, objectDict):
+        try:
+            self.dbClient.insert(self.collectionName, objectDict)
+        except Exception, ex:
+            self.logger.error('Cannot add %s with %s set to %s: %s' % (self.itemName, key, value, ex))
+            raise DbError(exception=ex)
+        return objectDict
+
+    def addByKey(self, key, objectDict):
+        value = objectDict.get(key)
+        if value is None:
+            raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
+
+        dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
+        if dbObjectDict is not None:
+            raise ObjectAlreadyExists('%s with %s set to %s already exists.' % (self.capitalizedItemName, key, value))
+
+        return self.__addDbObject(objectDict)
+
+    def addByName(self, objectDict):
+        return self.addByKey('_name', objectDict)
+
+    def addByKeys(self, keyList, objectDict):
+        queryDict = {}
+        for key in keyList:
+            value = objectDict.get(key)
+            if value is None:
+                raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
+            queryDict[key] = value
+
+        dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
+        if dbObjectDict is not None:
+            raise ObjectAlreadyExists('%s with properties %s already exists.' % (self.capitalizedItemName, queryDict))
+
+        return self.__addDbObject(objectDict)
+
+    def addByUniqueKeys(self, objectDict):
+        return self.addByKeys(self.UNIQUE_KEYS_LIST, objectDict)
+
+    def __updateDbObject(self, dbObjectDict, objectDict):
+        try:
+            id = dbObjectDict.get('_id')
+            dbObjectDict.update(objectDict)
+            self.dbClient.update(self.collectionName,
+                {'_id' : id},
+                {'$set' : dbObjectDict})
+        except Exception, ex:
+            self.logger.error('Cannot update %s %s with %s: %s' % (self.itemName, dbObjectDict, objectDict, ex))
+            raise DbError(exception=ex)
+        return dbObjectDict
+
+    def updateByKey(self, key, objectDict):
+        value = objectDict.get(key)
+        if value is None:
+            raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
+
+        dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
+        if dbObjectDict is None:
+            raise ObjectNotFound('%s with %s set to %s not found.' % (self.capitalizedItemName, key, value))
+
+        return self.__updateDbObject(dbObjectDict, objectDict)
+
+    def updateByKeys(self, keyList, objectDict):
+        queryDict = {}
+        for key in keyList:
+            value = objectDict.get(key)
+            if value is None:
+                raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
+            queryDict[key] = value
+
+        dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
+        if dbObjectDict is None:
+            raise ObjectNotFound('%s with properties %s not found.' % (self.capitalizedItemName, queryDict))
+
+        return self.__updateDbObject(dbObjectDict, objectDict)
+
+    def updateByUniqueKeys(self, objectDict):
+        return self.updateByKeys(self.UNIQUE_KEYS_LIST, objectDict)
+
+    def updateById(self, objectDict):
+        # Convert 'id' to '_id' key if needed, and wrap it with ObjectId()
+        if not objectDict.has_key('_id') and objectDict.has_key('id'):
+            objectDict['_id'] = ObjectId(objectDict['id'])
+            del objectDict['id']
+        return self.updateByKey('_id', objectDict)
+
+    def updateByName(self, objectDict):
+        return self.updateByKey('_name', objectDict)
+
+    def updateOrAddByKey(self, key, objectDict):
+        value = objectDict.get(key)
+        if value is None:
+            raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
+
+        dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
+        if dbObjectDict is None:
+            return self.__addDbObject(objectDict)
+        else:
+            return self.__updateDbObject(dbObjectDict, objectDict)
+
+    def updateOrAddByKeys(self, keyList, objectDict):
+        queryDict = {}
+        for key in keyList:
+            value = objectDict.get(key)
+            if value is None:
+                raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
+            queryDict[key] = value
+
+        dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
+        if dbObjectDict is None:
+            return self.__addDbObject(objectDict)
+        else:
+            return self.__updateDbObject(dbObjectDict, objectDict)
+
+    def updateOrAddByUniqueKeys(self, objectDict):
+        return self.updateOrAddByKeys(self.UNIQUE_KEYS_LIST, objectDict)
+
+    def updateOrAddByName(self, objectDict):
+        return self.updateOrAddByKey('_name', objectDict)
+
+#######################################################################
+# Testing
+if __name__ == '__main__':
+    from dmMongoClient import DmMongoClient
+    from bson.objectid import ObjectId
+    mongo = DmMongoClient('dm')
+    fileCollectionImpl = DmMongoCollection('files', mongo)
+    #objectDict = {'name' : 'xyz-001', 'experiment' : 'myexp-D', 'update' : 'sv2', 'locationList' : '[/opt/xyz, /data/xyz]'}
+    #print fileCollectionImpl.updateOrAddByKey('name', objectDict)
+    #f = fileCollectionImpl.findByKey('name', 'xyz-001')
+    #print f
+    #print type(f['_id'])
+    #print fileCollectionImpl.findById('556de0059e058b0ef4c4413b')
+    #print fileCollectionImpl.findByQueryDict({'experiment' : 'exp-001'}, {'locationList' : 1})
+
+    #print 'LIST BY ID\n', fileCollectionImpl.listById()
+    #print 'LIST BY NAME\n', fileCollectionImpl.listByKey('name')
+    print 'LIST ALL\n'
+    for f in fileCollectionImpl.listAll():
+        print 'FILE: %s\n' % f
+
+
+    #print fileCollectionImpl.updateByKeys(['name', 'experiment'], objectDict)
+

src/python/dm/common/mongodb/impl/experimentCollection.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.dbError import DbError
+from dmMongoCollection import DmMongoCollection
+
+class ExperimentCollection(DmMongoCollection):
+    """Class responsible for updating experiment collection in mongo db."""
+
+    UNIQUE_KEYS_LIST = [ '_id', 'name' ]
+
+    def __init__(self, dbClient):
+        DmMongoCollection.__init__(self, 'experiments', dbClient)
+
+#######################################################################
+# Testing
+if __name__ == '__main__':
+    from dmMongoClient import DmMongoClient
+    mongo = DmMongoClient('dm')
+    experimentCollection = ExperimentCollection(mongo)
+    experimentInfo = {'name' : 'exp-001', 'owner' : 'sv'}
+    #print experimentCollection.add(experimentInfo)
+    print experimentCollection.updateByName(experimentInfo)
+    print experimentCollection.findByName('exp-001')
+    print experimentCollection.findByQueryDict({'experiment' : 'exp-001'}, {'owner' : 1})
+
+
+

src/python/dm/common/mongodb/impl/fileCollection.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.dbError import DbError
+from dmMongoCollection import DmMongoCollection
+
+class FileCollection(DmMongoCollection):
+    """Class responsible for updating file collection in mongo db."""
+
+    UNIQUE_KEYS_LIST = [ 'experimentFilePath', 'experimentName' ]
+    NAME_KEY = 'experimentFilePath'
+    ITEM_TYPE = 'file'
+
+    def __init__(self, dbClient, experimentName=None):
+        collectionName = 'files'
+        if experimentName:
+            collectionName = '%s-files' % experimentName
+        DmMongoCollection.__init__(self, collectionName, dbClient)
+
+#######################################################################
+# Testing
+if __name__ == '__main__':
+    from dmMongoClient import DmMongoClient
+    mongo = DmMongoClient('dm')
+    fileCollection = FileCollection(mongo, 'exp01')
+    fileInfo = {'fileName' : 'xyz-001', 'experimentName' : 'exp01', 'update' : 'sv2', 'locationList' : '[/opt/xyz, /data/xyz]', 'experimentFilePath' : 'd1/xyz-001'}
+    print fileCollection.addByUniqueKeys(fileInfo)
+    #print type(fileCollection.findByName('xyz-001'))
+    print fileCollection.findByQueryDict({'experimentName' : 'exp01'}, {'locationList' : 1})
+    fileInfo['owner'] = 'ddm1'
+    print fileCollection.updateByUniqueKeys(fileInfo)
+
+
+

src/python/dm/common/mongodb/impl/mongoDbManager.py

+#!/usr/bin/env python
+
+import threading
+import os.path
+import sqlalchemy 
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import mapper
+from sqlalchemy.orm import relationship
+
+from dm.common.exceptions.commandFailed import CommandFailed
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dmMongoClient import DmMongoClient
+
+class MongoDbManager:
+    """ Singleton class for mongo db management. """
+
+    CONFIG_SECTION_NAME = 'MongoDbManager'
+    MONGO_DB_NAME_KEY = 'mongoDbName'
+    MONGO_DB_USER_KEY = 'mongoDbUser'
+    MONGO_DB_URI_KEY = 'mongoDbUri' 
+    MONGO_DB_PASSWORD_FILE_KEY = 'mongoDbPasswordFile' 
+    CONFIG_OPTION_NAME_LIST = [ 'dbName', 'dbUser', 'dbPasswordFile' ]
+
+    # Singleton.
+    __lock = threading.RLock()
+    __instance = None
+
+    @classmethod
+    def getInstance(cls):
+        from dm.common.mongodb.impl.mongoDbManager import MongoDbManager
+        try:
+            mgr = MongoDbManager()
+        except MongoDbManager, ex:
+            mgr = ex
+        return mgr 
+
+    def __init__(self):
+        MongoDbManager.__lock.acquire()
+        try:
+            if MongoDbManager.__instance is not None:
+                raise MongoDbManager.__instance
+            MongoDbManager.__instance = self
+            self.lock = threading.RLock()
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+            cm = ConfigurationManager.getInstance()
+            self.dbName = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_NAME_KEY)
+            self.logger.debug('Mongo DB name: %s' % self.dbName)
+            self.dbUri = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_URI_KEY)
+            self.logger.debug('Mongo DB URI: %s' % self.dbUri)
+            self.dbUser = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_USER_KEY)
+            self.logger.debug('Mongo DB user: %s' % self.dbUser)
+            self.dbPasswordFile = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_PASSWORD_FILE_KEY)
+            self.logger.debug('Mongo DB password file: %s' % self.dbPasswordFile)
+            #dbPassword = open(dbPasswordFile, 'r').readline().strip()
+
+        finally:
+            MongoDbManager.__lock.release()
+
+    def getLogger(self):
+        return self.logger
+
+    def getDbClient(self):
+        return DmMongoClient(self.dbName, self.dbUri)
+
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    ConfigurationManager.getInstance().setConsoleLogLevel('debug')
+    mgr = MongoDbManager.getInstance()
+
+
+

src/python/dm/common/mongodb/impl/processingJobCollection.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.dbError import DbError
+from dmMongoCollection import DmMongoCollection
+
+class ProcessingJobCollection(DmMongoCollection):
+    """Class responsible for updating processing job collection in mongo db."""
+
+    UNIQUE_KEYS_LIST = [ 'workflowId', 'timeStamp' ]
+    NAME_KEY = 'name'
+    ITEM_TYPE = 'processingJob'
+
+    def __init__(self, dbClient, prefix=None):
+        collectionName = 'processing-jobs'
+        if prefix:
+            collectionName = '%s-processing-jobs' % prefix
+        DmMongoCollection.__init__(self, collectionName, dbClient)
+
+#######################################################################
+# Testing
+if __name__ == '__main__':
+    from dmMongoClient import DmMongoClient
+    mongo = DmMongoClient('dm')
+    processingJobCollection = ProcessingJobCollection(mongo)
+    workflowInfo = {'name' : 'workflow-01', 'owner' : 'sveseli', 'stages' : 
+        [ 
+            { 'id' : '1', 'executable' : '/bin/date', 'args' : [ ] }, 
+            { 'id' : '2', 'executable' : '/bin/ls', 'args' : [ '-l', 'INPUT_FILE' ], 'parallelizeExecution' : False},
+            { 'id' : '3', 'executable' : '/bin/cp', 'args' : [ 'INPUT_FILE', 'OUTPUT_FILE=INPUT_FILE.processed' ], 'parallelizeExecution' : False} 
+        ]
+    }
+    print workflowCollection.addByUniqueKeys(workflowInfo)
+    print workflowCollection.findByQueryDict({'name' : 'workflow-01', 'owner' : 'sveseli'}, {'steps' : 1})
+    workflowInfo['description'] = 'My first workflow'
+    print workflowCollection.updateByUniqueKeys(workflowInfo)
+
+
+

src/python/dm/common/mongodb/impl/workflowCollection.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.dbError import DbError
+from dmMongoCollection import DmMongoCollection
+
+class WorkflowCollection(DmMongoCollection):
+    """Class responsible for updating workflow collection in mongo db."""
+
+    UNIQUE_KEYS_LIST = [ 'name', 'owner' ]
+    NAME_KEY = 'name'
+    ITEM_TYPE = 'workflow'
+
+    def __init__(self, dbClient, prefix=None):
+        collectionName = 'workflows'
+        if prefix:
+            collectionName = '%s-workflows' % prefix
+        DmMongoCollection.__init__(self, collectionName, dbClient)
+
+#######################################################################
+# Testing
+if __name__ == '__main__':
+    from dmMongoClient import DmMongoClient
+    mongo = DmMongoClient('dm')
+    workflowCollection = WorkflowCollection(mongo)
+    workflowInfo = {'name' : 'workflow-01', 'owner' : 'sveseli', 'stages' : 
+        [ 
+            { 'id' : '1', 'executable' : '/bin/date', 'args' : [ ] }, 
+            { 'id' : '2', 'executable' : '/bin/ls', 'args' : [ '-l', 'INPUT_FILE' ], 'parallelizeExecution' : False},
+            { 'id' : '3', 'executable' : '/bin/cp', 'args' : [ 'INPUT_FILE', 'OUTPUT_FILE=INPUT_FILE.processed' ], 'parallelizeExecution' : False} 
+        ]
+    }
+    print workflowCollection.addByUniqueKeys(workflowInfo)
+    print workflowCollection.findByQueryDict({'name' : 'workflow-01', 'owner' : 'sveseli'}, {'steps' : 1})
+    workflowInfo['description'] = 'My first workflow'
+    print workflowCollection.updateByUniqueKeys(workflowInfo)
+
+
+

src/python/dm/common/objects/allowedExperimentStationExperimentType.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class AllowedExperimentStationExperimentType(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'experimentStationId', 'experimentTypeId' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/apsUserInfo.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class ApsUserInfo(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'badgeNumber', 'firstName', 'middleName', 'lastName', 'email', 'passwordHashValue' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+

src/python/dm/common/objects/authorizationPrincipal.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class AuthorizationPrincipal(DmObject):
+
+    def __init__(self, dict={}, name=None, token=None, userInfo={}):
+        DmObject.__init__(self, dict)
+        if name is not None:
+            self['name'] = name
+        if token is not None:
+            self['token'] = token
+        if userInfo is not None and len(userInfo):
+            self['userInfo'] = userInfo
+
+    def getName(self):
+        return self.get('name')
+
+    def getAuthenticationToken(self):
+        return self.get('token')
+
+    def getToken(self):
+        return self.get('token')
+
+    def setSessionRole(self, role):
+        self['sessionRole'] = role
+
+    def getSessionRole(self):
+        return self.get('sessionRole')
+
+    def setUserInfo(self, userInfo):
+        self['userInfo'] = userInfo
+
+    def getUserInfo(self):
+        return self.get('userInfo')
+
+    def setUserSystemRoleDict(self, userSystemRoleDict):
+        self['userSystemRoleDict'] = userSystemRoleDict
+
+    def getUserSystemRoleDict(self):
+        return self.get('userSystemRoleDict')
+
+    def setUserExperimentRoleDict(self, userExperimentRoleDict):
+        self['userExperimentRoleDict'] = userExperimentRoleDict
+
+    def getUserExperimentRoleDict(self):
+        return self.get('userExperimentRoleDict')
+

src/python/dm/common/objects/beamlineInfo.py

+#!/usr/bin/env python
+
+from dmObject import DmObject
+
+class BeamlineInfo(DmObject):
+
+    DEFAULT_KEY_LIST = [ 'id', 'name' ]
+
+    def __init__(self, dict):
+        DmObject.__init__(self, dict)
+