Skip to content
Snippets Groups Projects
Commit c3353915 authored by sveseli's avatar sveseli
Browse files

added mongodb client, base collection class, and several concrete collections

parent 16f1363b
No related branches found
No related tags found
No related merge requests found
Showing with 601 additions and 0 deletions
#!/usr/bin/env python
from dm.common.exceptions.dmException import DmException
from dm.common.utility.loggingManager import LoggingManager
from dm.common.mongodb.impl.mongoDbManager import MongoDbManager
class DmMongoDbApi:
""" Base Mongo DB API class. """
def __init__(self):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
self.dbClient = MongoDbManager.getInstance().getDbClient()
# Decorator for all DB methods
@classmethod
def executeDbCall(cls, func):
def dbCall(*args, **kwargs):
try:
try:
return func(*args, **kwargs)
except DmException, ex:
raise
except Exception, ex:
cls.getLogger().exception('%s' % ex)
raise DmException(exception=ex)
finally:
# For now, do nothing
pass
return dbCall
@classmethod
def getLogger(cls):
logger = LoggingManager.getInstance().getLogger(cls.__name__)
return logger
@classmethod
def listToDmObjects(cls, mongoDbObjectList, dmObjectClass):
dmObjectList = []
for o in mongoDbObjectList:
dmObjectList.append(cls.toDmObject(o, dmObjectClass))
return dmObjectList
@classmethod
def toDmObject(cls, mongoDbObject, dmObjectClass):
cls.scrubMongoDbObject(mongoDbObject)
return dmObjectClass(mongoDbObject)
@classmethod
def scrubMongoDbObject(cls, mongoDbObject):
for key in ['_id']:
if mongoDbObject.has_key(key):
# Remove leading underscore
newKey = key[1:]
mongoDbObject[newKey] = str(mongoDbObject[key])
del mongoDbObject[key]
#######################################################################
# Testing.
if __name__ == '__main__':
api = DmMongoDbApi()
#!/usr/bin/env python
import copy
from dm.common.exceptions.dmException import DmException
from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
from dm.common.mongodb.impl.fileCollection import FileCollection
from dm.common.mongodb.api.dmMongoDbApi import DmMongoDbApi
from dm.common.objects.fileMetadata import FileMetadata
class FileMongoDbApi(DmMongoDbApi):
def __init__(self):
DmMongoDbApi.__init__(self)
self.fileCollection = FileCollection(self.dbClient)
@DmMongoDbApi.executeDbCall
def addExperimentFile(self, fileInfo, **kwargs):
dbFileMetadata = self.fileCollection.addByUniqueKeys(fileInfo)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def getFiles(self, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
return self.listToDmObjects(self.fileCollection.findByQueryDict(queryDict, returnFieldDict), FileMetadata)
@DmMongoDbApi.executeDbCall
def getFileById(self, id, **kwargs):
dbFileMetadata = self.fileCollection.findById(id)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def getExperimentFile(self, experimentName, fileName, **kwargs):
queryDict = { 'name' : fileName, 'experimentName' : experimentName }
dbFileMetadata = self.fileCollection.findByUniqueKeys(queryDict)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def getExperimentFiles(self, experimentName, queryDict={}, returnFieldDict=FileCollection.ALL_FIELDS_DICT, **kwargs):
queryDict2 = copy.copy(queryDict)
queryDict2['experimentName'] = experimentName
return self.getFiles(queryDict2, returnFieldDict)
@DmMongoDbApi.executeDbCall
def updateFileById(self, fileInfo, **kwargs):
dbFileMetadata = self.fileCollection.updateById(fileInfo)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def updateExperimentFile(self, fileInfo, **kwargs):
dbFileMetadata = self.fileCollection.updateByUniqueKeys(fileInfo)
return self.toDmObject(dbFileMetadata, FileMetadata)
@DmMongoDbApi.executeDbCall
def updateOrAddExperimentFile(self, fileInfo, **kwargs):
dbFileMetadata = self.fileCollection.updateOrAddByUniqueKeys(fileInfo)
return self.toDmObject(dbFileMetadata, FileMetadata)
#######################################################################
# Testing.
if __name__ == '__main__':
api = FileMongoDbApi()
files = api.getFiles()
for file in files:
print file.getDictRep()
print file.__dict__
file = api.getExperimentFile('exp-01', 'xyz-001')
print file
import time
t = long(time.time())
print t
fileName = 'f-%s' % t
fileInfo = {'name' : fileName, 'intKey' : 1, 'doubleKey' : 2.0, 'stringKey' : 'myString' , 'dictKey' : {'a' : 'A', 'b' : 'B', 'c' : 3}, 'experimentName' : 'exp2'}
file = api.addExperimentFile(fileInfo)
print '\nADDED FILE\n', file
fileInfo = {'name' : fileName, 'intKey' : 101}
file = api.updateExperimentFile(fileInfo)
print '\nUPDATED FILE\n', file
print '\nFILES: \n', api.getFiles()
print '\nFILES FOR EXPERIMENT exp1: \n', api.getFiles(queryDict={'experimentName' : 'exp1'})
#!/usr/bin/env python
from dm.common.utility.loggingManager import LoggingManager
from dm.common.exceptions.invalidArgument import InvalidArgument
from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
from dm.common.exceptions.objectNotFound import ObjectNotFound
from dm.common.exceptions.dbError import DbError
from dmMongoCollection import DmMongoCollection
class DatasetCollection(DmMongoCollection):
"""Class responsible for updating dataset collection in mongo db."""
UNIQUE_KEYS_LIST = [ 'name' ]
def __init__(self, dbClient):
DmMongoCollection.__init__(self, 'datasets', dbClient)
#######################################################################
# Testing
if __name__ == '__main__':
from dmMongoClient import DmMongoClient
mongo = DmMongoClient('dm')
datasetCollection = DatasetCollection(mongo)
datasetInfo = {'name' : 'ds-001',
'owner' : 'sv',
'experiment' : 'exp-001',
'voltage' : { 'gt' : 400},
'current' : { 'lt' : 100},
}
#print datasetCollection.add(datasetInfo)
print datasetCollection.updateByName(datasetInfo)
print datasetCollection.findByName('ds-001')
print datasetCollection.findByQueryDict({'dataset' : 'ds-001'}, {'owner' : 1})
#!/usr/bin/env python
from pymongo import MongoClient
from dm.common.utility.loggingManager import LoggingManager
from dm.common.exceptions.dbError import DbError
class DmMongoClient(object):
""" DM MongoDB client."""
def __init__(self, dbName, dbUri='mongodb://localhost:27017/'):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
try:
self.client = MongoClient(dbUri)
self.db = self.client[dbName]
self.dbName = dbName
except Exception, ex:
self.logger.error('Cannot connect to Mongo DB: %s' % ex)
raise DbError(exception=ex)
def drop(self, collection):
return self.db[collection].drop()
def insert(self, collection, dict, **kwargs):
return self.db[collection].insert(dict, **kwargs)
def update(self, collection, query, update, **kwargs):
key = '.'.join((collection, str(update['$set'].keys()[0])))
return self.db[collection].update(query, update, **kwargs)
def findOne(self, collection, criteria={}):
return self.db[collection].find_one(criteria)
def find(self, collection, criteria={}, projections={}):
return self.db[collection].find(criteria, projections)
def findAsList(self, collection, criteria={}, projections={}):
return list(self.db[collection].find(criteria, projections))
def getCollectionNames(self):
return self.db.collection_names()
#######################################################################
# Testing
if __name__ == '__main__':
mongo = DmMongoClient('dm')
mongo.drop('students')
id = mongo.insert('students', { '_id' : 1, 'semester' : 1, 'grades' : [ 70, 87, 90 ] })
print 'Student #1 id: ', id
id = mongo.insert('students', { '_id' : 2, 'semester' : 1, 'grades' : [ 90, 88, 92 ] } )
print 'Student #2 id: ', id
print mongo.findAsList('students', criteria={ 'semester' : 1, 'grades': { '$gte' : 85 } }, projections={ 'grades.$' : 1 } )
result = mongo.update('files', {'_id' : 2}, {'$set' : {'experimentId' : 2}}, upsert=True)
print result
doc = mongo.findOne('files', {'experimentId' : 1})
print doc
docs = mongo.find('files', projections={'experimentId' : 1})
print docs.count()
docs = mongo.findAsList('files', projections={'experimentId' : 1})
print docs
print 'Collection names: ', mongo.getCollectionNames()
mongo.drop('students')
print 'Collection names (after drop): ', mongo.getCollectionNames()
#!/usr/bin/env python
from bson.objectid import ObjectId
from dm.common.utility.loggingManager import LoggingManager
from dm.common.exceptions.invalidArgument import InvalidArgument
from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
from dm.common.exceptions.objectNotFound import ObjectNotFound
from dm.common.exceptions.dbError import DbError
class DmMongoCollection(object):
"""Base collection in mongo db."""
ALL_FIELDS_DICT = {'__return_only_id__' : False}
UNIQUE_KEYS_LIST = []
def __init__(self, collectionName, dbClient):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
self.dbClient = dbClient
# Collection consists of items: [item]
self.collectionName = collectionName
# items => item
self.itemName = collectionName[:-1]
# item => Item
self.capitalizedItemName = self.itemName.capitalize()
@classmethod
def getUniqueKeys(cls):
return cls.UNIQUE_KEYS_LIST
def findByKey(self, key, value):
dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
if dbObjectDict is None:
raise ObjectNotFound('%s with %s=%s not found.' % (self.capitalizedItemName, key, value))
return dbObjectDict
def findByKeys(self, keyList, queryDict):
for key in keyList:
if not queryDict.has_key(key):
raise InvalidArgument('%s query dictionary does not specify key %s.' % (self.capitalizedItemName, key))
dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
if dbObjectDict is None:
raise ObjectNotFound('%s with properties %s not found.' % (self.capitalizedItemName, queryDict))
return dbObjectDict
def findByUniqueKeys(self, queryDict):
return self.findByKeys(self.UNIQUE_KEYS_LIST, queryDict)
def findById(self, id):
return self.findByKey('_id', ObjectId(id))
def findByName(self, name):
return self.findByKey('name', name)
def findByQueryDict(self, queryDict, returnFieldDict=ALL_FIELDS_DICT):
return self.dbClient.findAsList(self.collectionName, queryDict, returnFieldDict)
def listByKey(self, key):
return self.dbClient.findAsList(self.collectionName, {}, {key : True})
def listById(self):
return self.dbClient.findAsList(self.collectionName, {}, {})
def listByName(self):
return self.findByName('name')
def listByKeys(self, keyList):
returnFieldDict = {}
for key in keyList:
returnFieldDict[key] = True
return self.dbClient.findAsList(self.collectionName, {}, returnFieldDict)
def listAll(self):
return self.dbClient.findAsList(self.collectionName, {}, DmMongoCollection.ALL_FIELDS_DICT)
def __addDbObject(self, objectDict):
try:
self.dbClient.insert(self.collectionName, objectDict)
except Exception, ex:
self.logger.error('Cannot add %s with %s set to %s: %s' % (self.itemName, key, value, ex))
raise DbError(exception=ex)
return objectDict
def addByKey(self, key, objectDict):
value = objectDict.get(key)
if value is None:
raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
if dbObjectDict is not None:
raise ObjectAlreadyExists('%s with %s set to %s already exists.' % (self.capitalizedItemName, key, value))
return self.__addDbObject(objectDict)
def addByName(self, objectDict):
return self.addByKey('name', objectDict)
def addByKeys(self, keyList, objectDict):
queryDict = {}
for key in keyList:
value = objectDict.get(key)
if value is None:
raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
queryDict[key] = value
dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
if dbObjectDict is not None:
raise ObjectAlreadyExists('%s with properties %s already exists.' % (self.capitalizedItemName, queryDict))
return self.__addDbObject(objectDict)
def addByUniqueKeys(self, objectDict):
return self.addByKeys(self.UNIQUE_KEYS_LIST, objectDict)
def __updateDbObject(self, dbObjectDict, objectDict):
try:
id = dbObjectDict.get('_id')
dbObjectDict.update(objectDict)
self.dbClient.update(self.collectionName,
{'_id' : id},
{'$set' : dbObjectDict})
except Exception, ex:
self.logger.error('Cannot update %s %s with %s: %s' % (self.itemName, dbObjectDict, objectDict, ex))
raise DbError(exception=ex)
return dbObjectDict
def updateByKey(self, key, objectDict):
value = objectDict.get(key)
if value is None:
raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
if dbObjectDict is None:
raise ObjectNotFound('%s with %s set to %s not found.' % (self.capitalizedItemName, key, value))
return self.__updateDbObject(dbObjectDict, objectDict)
def updateByKeys(self, keyList, objectDict):
queryDict = {}
for key in keyList:
value = objectDict.get(key)
if value is None:
raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
queryDict[key] = value
dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
if dbObjectDict is None:
raise ObjectNotFound('%s with properties %s not found.' % (self.capitalizedItemName, queryDict))
return self.__updateDbObject(dbObjectDict, objectDict)
def updateByUniqueKeys(self, objectDict):
return self.updateByKeys(self.UNIQUE_KEYS_LIST, objectDict)
def updateById(self, objectDict):
# Convert 'id' to '_id' key if needed, and wrap it with ObjectId()
if not objectDict.has_key('_id') and objectDict.has_key('id'):
objectDict['_id'] = ObjectId(objectDict['id'])
del objectDict['id']
return self.updateByKey('_id', objectDict)
def updateByName(self, objectDict):
return self.updateByKey('name', objectDict)
def updateOrAddByKey(self, key, objectDict):
value = objectDict.get(key)
if value is None:
raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
dbObjectDict = self.dbClient.findOne(self.collectionName, {key : value})
if dbObjectDict is None:
return self.__addDbObject(objectDict)
else:
return self.__updateDbObject(dbObjectDict, objectDict)
def updateOrAddByKeys(self, keyList, objectDict):
queryDict = {}
for key in keyList:
value = objectDict.get(key)
if value is None:
raise InvalidArgument('%s info dictionary does not specify key %s.' % (self.capitalizedItemName, key))
queryDict[key] = value
dbObjectDict = self.dbClient.findOne(self.collectionName, queryDict)
if dbObjectDict is None:
return self.__addDbObject(objectDict)
else:
return self.__updateDbObject(dbObjectDict, objectDict)
def updateOrAddByUniqueKeys(self, objectDict):
return self.updateOrAddByKeys(self.UNIQUE_KEYS_LIST, objectDict)
def updateOrAddByName(self, objectDict):
return self.updateOrAddByKey('name', objectDict)
#######################################################################
# Testing
if __name__ == '__main__':
from dmMongoClient import DmMongoClient
from bson.objectid import ObjectId
mongo = DmMongoClient('dm')
fileCollectionImpl = DmMongoCollection('files', mongo)
#objectDict = {'name' : 'xyz-001', 'experiment' : 'myexp-D', 'update' : 'sv2', 'locationList' : '[/opt/xyz, /data/xyz]'}
#print fileCollectionImpl.updateOrAddByKey('name', objectDict)
#f = fileCollectionImpl.findByKey('name', 'xyz-001')
#print f
#print type(f['_id'])
#print fileCollectionImpl.findById('556de0059e058b0ef4c4413b')
#print fileCollectionImpl.findByQueryDict({'experiment' : 'exp-001'}, {'locationList' : 1})
#print 'LIST BY ID\n', fileCollectionImpl.listById()
#print 'LIST BY NAME\n', fileCollectionImpl.listByKey('name')
print 'LIST ALL\n'
for f in fileCollectionImpl.listAll():
print 'FILE: %s\n' % f
#print fileCollectionImpl.updateByKeys(['name', 'experiment'], objectDict)
#!/usr/bin/env python
from dm.common.utility.loggingManager import LoggingManager
from dm.common.exceptions.invalidArgument import InvalidArgument
from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
from dm.common.exceptions.objectNotFound import ObjectNotFound
from dm.common.exceptions.dbError import DbError
from dmMongoCollection import DmMongoCollection
class ExperimentCollection(DmMongoCollection):
"""Class responsible for updating experiment collection in mongo db."""
UNIQUE_KEYS_LIST = [ 'name' ]
def __init__(self, dbClient):
DmMongoCollection.__init__(self, 'experiments', dbClient)
#######################################################################
# Testing
if __name__ == '__main__':
from dmMongoClient import DmMongoClient
mongo = DmMongoClient('dm')
experimentCollection = ExperimentCollection(mongo)
experimentInfo = {'name' : 'exp-001', 'owner' : 'sv'}
#print experimentCollection.add(experimentInfo)
print experimentCollection.updateByName(experimentInfo)
print experimentCollection.findByName('exp-001')
print experimentCollection.findByQueryDict({'experiment' : 'exp-001'}, {'owner' : 1})
#!/usr/bin/env python
from dm.common.utility.loggingManager import LoggingManager
from dm.common.exceptions.invalidArgument import InvalidArgument
from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
from dm.common.exceptions.objectNotFound import ObjectNotFound
from dm.common.exceptions.dbError import DbError
from dmMongoCollection import DmMongoCollection
class FileCollection(DmMongoCollection):
"""Class responsible for updating file collection in mongo db."""
UNIQUE_KEYS_LIST = [ 'name', 'experimentName' ]
def __init__(self, dbClient):
DmMongoCollection.__init__(self, 'files', dbClient)
#######################################################################
# Testing
if __name__ == '__main__':
from dmMongoClient import DmMongoClient
mongo = DmMongoClient('dm')
fileCollection = FileCollection(mongo)
fileInfo = {'name' : 'xyz-001', 'experimentName' : 'myexp-001', 'update' : 'sv2', 'locationList' : '[/opt/xyz, /data/xyz]'}
print fileCollection.updateByName(fileInfo)
print type(fileCollection.findByName('xyz-001'))
print fileCollection.findByQueryDict({'experiment' : 'exp-001'}, {'locationList' : 1})
fileInfo['experimentName'] = 'ddm1'
print fileCollection.addByUniqueKeys(fileInfo)
#!/usr/bin/env python
import threading
import os.path
import sqlalchemy
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import mapper
from sqlalchemy.orm import relationship
from dm.common.exceptions.commandFailed import CommandFailed
from dm.common.exceptions.configurationError import ConfigurationError
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.configurationManager import ConfigurationManager
from dmMongoClient import DmMongoClient
class MongoDbManager:
""" Singleton class for mongo db management. """
CONFIG_SECTION_NAME = 'MongoDbManager'
MONGO_DB_NAME_KEY = 'mongoDbName'
MONGO_DB_USER_KEY = 'mongoDbUser'
MONGO_DB_URI_KEY = 'mongoDbUri'
MONGO_DB_PASSWORD_FILE_KEY = 'mongoDbPasswordFile'
CONFIG_OPTION_NAME_LIST = [ 'dbName', 'dbUser', 'dbPasswordFile' ]
# Singleton.
__lock = threading.RLock()
__instance = None
@classmethod
def getInstance(cls):
from dm.common.mongodb.impl.mongoDbManager import MongoDbManager
try:
mgr = MongoDbManager()
except MongoDbManager, ex:
mgr = ex
return mgr
def __init__(self):
MongoDbManager.__lock.acquire()
try:
if MongoDbManager.__instance is not None:
raise MongoDbManager.__instance
MongoDbManager.__instance = self
self.lock = threading.RLock()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
cm = ConfigurationManager.getInstance()
self.dbName = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_NAME_KEY)
self.logger.debug('Mongo DB name: %s' % self.dbName)
self.dbUri = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_URI_KEY)
self.logger.debug('Mongo DB URI: %s' % self.dbUri)
self.dbUser = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_USER_KEY)
self.logger.debug('Mongo DB user: %s' % self.dbUser)
self.dbPasswordFile = cm.getConfigOption(MongoDbManager.CONFIG_SECTION_NAME, MongoDbManager.MONGO_DB_PASSWORD_FILE_KEY)
self.logger.debug('Mongo DB password file: %s' % self.dbPasswordFile)
#dbPassword = open(dbPasswordFile, 'r').readline().strip()
finally:
MongoDbManager.__lock.release()
def getLogger(self):
return self.logger
def getDbClient(self):
return DmMongoClient(self.dbName, self.dbUri)
#######################################################################
# Testing.
if __name__ == '__main__':
ConfigurationManager.getInstance().setConsoleLogLevel('debug')
mgr = MongoDbManager.getInstance()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment