Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • DM/dm-docs
  • hammonds/dm-docs
  • hparraga/dm-docs
3 results
Show changes
Showing
with 1336 additions and 0 deletions
#!/usr/bin/env python
import os
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.osUtility import OsUtility
from dm.common.utility.fileUtility import FileUtility
from dm.common.exceptions.fileProcessingError import FileProcessingError
from dm.common.processing.plugins.fileTransferPlugin import FileTransferPlugin
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
class RsyncWithChecksumAndDeleteFileTransferPlugin(FileTransferPlugin):
COMMAND = 'rsync -arvlPR'
def __init__(self, src=None, dest=None):
FileTransferPlugin.__init__(self, self.COMMAND, src, dest)
self.dsFileApi = DsRestApiFactory.getFileRestApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
filePath = fileInfo.get('filePath')
dataDirectory = fileInfo.get('dataDirectory')
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
storageHost = fileInfo.get('storageHost')
storageDirectory = fileInfo.get('storageDirectory')
destUrl = self.getDestUrl(filePath, dataDirectory, storageHost, storageDirectory)
# Use relative path with respect to data directory as a source
srcUrl = self.getSrcUrl(filePath, dataDirectory)
# Calculate checksum
FileUtility.statFile(filePath, fileInfo)
FileUtility.getMd5Sum(filePath, fileInfo)
self.logger.debug('File info before transfer: %s' % fileInfo)
# Transfer file
self.start(src=srcUrl, dest=destUrl, templateInfo=fileInfo, cwd=dataDirectory)
# Get remote checksum
fileInfo2 = {}
fileInfo2['experimentFilePath'] = experimentFilePath
fileInfo2['experimentName'] = experimentName
fileMetadata = self.dsFileApi.statFile(experimentFilePath, experimentName, fileInfo2)
self.logger.debug('File stat after transfer: %s' % fileMetadata)
# Verify checksum
if fileMetadata.get('md5Sum') != fileInfo.get('md5Sum'):
self.logger.error('Checksum mismatch for file: %s' % filePath)
raise FileProcessingError('Checksum mismatch detected for file: %s' % filePath)
# Remove file
self.logger.debug('Checksum test passed, deleting file %s' % filePath)
OsUtility.removeFile(srcUrl)
#######################################################################
# Testing.
if __name__ == '__main__':
ft = RsyncWithChecksumAndDeleteFileTransferPlugin('/tmp/xyz', '/tmp/xyz2')
ft.start()
print 'StdOut: ', ft.getStdOut()
print 'StdErr: ', ft.getStdErr()
print 'Exit Status: ', ft.getExitStatus()
#!/usr/bin/env python
import os
import time
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi
from dm.common.utility.dmSubprocess import DmSubprocess
class ScriptProcessingPlugin(FileProcessor):
PROCESSING_SCRIPT_KEY = 'processingScript'
def __init__(self):
FileProcessor.__init__(self)
self.fileMongoDbApi = FileMongoDbApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
experimentName = fileInfo.get('experimentName')
experimentFilePath = fileInfo.get('experimentFilePath')
daqInfo = fileInfo.get('daqInfo', {})
processingScript = daqInfo.get(self.PROCESSING_SCRIPT_KEY)
if not processingScript:
self.logger.debug('Ignoring file %s for experiment %s' % (experimentFilePath, experimentName))
return
self.logger.debug('Processing file %s for experiment %s' % (experimentFilePath, experimentName))
storageDirectory = fileInfo.get('storageDirectory')
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Process file
try:
p = DmSubprocess('%s %s' % (processingScript, storageFilePath))
p.run()
stdOut = p.getStdOut()
except Exception, ex:
self.logger.error('Cannot process file %s for experiment %s: %s' % (experimentFilePath, experimentName, ex))
return
# Prepare catalogging entry
fileInfo2 = {}
fileInfo2['fileName'] = os.path.basename(experimentFilePath)
fileInfo2['experimentName'] = experimentName
fileInfo2['processingScriptOutput'] = '%s' % stdOut.strip()
self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
import time
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi
class SddsParameterProcessingPlugin(FileProcessor):
PROCESS_SDDS_PARAMETERS_KEY = 'processSddsParameters'
def __init__(self):
FileProcessor.__init__(self)
self.fileMongoDbApi = FileMongoDbApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
daqInfo = fileInfo.get('daqInfo', {})
processSddsParameters = daqInfo.get(self.PROCESS_SDDS_PARAMETERS_KEY)
if not processSddsParameters:
self.logger.debug('Ignoring file %s for experiment %s' % (experimentFilePath, experimentName))
return
self.logger.debug('Processing file %s for experiment %s' % (fileInfo, experimentName))
storageDirectory = fileInfo.get('storageDirectory')
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Load file
try:
import sddsdata
from sdds import SDDS
s = SDDS(0)
self.logger.error('Loading file %s for experiment %s' % (experimentFilePath, experimentName))
s.load(storageFilePath)
except Exception, ex:
self.logger.error('Cannot process file %s for experiment %s: %s' % (experimentFilePath, experimentName, ex))
return
# Prepare catalogging entry
fileInfo2 = {}
fileInfo2['fileName'] = os.path.basename(experimentFilePath)
fileInfo2['experimentName'] = experimentName
for i in range(0,len(s.parameterName)):
parameterName = s.parameterName[i]
parameterData = s.parameterData[i][0]
fileInfo2[parameterName] = parameterData
self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)
self.logger.error('SDDS terminate file %s for experiment %s' % (experimentFilePath, experimentName))
sddsdata.Terminate(0)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
import time
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.utility.dmSubprocess import DmSubprocess
class SgeJobSubmissionPlugin(FileProcessor):
SGE_JOB_SCRIPT_KEY = 'sgeJobScript'
def __init__(self, sgeRoot):
FileProcessor.__init__(self)
self.sgeRoot = sgeRoot
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
experimentName = fileInfo.get('experimentName')
experimentFilePath = fileInfo.get('experimentFilePath')
daqInfo = fileInfo.get('daqInfo', {})
sgeJobScript = daqInfo.get(self.SGE_JOB_SCRIPT_KEY)
if not sgeJobScript:
self.logger.debug('Ignoring file %s for experiment %s' % (experimentFilePath, experimentName))
return
self.logger.debug('Processing file %s for experiment %s' % (experimentFilePath, experimentName))
storageDirectory = fileInfo.get('storageDirectory')
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Process file
try:
p = DmSubprocess('. %s/default/common/settings.sh ; qsub -v DM_EXPERIMENT_NAME=%s %s %s' % (self.sgeRoot, experimentName, sgeJobScript, storageFilePath))
p.run()
except Exception, ex:
self.logger.error('Cannot process file %s for experiment %s: %s' % (experimentFilePath, experimentName, ex))
return
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.loggingManager import LoggingManager
class AuthorizationPrincipalAuthenticator:
def __init__(self, name=None):
self.name = name
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def getName(self):
return self.name
def authenticatePrincipal(self, principal, password):
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
from dm.common.exceptions.authorizationError import AuthorizationError
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.utility.objectCache import ObjectCache
from dm.common.utility.cryptUtility import CryptUtility
from dm.common.utility.objectUtility import ObjectUtility
from dm.common.utility.singleton import Singleton
class AuthorizationPrincipalManager(DmObjectManager, Singleton):
DEFAULT_CACHE_SIZE = 10000 # number of items
DEFAULT_CACHE_OBJECT_LIFETIME = 3600 # seconds
CONFIG_SECTION_NAME = 'AuthorizationPrincipalManager'
PRINCIPAL_RETRIEVER_KEY = 'principalretriever'
PRINCIPAL_AUTHENTICATOR_KEY = 'principalauthenticator'
# Singleton instance.
__instance = None
def __init__(self):
if AuthorizationPrincipalManager.__instance:
return
AuthorizationPrincipalManager.__instance = self
DmObjectManager.__init__(self)
self.configurationManager = ConfigurationManager.getInstance()
self.principalRetriever = None
self.principalAuthenticatorList = []
self.objectCache = ObjectCache(AuthorizationPrincipalManager.DEFAULT_CACHE_SIZE, AuthorizationPrincipalManager.DEFAULT_CACHE_OBJECT_LIFETIME)
self.configure()
@classmethod
def cryptPassword(cls, cleartext):
return CryptUtility.cryptPassword(cleartext)
@classmethod
def cryptPasswordWithPbkdf2(cls, cleartext):
return CryptUtility.cryptPasswordWithPbkdf2(cleartext)
def configure(self):
configItems = self.configurationManager.getConfigItems(AuthorizationPrincipalManager.CONFIG_SECTION_NAME)
self.logger.debug('Got config items: %s' % configItems)
# Create principal retriever
principalRetriever = self.configurationManager.getConfigOption(AuthorizationPrincipalManager.CONFIG_SECTION_NAME, AuthorizationPrincipalManager.PRINCIPAL_RETRIEVER_KEY)
(moduleName,className,constructor) = self.configurationManager.getModuleClassConstructorTuple(principalRetriever, AuthorizationPrincipalManager)
self.logger.debug('Creating principal retriever class: %s' % className)
self.principalRetriever = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.logger.debug('Authorization principal retriever: %s' % (self.principalRetriever))
# Create principal authenticators
for (key,value) in configItems:
if key.startswith(AuthorizationPrincipalManager.PRINCIPAL_AUTHENTICATOR_KEY):
(moduleName,className,constructor) = self.configurationManager.getModuleClassConstructorTuple(value, AuthorizationPrincipalManager)
self.logger.debug('Creating principal authenticator class: %s' % className)
principalAuthenticator = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.addAuthorizationPrincipalAuthenticator(principalAuthenticator)
self.logger.debug('Authorization principal authenticator: %s' % (principalAuthenticator))
def addAuthorizationPrincipalAuthenticator(self, principalAuthenticator):
self.principalAuthenticatorList.append(principalAuthenticator)
def getAuthenticatedAuthorizationPrincipal(self, username, password):
""" Get principal based on a username and password """
# First try cache.
#self.logger.debug('Trying username %s from the cache' % username)
principal = None
principal = self.objectCache.get(username)
if principal is None:
# Try principal retriever
principal = self.principalRetriever.getAuthorizationPrincipal(username)
if principal is None:
self.logger.debug('No principal for username: %s' % username)
return
# Try all authorization principal authenticators.
for principalAuthenticator in self.principalAuthenticatorList:
self.logger.debug('Attempting to authenticate %s by %s' % (username, principalAuthenticator.getName()))
authenticatedPrincipal = principalAuthenticator.authenticatePrincipal(principal, password)
if authenticatedPrincipal is not None:
self.logger.debug('Adding authorization principal %s to the cache, authenticated by %s' % (principal.getName(),principalAuthenticator.getName()))
self.objectCache.put(username, authenticatedPrincipal)
return authenticatedPrincipal
return None
def removeAuthorizationPrincipal(self, username):
""" Clear principal from the cache. """
self.objectCache.remove(username)
#######################################################################
# Testing.
if __name__ == '__main__':
am = AuthorizationPrincipalManager.getInstance()
authPrincipal = am.getAuthorizationPrincipal('sveseli', 'sv')
print 'Auth principal: ', authPrincipal
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.utility.loggingManager import LoggingManager
class AuthorizationPrincipalRetriever:
def __init__(self, name=None):
self.name = name
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def getName(self):
return self.name
def getAuthorizationPrincipal(self, username):
return None
def setAuthorizationPrincipalSessionRole(self, principal):
if principal is None:
return
for userSystemRoleId in principal.get('userSystemRoleDict', {}).keys():
if userSystemRoleId == dmRole.DM_ADMIN_SYSTEM_ROLE_ID:
principal.setSessionRole(dmRole.DM_ADMIN_SESSION_ROLE)
return
principal.setSessionRole(dmRole.DM_USER_SESSION_ROLE)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.cryptUtility import CryptUtility
from authorizationPrincipalAuthenticator import AuthorizationPrincipalAuthenticator
class CryptedPasswordPrincipalAuthenticator(AuthorizationPrincipalAuthenticator):
def __init__(self):
AuthorizationPrincipalAuthenticator.__init__(self, self.__class__.__name__)
def authenticatePrincipal(self, principal, password):
if principal is not None:
principalToken = principal.getToken()
if principalToken is not None and len(principalToken):
if CryptUtility.verifyPasswordWithPbkdf2(password, principalToken):
self.logger.debug('Authentication successful for %s' % principal.getName())
return principal
else:
self.logger.debug('Authentication failed for %s' % principal.getName())
else:
self.logger.debug('Token is empty for %s, authentication not performed' % principal.getName())
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.db.api.userDbApi import UserDbApi
from authorizationPrincipalRetriever import AuthorizationPrincipalRetriever
class DbPrincipalRetriever(AuthorizationPrincipalRetriever):
def __init__(self):
AuthorizationPrincipalRetriever.__init__(self, self.__class__.__name__)
self.dbApi = UserDbApi()
def getAuthorizationPrincipal(self, username):
principal = None
try:
user = self.dbApi.getUserWithPasswordByUsername(username)
principal = AuthorizationPrincipal(name=username, token=user.get('password'))
principal.setUserSystemRoleDict(user.get('userSystemRoleDict', {}))
principal.setUserExperimentRoleDict(user.get('userExperimentRoleDict', {}))
self.setAuthorizationPrincipalSessionRole(principal)
except Exception, ex:
self.logger.debug(ex)
return principal
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.ldapUtility import LdapUtility
from authorizationPrincipalAuthenticator import AuthorizationPrincipalAuthenticator
class LdapPasswordPrincipalAuthenticator(AuthorizationPrincipalAuthenticator):
def __init__(self, serverUrl, dnFormat):
AuthorizationPrincipalAuthenticator.__init__(self, self.__class__.__name__)
self.ldapUtility = LdapUtility(serverUrl, dnFormat)
def authenticatePrincipal(self, principal, password):
if principal is not None:
try:
self.logger.debug('Checking credentials for %s' % principal.getName())
self.ldapUtility.checkCredentials(principal.getName(), password)
return principal
except Exception, ex:
self.logger.debug(ex)
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.utility.cryptUtility import CryptUtility
from authorizationPrincipalRetriever import AuthorizationPrincipalRetriever
class NoOpPrincipalRetriever(AuthorizationPrincipalRetriever):
def __init__(self):
AuthorizationPrincipalRetriever.__init__(self, self.__class__.__name__)
def getAuthorizationPrincipal(self, username):
# Set password to be the same as username
noOpPassword = CryptUtility.cryptPasswordWithPbkdf2(username)
principal = AuthorizationPrincipal(name=username, token=noOpPassword)
self.setAuthorizationPrincipalSessionRole(principal)
return principal
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.loggingManager import LoggingManager
class SessionManager:
def __init__(self, name=None):
self.name = name
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def getName(self):
return self.name
def addSession(self, sessionId, sessionInfo):
return None
def checkSession(self, sessionId):
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.utility.objectUtility import ObjectUtility
from dm.common.utility.singleton import Singleton
from dm.common.exceptions.objectNotFound import ObjectNotFound
class SingleSignOnManager(DmObjectManager, Singleton):
DEFAULT_SESSION_TIMEOUT_IN_SECONDS = 3600 # seconds
CONFIG_SECTION_NAME = 'SingleSignOnManager'
SESSION_MANAGER_KEY = 'sessionManager'
SESSION_TIMEOUT_IN_SECONDS_KEY = 'sessionTimeoutInSeconds'
# Singleton instance.
__instance = None
def __init__(self):
if SingleSignOnManager.__instance:
return
SingleSignOnManager.__instance = self
DmObjectManager.__init__(self)
self.configurationManager = ConfigurationManager.getInstance()
self.sessionManager = None
self.configure()
def configure(self):
configItems = self.configurationManager.getConfigItems(SingleSignOnManager.CONFIG_SECTION_NAME)
self.logger.debug('Got config items: %s' % configItems)
if not configItems:
return
sessionTimeout = self.configurationManager.getConfigOption(SingleSignOnManager.CONFIG_SECTION_NAME, SingleSignOnManager.SESSION_TIMEOUT_IN_SECONDS_KEY, SingleSignOnManager.DEFAULT_SESSION_TIMEOUT_IN_SECONDS)
self.logger.debug('Session timeout: %s' % sessionTimeout)
# Create session manager
sessionManager = self.configurationManager.getConfigOption(SingleSignOnManager.CONFIG_SECTION_NAME, SingleSignOnManager.SESSION_MANAGER_KEY)
(moduleName,className,constructor) = self.configurationManager.getModuleClassConstructorTuple(sessionManager, SingleSignOnManager)
self.logger.debug('Creating session manager class: %s' % className)
self.sessionManager = ObjectUtility.createObjectInstance(moduleName, className, constructor)
def addSession(self, sessionId, sessionInfo):
if self.sessionManager:
self.logger.debug('Adding session id %s: %s' % (sessionId,sessionInfo))
try:
return self.sessionManager.addSession(sessionId, sessionInfo)
except Exception, ex:
self.logger.error(ex)
return None
def checkSession(self, sessionId):
if self.sessionManager:
self.logger.debug('Checking session id: %s' % sessionId)
try:
return self.sessionManager.checkSession(sessionId)
except ObjectNotFound, ex:
self.logger.debug(ex)
except Exception, ex:
self.logger.error(ex)
return None
#######################################################################
# Testing.
if __name__ == '__main__':
am = AuthorizationPrincipalManager.getInstance()
authPrincipal = am.getAuthorizationPrincipal('sveseli', 'sv')
print 'Auth principal: ', authPrincipal
#!/usr/bin/env python
#
# Base DM controller class.
#
#######################################################################
import cherrypy
import httplib
import json
from sys import exc_info
from dm.common.utility.loggingManager import LoggingManager
from dm.common.constants import dmStatus
from dm.common.constants import dmHttpHeaders
from dm.common.exceptions.dmException import DmException
from dm.common.exceptions import dmExceptionMap
from dm.common.exceptions.internalError import InternalError
#######################################################################
class DmController(object):
""" Base controller class. """
def __init__(self):
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
@classmethod
def getLogger(cls):
logger = LoggingManager.getInstance().getLogger(cls.__name__)
return logger
@classmethod
def addDmResponseHeaders(cls, status=dmStatus.DM_OK, msg='Success', exceptionType=None):
cherrypy.response.headers[dmHttpHeaders.DM_STATUS_CODE_HTTP_HEADER] = status
cherrypy.response.headers[dmHttpHeaders.DM_STATUS_MESSAGE_HTTP_HEADER] = msg
if exceptionType is not None:
cherrypy.response.headers[dmHttpHeaders.DM_EXCEPTION_TYPE_HTTP_HEADER] = exceptionType
@classmethod
def addDmSessionRoleHeaders(cls, role):
cherrypy.response.headers[dmHttpHeaders.DM_SESSION_ROLE_HTTP_HEADER] = role
@classmethod
def addDmExceptionHeaders(cls, ex):
cls.handleException(ex)
@classmethod
def handleCpException(cls):
cherrypy.response.status = httplib.OK
ex = exc_info()[1]
if ex == None:
ex = InternalError('Internal Webservice Error')
cls.handleException(ex)
@classmethod
def handleException(cls, ex):
exClass = ex.__class__.__name__.split('.')[-1]
status = None
msg = '%s' % ex
msg = msg.split('\n')[0]
for code in dmExceptionMap.DM_EXCEPTION_MAP.keys():
exStr = dmExceptionMap.DM_EXCEPTION_MAP.get(code).split('.')[-1]
if exStr == exClass:
status = code
if not status:
status = dmStatus.DM_ERROR
cls.addDmResponseHeaders(status, msg, exClass)
@classmethod
def formatJsonResponse(cls, response):
cherrypy.response.headers['Content-Type'] = 'application/json'
return '%s' % (response)
@classmethod
def toJson(cls, o):
return json.dumps(o)
@classmethod
def fromJson(cls, s):
return json.loads(s)
@classmethod
def listToJson(cls, dmObjectList):
jsonList = []
for dmObject in dmObjectList:
jsonList.append(dmObject.getDictRep(keyList='ALL'))
return json.dumps(jsonList)
@classmethod
def getSessionUser(cls):
return cherrypy.serving.session.get('user')
@classmethod
def getSessionUsername(cls):
return cherrypy.serving.session.get('_cp_username')
# Exception decorator for all exposed method calls
@classmethod
def execute(cls, func):
def decorate(*args, **kwargs):
try:
response = func(*args, **kwargs)
except DmException, ex:
cls.getLogger().error('%s' % ex)
cls.handleException(ex)
response = ex.getFullJsonRep()
except Exception, ex:
cls.getLogger().error('%s' % ex)
cls.handleException(ex)
response = InternalError(ex).getFullJsonRep()
return cls.formatJsonResponse(response)
return decorate
#!/usr/bin/env python
#
# Base web service class.
#
####################################################################
import sys
import os
import cherrypy
from cherrypy.process import plugins
from cherrypy import server
from dm.common.constants import dmStatus
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.dmModuleManager import DmModuleManager
from dm.common.exceptions.configurationError import ConfigurationError
####################################################################
class DmRestWebServiceBase:
DEFAULT_N_SERVER_REQUEST_THREADS = 50
DEFAULT_SERVER_SOCKET_TIMEOUT = 30
CONFIG_SECTION_NAME = 'WebService'
CONFIG_OPTION_NAME_LIST = [ 'serviceHost', 'servicePort',
'sslCertFile', 'sslKeyFile', 'sslCaCertFile', 'stationName' ]
class SignalHandler:
def __init__(self, signal, oldSignalHandler):
self.signal = signal
self.oldSignalHandler = oldSignalHandler
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def signalHandler(self):
self.logger.debug('%s signal handler called' % self.signal)
DmModuleManager.getInstance().stopModules()
self.oldSignalHandler()
def __init__(self, routeMapper):
self.configurationManager = ConfigurationManager.getInstance()
self.routeMapper = routeMapper
self.options = None
self.args = None
self.logger = None
def prepareOptions(self):
from optparse import OptionParser
p = OptionParser()
p.add_option('-d', '--daemon', action="store_true",
dest='daemonFlag', default=False,
help="Run as a daemon.")
p.add_option('-p', '--pid-file',
dest='pidFile', default=None,
help="Store process id in the given file.")
p.add_option('', '--config-file',
dest='configFile', default=None,
help="Service configuration file.")
p.add_option('-P', '--port',
dest='servicePort', default=None,
help="Service port.")
p.add_option('-H', '--host',
dest='serviceHost', default=None,
help="Service host.")
p.add_option('-C', '--ssl-ca-cert',
dest='sslCaCertFile', default=None,
help='SSL CA certificate path (used for client SSL certificate verification). Requires --ssl-key and --ssl-cert.')
p.add_option('-c', '--ssl-cert',
dest='sslCertFile', default=None,
help='SSL certificate path. SSL operation requires both --ssl-key and --ssl-cert. Client SSL certificate verification also requires --ssl-ca-cert.')
p.add_option('-k', '--ssl-key',
dest='sslKeyFile', default=None,
help='SSL key path. SSL operation requires both --ssl-key and --ssl-cert. Client SSL certificate verification also requires --ssl-ca-cert.')
p.add_option('', '--n-server-threads',
dest='nServerThreads', default=DmRestWebServiceBase.DEFAULT_N_SERVER_REQUEST_THREADS,
help='Number of service request handler threads (defaut: %s).' % DmRestWebServiceBase.DEFAULT_N_SERVER_REQUEST_THREADS)
return p
def initDmModules(self):
return None
def getDefaultServerHost(self):
return None
def getDefaultServerPort(self):
return None
# Instantiate modified signal handler that stops dm modules first,
# and then does the default action.
def modifySignalHandlers(self, engine):
pluginsSignalHandler = plugins.SignalHandler(engine)
handlers = pluginsSignalHandler.handlers
# Add handler for interrupt
handlers['SIGINT'] = engine.exit
# Modify all signal handlers
for signal in handlers.keys():
self.logger.debug('Modifying signal: %s' % signal)
oldSignalHandler = handlers[signal]
self.logger.debug('Old signal handler: %s' % oldSignalHandler)
signalHandler = DmRestWebServiceBase.SignalHandler(signal, oldSignalHandler)
self.logger.debug('Setting signal handler to: %s' % signalHandler.signalHandler)
handlers[signal] = signalHandler.signalHandler
pluginsSignalHandler.subscribe()
def initServerLog(self):
cherrypyLogLevel = self.configurationManager.getCherrypyLogLevel()
cherrypy.log.error_log.setLevel(cherrypyLogLevel)
cherrypy.log.error_file = self.configurationManager.getCherrypyLogFile()
cherrypy.log.error_log.propagate = False
cherrypy.log.access_log.setLevel(cherrypyLogLevel)
cherrypy.log.access_file = self.configurationManager.getCherrypyAccessFile()
cherrypy.log.access_log.propagate = False
def updateServerConfig(self):
serviceHost = self.configurationManager.getServiceHost()
servicePort = int(self.configurationManager.getServicePort())
nServerThreads = int(self.options.nServerThreads)
configDict = {
'server.socket_host' : serviceHost,
'server.socket_port' : servicePort,
'server.thread_pool' : nServerThreads,
'log.screen' : (self.options.daemonFlag != True),
}
cherrypy.config.update(configDict)
def readConfigFile(self, configFile):
configFile = self.options.configFile
if not configFile:
configFile = self.configurationManager.getConfigFile()
else:
self.configurationManager.setConfigFile(configFile)
if not os.path.exists(configFile):
raise ConfigurationError('Configuration file %s does not exist.' % configFile)
# Read file and set config options
self.configurationManager.setOptionsFromConfigFile(DmRestWebServiceBase.CONFIG_SECTION_NAME, DmRestWebServiceBase.CONFIG_OPTION_NAME_LIST, configFile)
def readCommandLineOptions(self):
# This method should be called after reading config file
# in case some options are overridden
if self.options.sslCaCertFile != None:
self.configurationManager.setSslCaCertFile(self.options.sslCaCertFile)
if self.options.sslCertFile != None:
self.configurationManager.setSslCertFile(self.options.sslCertFile)
if self.options.sslKeyFile != None:
self.configurationManager.setSslKeyFile(self.options.sslKeyFile)
if self.options.serviceHost != None:
self.configurationManager.setServiceHost(self.options.serviceHost)
if self.options.servicePort != None:
self.configurationManager.setServicePort(self.options.servicePort)
def prepareServer(self):
try:
optionParser = self.prepareOptions()
(self.options, self.args) = optionParser.parse_args()
# Read config file and override with command line options
self.readConfigFile(self.options.configFile)
self.readCommandLineOptions()
# Turn off console log for daemon mode.
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
if self.options.daemonFlag:
LoggingManager.getInstance().setConsoleLogLevel('CRITICAL')
dispatch = self.routeMapper.setupRoutes()
self.logger.debug('Using route dispatch: %s' % dispatch)
config = {
'/' : {
'request.dispatch' : dispatch,
},
}
# No root controller as we provided our own.
cherrypy.tree.mount(root=None, config=config)
self.initServerLog()
self.updateServerConfig()
self.logger.info('Using host %s' % self.configurationManager.getServiceHost())
self.logger.info('Using port %s' % self.configurationManager.getServicePort())
self.logger.debug('Using %s request handler threads' % self.options.nServerThreads)
except Exception, ex:
if self.logger is not None:
self.logger.exception(ex)
else:
import traceback
print '\n%s' % sys.exc_info()[1]
traceback.print_exc(file=sys.stderr)
sys.exit(dmStatus.DM_ERROR)
# Run server.
def __runServer(self):
self.logger.info('Starting service')
engine = cherrypy.engine
# Set up Deamonization
if self.options.daemonFlag:
plugins.Daemonizer(engine).subscribe()
self.logger.debug('Daemon mode: %s' % self.options.daemonFlag)
if self.options.pidFile != None:
plugins.PIDFile(engine, self.options.pidFile).subscribe()
self.logger.debug('Using PID file: %s' % self.options.pidFile)
sslCertFile = self.configurationManager.getSslCertFile()
sslKeyFile = self.configurationManager.getSslKeyFile()
sslCaCertFile = self.configurationManager.getSslCaCertFile()
if sslCertFile != None and sslKeyFile != None:
server.ssl_ca_certificate = None
if sslCaCertFile != None:
server.ssl_ca_certificate = self.options.sslCaCertFile
self.logger.info('Using SSL CA cert file: %s' % sslCaCertFile)
server.ssl_certificate = sslCertFile
self.logger.info('Using SSL cert file: %s' % sslCertFile)
server.ssl_private_key = sslKeyFile
self.logger.info('Using SSL key file: %s' % sslKeyFile)
server.ssl_module = 'builtin'
#server.ssl_module = 'pyopenssl'
# Increase timeout to prevent early SSL connection terminations
server.socket_timeout = DmRestWebServiceBase.DEFAULT_SERVER_SOCKET_TIMEOUT
# Setup the signal handler to stop the application while running.
if hasattr(engine, 'signal_handler'):
self.logger.debug('Subscribing signal handler')
engine.signal_handler.subscribe()
self.modifySignalHandlers(engine)
# Turn off autoreloader.
self.logger.debug('Turning off autoreloader')
engine.autoreload.unsubscribe()
# Start the engine.
try:
self.logger.debug('Starting engine')
engine.start()
# Prepare dm services.
self.logger.debug('Starting modules')
self.initDmModules()
DmModuleManager.getInstance().startModules()
except Exception, ex:
self.logger.exception('Service exiting: %s' % ex)
DmModuleManager.getInstance().stopModules()
return dmStatus.DM_ERROR
self.logger.info('Service ready')
engine.block()
DmModuleManager.getInstance().stopModules()
self.logger.info('Service done')
return dmStatus.DM_OK
# Run server instance.
def run(self):
self.prepareServer()
sys.exit(self.__runServer())
####################################################################
# Testing
if __name__ == '__main__':
pass
#!/usr/bin/env python
#
# Base DM session controller class.
#
import cherrypy
from dm.common.constants import dmRole
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.service.dmController import DmController
from dm.common.service.loginController import LoginController
class DmSessionController(DmController):
""" Base session controller class. """
_cp_config = {
'tools.sessions.on': True,
'tools.auth.on': True
}
#auth = LoginController()
# Add before_handler for authorization
cherrypy.tools.auth = cherrypy.Tool('before_handler', LoginController.authCheck)
def __init__(self):
DmController.__init__(self)
@classmethod
def require(cls, *conditions):
"""
Decorator that appends conditions to the auth.require config
variable.
"""
def decorate(f):
if not hasattr(f, '_cp_config'):
f._cp_config = dict()
if 'auth.require' not in f._cp_config:
f._cp_config['auth.require'] = []
f._cp_config['auth.require'].extend(conditions)
return f
return decorate
@classmethod
def anyOf(cls, *conditions):
""" Returns True if any of the conditions match. """
def check():
for c in conditions:
if c():
return True
return False
return check
@classmethod
def allOf(cls, *conditions):
""" Returns True if all of the conditions match. """
def check():
for c in conditions:
if not c():
return False
return True
return check
@classmethod
def isLoggedIn(cls):
""" Returns True if session has been established. """
def userIsLoggedIn():
role = cherrypy.session.get(LoginController.SESSION_ROLE_KEY, None)
if role is not None:
return True
return False
return userIsLoggedIn
@classmethod
def isAdministrator(cls):
def userIsAdministrator():
result = (cherrypy.session.get(LoginController.SESSION_ROLE_KEY, None) == dmRole.DM_ADMIN_SESSION_ROLE)
return result
return userIsAdministrator
@classmethod
def hasAdministratorRole(cls):
sessionRole = cherrypy.session.get(LoginController.SESSION_ROLE_KEY, None)
return (sessionRole == dmRole.DM_ADMIN_SESSION_ROLE)
@classmethod
def hasManagerRole(cls, experimentStationIdOrName):
systemRoleDict = cherrypy.session.get(LoginController.SESSION_SYSTEM_ROLE_DICT_KEY, None)
experimentStationIdOrNameList = systemRoleDict.get(dmRole.DM_MANAGER_SYSTEM_ROLE_ID, [])
if not experimentStationIdOrNameList:
# Remote sessions may come with string key
experimentStationIdOrNameList = systemRoleDict.get(str(dmRole.DM_MANAGER_SYSTEM_ROLE_ID), [])
return (experimentStationIdOrNameList.count(experimentStationIdOrName) > 0)
@classmethod
def hasPiRole(cls, experimentIdOrName):
experimentRoleDict = cherrypy.session.get(LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY, None)
experimentIdOrNameList = experimentRoleDict.get(dmRole.DM_PI_EXPERIMENT_ROLE_ID, [])
return (experimentIdOrNameList.count(experimentIdOrName) > 0)
@classmethod
def hasUserRole(cls, experimentIdOrName):
experimentRoleDict = cherrypy.session.get(LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY, None)
experimentIdOrNameList = experimentRoleDict.get(dmRole.DM_USER_EXPERIMENT_ROLE_ID, [])
return (experimentIdOrNameList.count(experimentIdOrName) > 0)
@classmethod
def canManageStation(cls):
def userCanManageStation():
if cls.hasAdministratorRole():
return True
stationName = ConfigurationManager.getInstance().getStationName()
return cls.hasManagerRole(stationName)
return userCanManageStation
#!/usr/bin/env python
import cherrypy
import datetime
import urllib
from cherrypy.lib import httpauth
from dm.common.constants import dmStatus
from dm.common.constants import dmRole
from dm.common.constants import dmHttpStatus
from dm.common.exceptions.dmException import DmException
from dm.common.exceptions.dmHttpError import DmHttpError
from dm.common.exceptions.authorizationError import AuthorizationError
from dm.common.exceptions.invalidSession import InvalidSession
from dm.common.utility.loggingManager import LoggingManager
from dm.common.service.dmController import DmController
from dm.common.service.auth.authorizationPrincipalManager import AuthorizationPrincipalManager
from dm.common.service.auth.singleSignOnManager import SingleSignOnManager
class LoginController(DmController):
""" Controller to provide login and logout actions. """
SESSION_USERNAME_KEY = '_cp_username'
SESSION_ROLE_KEY = 'sessionRole'
SESSION_SYSTEM_ROLE_DICT_KEY = 'systemRoleDict'
SESSION_EXPERIMENT_ROLE_DICT_KEY = 'experimentRoleDict'
ORIGINAL_SESSION_ID_KEY = 'originalid'
INVALID_SESSION_KEY = 'invalidSession'
_cp_config = {
'tools.sessions.on' : True,
'tools.auth.on' : True
}
def __init__(self):
DmController.__init__(self)
def onLogin(self, username):
""" Called on successful login. """
return
def onLogout(self, username):
""" Called on logout. """
return
@classmethod
def getLoginForm(cls, msg='Enter username and password:', username='', fromPage='/'):
return """
<html>
<body>
<form method='post' action='/auth/login'>
<input type='hidden' name='fromPage' value='%(fromPage)s' />
<h2>DM Service</h2>
<p/>
%(msg)s
<p/>
<table border='0'>
<tr>
<td>Username:</td><td><input type='text' name="username" value='%(username)s'/></td>
</tr>
<tr>
<td>Password:</td><td><input type='password' name='password' /></td>
</tr>
<p/>
<tr>
<td></td>
<td><input type='submit' value='Log In' /></td>
</tr>
</table>
</form>
</body>
</html>""" % locals()
@classmethod
def parseBasicAuthorizationHeaders(cls):
try:
username = None
password = None
authorization = cherrypy.request.headers['authorization']
authorizationHeader = httpauth.parseAuthorization(authorization)
if authorizationHeader['auth_scheme'] == 'basic':
username = authorizationHeader['username']
password = authorizationHeader['password']
if username and password:
return (username, password)
else:
raise AuthorizationError('Username and/or password not supplied.')
except Exception, ex:
errorMsg = 'Could not extract username/password from authorization header: %s' % ex
raise AuthorizationError(errorMsg)
@classmethod
def checkCredentials(cls, username, password):
""" Verifies credentials for username and password."""
logger = LoggingManager.getInstance().getLogger('LoginController:checkCredentials')
logger.debug('Checking credential for User: %s' % (username))
#logger.debug('Checking credential for User: %s, Password: %s' % (username, password))
logger.debug('Session id: %s' % cherrypy.serving.session.id)
principal = AuthorizationPrincipalManager.getInstance().getAuthenticatedAuthorizationPrincipal(username, password)
logger.debug('Principal: %s' % (principal))
if principal:
cherrypy.session[LoginController.SESSION_ROLE_KEY] = principal.getSessionRole()
cherrypy.session[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY] = principal['userSystemRoleDict']
cherrypy.session[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY] = principal['userExperimentRoleDict']
logger.debug('Successful login from user: %s (role: %s)' % (username, principal.getSessionRole()))
# Try adding to SingleSignOnManager
sessionId = cherrypy.serving.session.id
sessionCache = cherrypy.session.cache
sessionInfo = {LoginController.SESSION_ROLE_KEY : principal.getSessionRole()}
sessionInfo[LoginController.SESSION_USERNAME_KEY] = username
sessionInfo[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY] = principal.get('userSystemRoleDict', {})
sessionInfo[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY] = principal.get('userExperimentRoleDict', {})
ssoManager = SingleSignOnManager.getInstance()
ssoManager.addSession(sessionId, sessionInfo)
else:
logger.debug('Login denied for user: %s' % username)
username = cherrypy.session.get(LoginController.SESSION_USERNAME_KEY, None)
if username is not None:
cherrypy.request.login = None
cherrypy.session[LoginController.INVALID_DM_SESSION_KEY] = True
raise AuthorizationError('Incorrect username or password.')
return principal
@classmethod
def authCheck(cls, *args, **kwargs):
"""
A tool that looks in config for 'auth.require'. If found and it
is not None, a login is required and the entry is evaluated as a list of
conditions that the user must fulfill.
"""
logger = LoggingManager.getInstance().getLogger('LoginController:authCheck')
conditions = cherrypy.request.config.get('auth.require', None)
#logger.debug('Headers: %s' % (cherrypy.request.headers))
#logger.debug('Request params: %s' % (cherrypy.request.params))
#logger.debug('Request query string: %s' % (cherrypy.request.query_string))
method = urllib.quote(cherrypy.request.request_line.split()[0])
params = urllib.quote(cherrypy.request.request_line.split()[1])
if conditions is None:
logger.debug('No conditions imposed')
return
sessionId = cherrypy.serving.session.id
sessionCache = cherrypy.session.cache
# If session cache does not have current session id, reuse original
# session id
if not sessionCache.has_key(sessionId) and cherrypy.serving.session.__dict__.has_key(LoginController.ORIGINAL_SESSION_ID_KEY):
logger.debug('Reusing original session id: %s' % sessionId)
sessionId = cherrypy.serving.session.__dict__.get(LoginController.ORIGINAL_SESSION_ID_KEY)
#logger.debug('Session: %s' % ((cherrypy.session.__dict__)))
logger.debug('Session cache length: %s' % (len(sessionCache)))
#logger.debug('Session cache: %s' % (sessionCache))
# Check session.
# Try SingleSignOnManager first
ssoManager = SingleSignOnManager.getInstance()
# SSO Manager returns session info
sessionInfo = ssoManager.checkSession(sessionId)
if not sessionInfo:
# Cache has tuple (sessionInfo, updateTime)
sessionTuple = sessionCache.get(sessionId)
if sessionTuple:
sessionInfo = sessionTuple[0]
logger.debug('Retrieved session info from cache: %s' % sessionInfo)
else:
logger.debug('Retrieved session %s from SSO Manager' % sessionId)
sessionCache[sessionId] = (sessionInfo, datetime.datetime.now())
if not sessionInfo:
errorMsg = 'Invalid or expired session id: %s.' % sessionId
logger.debug(errorMsg)
raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', InvalidSession(errorMsg))
username = sessionInfo.get(LoginController.SESSION_USERNAME_KEY)
if not username:
errorMsg = 'Invalid session id: %s (no username supplied).' % sessionId
logger.debug(errorMsg)
raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', InvalidSession(errorMsg))
cherrypy.session[LoginController.SESSION_ROLE_KEY] = sessionInfo[LoginController.SESSION_ROLE_KEY]
cherrypy.session[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY] = sessionInfo[LoginController.SESSION_SYSTEM_ROLE_DICT_KEY]
cherrypy.session[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY] = sessionInfo[LoginController.SESSION_EXPERIMENT_ROLE_DICT_KEY]
logger.debug('Session id %s is valid (username: %s)' % (sessionId, username))
cherrypy.request.login = username
for condition in conditions:
# A condition is just a callable that returns true or false
if not condition():
logger.debug('Authorization check %s() failed for username %s' % (condition.func_name, username))
errorMsg = 'Authorization check %s() failed for user %s.' % (condition.func_name, username)
raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', AuthorizationError(errorMsg))
@cherrypy.expose
def login(self, username=None, password=None, fromPage='/'):
self.logger.debug('Attempting login from username %s' % (username))
try:
if username is None or password is None:
self.logger.debug('Parsing auth headers for username %s' % (username))
(username, password) = LoginController.parseBasicAuthorizationHeaders()
self.logger.debug('Retrieving principal for username %s' % (username))
principal = LoginController.checkCredentials(username, password)
except DmHttpError, ex:
raise
except DmException, ex:
self.logger.debug('Authorization failed (username %s): %s' % (username, ex))
self.addDmExceptionHeaders(ex)
raise DmHttpError(dmHttpStatus.DM_HTTP_UNAUTHORIZED, 'User Not Authorized', ex)
# Authorization worked.
cherrypy.session[LoginController.SESSION_USERNAME_KEY] = cherrypy.request.login = username
self.onLogin(username)
self.addDmSessionRoleHeaders(principal.getSessionRole())
self.addDmResponseHeaders()
@cherrypy.expose
def logout(self, fromPage='/'):
sess = cherrypy.session
username = sess.get(LoginController.SESSION_USERNAME_KEY, None)
if username:
del sess[LoginController.SESSION_USERNAME_KEY]
cherrypy.request.login = None
self.onLogout(username)
#!/usr/bin/env python
#
# Login route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.service.loginController import LoginController
class LoginRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
loginController = LoginController()
routes = [
# Login
{
'name' : 'login',
'path' : '%s/login' % contextRoot,
'controller' : loginController,
'action' : 'login',
'method' : [ 'PUT', 'POST' ]
},
]
return routes