Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • DM/dm-docs
  • hammonds/dm-docs
  • hparraga/dm-docs
3 results
Show changes
Showing
with 1226 additions and 0 deletions
#!/usr/bin/env python
from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
class StartExperimentCli(DsWebServiceSessionCli):
def __init__(self):
DsWebServiceSessionCli.__init__(self)
self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
def checkArgs(self):
if self.options.experimentName is None:
raise InvalidRequest('Experiment name must be provided.')
def getExperimentName(self):
return self.options.experimentName
def runCommand(self):
self.parseArgs(usage="""
dm-start-experiment --experiment=EXPERIMENTNAME
Description:
Updates experiment start time in the DM database and prepares experiment data directory.
""")
self.checkArgs()
api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.startExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
#######################################################################
# Run command.
if __name__ == '__main__':
cli = StartExperimentCli()
cli.run()
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.ds_web_service.api.fileDsApi import FileDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class StatExperimentFileCli(DsWebServiceSessionCli):
def __init__(self):
DsWebServiceSessionCli.__init__(self)
self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
self.addOption('', '--relative-path', dest='experimentFilePath', help='Experiment (relative) file path.')
def checkArgs(self):
if self.options.experimentFilePath is None:
raise InvalidRequest('Experiment file path must be provided.')
if self.options.experimentName is None:
raise InvalidRequest('Experiment name must be provided.')
def getExperimentName(self):
return self.options.experimentName
def getExperimentFilePath(self):
return self.options.experimentFilePath
def runCommand(self):
self.parseArgs(usage="""
dm-stat-file --experiment=EXPERIMENTNAME --relative-path=EXPERIMENTFILEPATH
Description:
Retrieves stat information for a given file.
""")
self.checkArgs()
api = FileDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileMetadata = api.statFile(self.getExperimentFilePath(), self.getExperimentName())
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
#######################################################################
# Run command.
if __name__ == '__main__':
cli = StatExperimentFileCli()
cli.run()
#!/usr/bin/env python
from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
class StopExperimentCli(DsWebServiceSessionCli):
def __init__(self):
DsWebServiceSessionCli.__init__(self)
self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
def checkArgs(self):
if self.options.experimentName is None:
raise InvalidRequest('Experiment name must be provided.')
def getExperimentName(self):
return self.options.experimentName
def runCommand(self):
self.parseArgs(usage="""
dm-stop-experiment --experiment=EXPERIMENTNAME
Description:
Updates experiment end date in the DM database and checks experiment data permissions.
""")
self.checkArgs()
api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.stopExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
#######################################################################
# Run command.
if __name__ == '__main__':
cli = StopExperimentCli()
cli.run()
#!/usr/bin/env python
from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
class UpdateExperimentCli(DsWebServiceSessionCli):
def __init__(self):
DsWebServiceSessionCli.__init__(self)
self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
def checkArgs(self):
if self.options.experimentName is None:
raise InvalidRequest('Experiment name must be provided.')
def getExperimentName(self):
return self.options.experimentName
def runCommand(self):
self.parseArgs(usage="""
dm-update-experiment --experiment=EXPERIMENTNAME
Description:
Updates experiment group users.
""")
self.checkArgs()
api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.updateExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
#######################################################################
# Run command.
if __name__ == '__main__':
cli = UpdateExperimentCli()
cli.run()
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.db.api.userDbApi import UserDbApi
from dm.common.service.auth.authorizationPrincipalRetriever import AuthorizationPrincipalRetriever
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
class DsAuthPrincipalRetriever(AuthorizationPrincipalRetriever):
def __init__(self):
AuthorizationPrincipalRetriever.__init__(self, self.__class__.__name__)
self.authApi = DsRestApiFactory.getAuthDsApi()
def getAuthorizationPrincipal(self, username):
principal = self.authApi.getAuthorizationPrincipal(username)
return principal
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.db.api.userDbApi import UserDbApi
from dm.common.service.auth.sessionManager import SessionManager
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
class DsSessionManager(SessionManager):
def __init__(self):
SessionManager.__init__(self, self.__class__.__name__)
self.authApi = DsRestApiFactory.getAuthDsApi()
def addSession(self, sessionId, sessionInfo):
return self.authApi.addSession(sessionId, sessionInfo)
def checkSession(self, sessionId):
return self.authApi.checkSession(sessionId)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
#
# Auth route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from authSessionController import AuthSessionController
class AuthRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
authSessionController = AuthSessionController()
routes = [
# Get authorization principal
{
'name' : 'getAuthorizationPrincipal',
'path' : '%s/authorizationPrincipals/:(username)' % contextRoot,
'controller' : authSessionController,
'action' : 'getAuthorizationPrincipal',
'method' : [ 'GET' ]
},
# Add session
{
'name' : 'addSession',
'path' : '%s/sessions/:(sessionId)' % contextRoot,
'controller' : authSessionController,
'action' : 'addSession',
'method' : [ 'POST' ]
},
# Check session
{
'name' : 'checkSession',
'path' : '%s/sessions/:(sessionId)' % contextRoot,
'controller' : authSessionController,
'action' : 'checkSession',
'method' : [ 'PUT' ]
},
]
return routes
#authenticateUser!/usr/bin/env python
import json
import cherrypy
from dm.common.utility.encoder import Encoder
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.service.dmSessionController import DmSessionController
from dm.ds_web_service.service.impl.authSessionControllerImpl import AuthSessionControllerImpl
class AuthSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.authSessionControllerImpl = AuthSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getAuthorizationPrincipal(self, username, **kwargs):
if not len(username):
raise InvalidRequest('Invalid username provided.')
response = self.authSessionControllerImpl.getAuthorizationPrincipal(username).getFullJsonRep()
self.logger.debug('Returning authorization principal for %s' % (username))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def addSession(self, sessionId, **kwargs):
if not len(sessionId):
raise InvalidRequest('Invalid session id provided.')
sessionInfo = kwargs.get('sessionInfo')
if not sessionInfo:
raise InvalidRequest('Missing session object.')
sessionInfo = json.loads(Encoder.decode(sessionInfo))
response = self.authSessionControllerImpl.addSession(sessionId, sessionInfo).getFullJsonRep()
self.logger.debug('Session %s added to cache' % (sessionInfo))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def checkSession(self, sessionId, **kwargs):
if not len(sessionId):
raise InvalidRequest('Invalid session id provided.')
response = self.authSessionControllerImpl.checkSession(sessionId).getFullJsonRep()
self.logger.debug('Verified session %s' % (sessionId))
return response
#!/usr/bin/env python
#
# DM DS Web Service
#
####################################################################
from dm.common.service.dmRestWebServiceBase import DmRestWebServiceBase
from dm.common.utility.dmModuleManager import DmModuleManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.processing.fileProcessingManager import FileProcessingManager
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
from dsWebServiceRouteMapper import DsWebServiceRouteMapper
####################################################################
class DsWebService(DmRestWebServiceBase):
def __init__(self):
DmRestWebServiceBase.__init__(self, DsWebServiceRouteMapper)
def initDmModules(self):
self.logger.debug('Initializing dm modules')
# Add modules that will be started.
moduleManager = DmModuleManager.getInstance()
moduleManager.addModule(ExperimentManager.getInstance())
moduleManager.addModule(FileProcessingManager.getInstance())
self.logger.debug('Initialized dm modules')
def getDefaultServerHost(self):
return ConfigurationManager.getInstance().getServiceHost()
def getDefaultServerPort(self):
return ConfigurationManager.getInstance().getServicePort()
####################################################################
# Run service
if __name__ == '__main__':
ConfigurationManager.getInstance().setServiceName('ds-web-service')
service = DsWebService();
service.run()
#!/usr/bin/env python
#
# Route mapper for DM DS web service.
#
import sys
import os
import cherrypy
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.service.loginRouteDescriptor import LoginRouteDescriptor
from userRouteDescriptor import UserRouteDescriptor
from experimentRouteDescriptor import ExperimentRouteDescriptor
from fileRouteDescriptor import FileRouteDescriptor
from authRouteDescriptor import AuthRouteDescriptor
class DsWebServiceRouteMapper:
@classmethod
def setupRoutes(cls):
""" Setup RESTFul routes. """
logger = LoggingManager.getInstance().getLogger(cls.__name__)
contextRoot = ConfigurationManager.getInstance().getContextRoot()
logger.debug('Using context root: %s' % contextRoot)
# Get routes.
routes = LoginRouteDescriptor.getRoutes()
routes += AuthRouteDescriptor.getRoutes()
routes += UserRouteDescriptor.getRoutes()
routes += ExperimentRouteDescriptor.getRoutes()
routes += FileRouteDescriptor.getRoutes()
# Add routes to dispatcher.
d = cherrypy.dispatch.RoutesDispatcher()
for route in routes:
logger.debug('Connecting route: %s' % route)
d.connect(route['name'], route['path'], action=route['action'], controller=route['controller'], conditions=dict(method=route['method']))
return d
#!/usr/bin/env python
#
# Experiment route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from experimentSessionController import ExperimentSessionController
class ExperimentRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
experimentSessionController = ExperimentSessionController()
# Define routes.
routes = [
# Get experiment types
{
'name' : 'getExperimentTypes',
'path' : '%s/experimentTypes' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentTypes',
'method' : ['GET']
},
# Get experiment type by id
{
'name' : 'getExperimentTypeById',
'path' : '%s/experimentTypesById/:(id)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentTypeById',
'method' : ['GET']
},
# Get experiments
{
'name' : 'getExperiments',
'path' : '%s/experiments' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperiments',
'method' : ['GET']
},
# Get experiments by station
{
'name' : 'getExperimentsByStation',
'path' : '%s/experimentsByStation/:(stationName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentsByStation',
'method' : ['GET']
},
# Get experiment
{
'name' : 'getExperimentById',
'path' : '%s/experimentsById/:(id)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentById',
'method' : ['GET']
},
# Get experiment
{
'name' : 'getExperimentByName',
'path' : '%s/experimentsByName/:(name)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentByName',
'method' : ['GET']
},
# Add experiment
{
'name' : 'addExperiment',
'path' : '%s/experiments' % contextRoot,
'controller' : experimentSessionController,
'action' : 'addExperiment',
'method' : ['POST']
},
# Start experiment
{
'name' : 'startExperiment',
'path' : '%s/experiments/start' % contextRoot,
'controller' : experimentSessionController,
'action' : 'startExperiment',
'method' : ['PUT']
},
# Update experiment
{
'name' : 'updateExperiment',
'path' : '%s/experiments/update' % contextRoot,
'controller' : experimentSessionController,
'action' : 'updateExperiment',
'method' : ['PUT']
},
# Stop experiment
{
'name' : 'stopExperiment',
'path' : '%s/experiments/stop' % contextRoot,
'controller' : experimentSessionController,
'action' : 'stopExperiment',
'method' : ['PUT']
},
# Authorize download for a given UNIX username
{
'name' : 'authorizeDownload',
'path' : '%s/downloadAuthorizations/:(username)/:(experimentName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'authorizeDownload',
'method' : ['POST']
},
# Revoke authorization download for a given UNIX username
{
'name' : 'deauthorizeDownload',
'path' : '%s/downloadAuthorizations/:(username)/:(experimentName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'deauthorizeDownload',
'method' : ['DELETE']
},
]
return routes
#!/usr/bin/env python
import cherrypy
from dm.common.service.dmSessionController import DmSessionController
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.exceptions.authorizationError import AuthorizationError
from dm.common.utility.encoder import Encoder
from dm.ds_web_service.service.impl.experimentSessionControllerImpl import ExperimentSessionControllerImpl
class ExperimentSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.experimentSessionControllerImpl = ExperimentSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentTypes(self, **kwargs):
return self.listToJson(self.experimentSessionControllerImpl.getExperimentTypes())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentTypeById(self, id, **kwargs):
experimentType = self.experimentSessionControllerImpl.getExperimentTypeById(id)
return experimentType.getFullJsonRep()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getExperiments(self, **kwargs):
return self.listToJson(self.experimentSessionControllerImpl.getExperiments())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentsByStation(self, stationName, **kwargs):
if not self.hasAdministratorRole() and not self.hasManagerRole(stationName):
raise AuthorizationError('User %s cannot retrieve experiments for station %s.' % (self.getSessionUsername(), stationName))
return self.listToJson(self.experimentSessionControllerImpl.getExperimentsByStation(stationName))
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentByName(self, name, **kwargs):
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot retrieve experiment %s.' % (self.getSessionUsername(), name))
return experiment.getFullJsonRep()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentById(self, id, **kwargs):
experiment = self.experimentSessionControllerImpl.getExperimentById(id)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot retrieve experiment %s.' % (self.getSessionUsername(), id))
return experiment.getFullJsonRep()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def addExperiment(self, **kwargs):
name = kwargs.get('name')
if not name:
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
stationName = kwargs.get('stationName')
if not stationName:
raise InvalidRequest('Missing experiment station name.')
stationName = Encoder.decode(stationName)
typeName = kwargs.get('typeName')
if not typeName:
raise InvalidRequest('Missing experiment type.')
typeName = Encoder.decode(typeName)
description = kwargs.get('description')
if description is not None:
description = Encoder.decode(description)
startDate = kwargs.get('startDate')
if startDate is not None:
startDate = Encoder.decode(startDate)
endDate = kwargs.get('endDate')
if endDate is not None:
endDate = Encoder.decode(endDate)
if not self.hasAdministratorRole() and not self.hasManagerRole(stationName):
raise AuthorizationError('User %s is not allowed to add experiment on station %s.' % (self.getSessionUsername(), stationName))
response = self.experimentSessionControllerImpl.addExperiment(name, stationName, typeName, description, startDate, endDate).getFullJsonRep()
self.logger.debug('Added experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def startExperiment(self, **kwargs):
name = kwargs.get('name')
if not name:
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot start experiment %s.' % (self.getSessionUsername(), name))
response = self.experimentSessionControllerImpl.startExperiment(experiment).getFullJsonRep()
self.logger.debug('Started experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def updateExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot update experiment %s.' % (self.getSessionUsername(), name))
response = self.experimentSessionControllerImpl.updateExperiment(experiment).getFullJsonRep()
self.logger.debug('Updated experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def stopExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot stop experiment %s.' % (self.getSessionUsername(), name))
response = self.experimentSessionControllerImpl.stopExperiment(experiment).getFullJsonRep()
self.logger.debug('Stopped experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def authorizeDownload(self, username, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
experiment = self.experimentSessionControllerImpl.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s is not allowed to authorize download for experiment %s.' % (self.getSessionUsername(), experimentName))
response = self.experimentSessionControllerImpl.authorizeDownload(username, experiment).getFullJsonRep()
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def deauthorizeDownload(self, username, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
experiment = self.experimentSessionControllerImpl.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s is not allowed to de-authorize download for experiment %s.' % (self.getSessionUsername(), experimentName))
response = self.experimentSessionControllerImpl.deauthorizeDownload(username, experiment).getFullJsonRep()
return response
#!/usr/bin/env python
#
# File route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from fileSessionController import FileSessionController
class FileRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
fileSessionController = FileSessionController()
# Define routes.
routes = [
# Process file
{
'name' : 'processFile',
'path' : '%s/files/processFile' % contextRoot,
'controller' : fileSessionController,
'action' : 'processFile',
'method' : ['POST']
},
# Stat file
{
'name' : 'statFile',
'path' : '%s/files/statFile' % contextRoot,
'controller' : fileSessionController,
'action' : 'statFile',
'method' : ['POST']
},
# Process directory
{
'name' : 'processDirectory',
'path' : '%s/files/processDirectory' % contextRoot,
'controller' : fileSessionController,
'action' : 'processDirectory',
'method' : ['POST']
},
]
return routes
#!/usr/bin/env python
import cherrypy
import json
from dm.common.service.dmSessionController import DmSessionController
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.exceptions.authorizationError import AuthorizationError
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.common.utility.encoder import Encoder
from dm.ds_web_service.service.impl.fileSessionControllerImpl import FileSessionControllerImpl
class FileSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.experimentDbApi = ExperimentDbApi()
self.fileSessionControllerImpl = FileSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def processFile(self, **kwargs):
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
if not fileInfo.has_key('experimentFilePath'):
raise InvalidRequest('Experiment file path is missing.')
if not fileInfo.has_key('experimentName'):
raise InvalidRequest('Experiment name is missing.')
response = self.fileSessionControllerImpl.processFile(fileInfo).getFullJsonRep()
self.logger.debug('Processed file: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def statFile(self, **kwargs):
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
if not fileInfo.has_key('experimentFilePath'):
raise InvalidRequest('Experiment file path is missing.')
experimentName = fileInfo.get('experimentName')
if not experimentName:
raise InvalidRequest('Experiment name is missing.')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot stat file for experiment %s.' % (self.getSessionUsername(),experimentName))
response = self.fileSessionControllerImpl.statFile(fileInfo, experiment).getFullJsonRep()
self.logger.debug('File stat: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def processDirectory(self, **kwargs):
encodedDirectoryInfo = kwargs.get('directoryInfo')
if not encodedDirectoryInfo:
raise InvalidRequest('Invalid directory info provided.')
directoryInfo = json.loads(Encoder.decode(encodedDirectoryInfo))
if not directoryInfo.has_key('experimentName'):
raise InvalidRequest('Experiment name is missing.')
response = self.fileSessionControllerImpl.processDirectory(directoryInfo).getFullJsonRep()
self.logger.debug('Processed directory: %s' % response)
return response
#!/usr/bin/env python
#
# Implementation for user info controller.
#
import datetime
import cherrypy
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.objects.dmSession import DmSession
from dm.common.db.api.userDbApi import UserDbApi
from dm.common.exceptions.objectNotFound import ObjectNotFound
class AuthSessionControllerImpl(DmObjectManager):
""" User info controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.userDbApi = UserDbApi()
def getAuthorizationPrincipal(self, username):
principal = None
try:
user = self.userDbApi.getUserWithPasswordByUsername(username)
principal = AuthorizationPrincipal(name=username, token=user.get('password'))
principal.setUserSystemRoleDict(user.get('userSystemRoleDict', {}))
principal.setUserExperimentRoleDict(user.get('userExperimentRoleDict', {}))
principal.setSessionRole(dmRole.DM_USER_SESSION_ROLE)
for userSystemRoleId in principal.get('userSystemRoleDict', {}).keys():
if userSystemRoleId == dmRole.DM_ADMIN_SYSTEM_ROLE_ID:
principal.setSessionRole(dmRole.DM_ADMIN_SESSION_ROLE)
break
except Exception, ex:
self.logger.debug(ex)
return principal
def addSession(self, sessionId, sessionInfo):
sessionCache = cherrypy.session.cache
self.logger.debug('Session cache length: %s' % (len(sessionCache)))
sessionCache[sessionId] = (sessionInfo, datetime.datetime.now())
self.logger.debug('Session cache: %s' % (sessionCache))
return DmSession(sessionInfo)
def checkSession(self, sessionId):
sessionCache = cherrypy.session.cache
sessionTuple = sessionCache.get(sessionId)
if not sessionTuple:
raise ObjectNotFound('Session %s not found in cache.' % sessionId)
sessionInfo = sessionTuple[0]
oldTimestamp = sessionTuple[1]
newTimestamp = datetime.datetime.now()
self.logger.debug('Updated timestamp from %s to %s for session id %s' % (oldTimestamp, newTimestamp, sessionId))
sessionCache[sessionId] = (sessionInfo, newTimestamp)
return DmSession(sessionInfo)
#!/usr/bin/env python
import threading
import time
import os
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.utility.singleton import Singleton
from dm.common.utility.osUtility import OsUtility
from dm.common.utility.valueUtility import ValueUtility
from dm.common.utility.objectUtility import ObjectUtility
from dm.common.utility.threadingUtility import ThreadingUtility
from dm.common.utility.fileUtility import FileUtility
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.common.processing.fileProcessingManager import FileProcessingManager
from dm.common.exceptions.objectNotFound import ObjectNotFound
from dm.common.exceptions.invalidRequest import InvalidRequest
from pathTracker import PathTracker
class ExperimentManager(Singleton):
CONFIG_SECTION_NAME = 'ExperimentManager'
STORAGE_DIRECTORY_KEY = 'storagedirectory'
STORAGE_ID_KEY = 'storageid'
MANAGE_STORAGE_PERMISSIONS_KEY = 'managestoragepermissions'
PLATFORM_UTILITY_KEY = 'platformutility'
RSYNC_SCRIPT_PERMISSIONS_MODE = 0755
FILE_PERMISSIONS_MODE = 0640
FILE_PERMISSIONS_MODE_STRING = '0640'
DIR_PERMISSIONS_MODE = 0750
DIRECTORY_PROCESSING_DELAY_IN_SECONDS = 1
# Singleton.
__instanceLock = threading.RLock()
__instance = None
def __init__(self):
ExperimentManager.__instanceLock.acquire()
try:
if ExperimentManager.__instance:
return
ExperimentManager.__instance = self
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
self.logger.debug('Initializing')
self.lock = threading.RLock()
self.experimentDbApi = ExperimentDbApi()
self.pathTracker = PathTracker()
self.platformUtility = None
self.__configure()
self.fileProcessingManager = FileProcessingManager.getInstance()
self.logger.debug('Initialization complete')
finally:
ExperimentManager.__instanceLock.release()
def __configure(self):
cm = ConfigurationManager.getInstance()
configItems = cm.getConfigItems(ExperimentManager.CONFIG_SECTION_NAME)
self.logger.debug('Got config items: %s' % configItems)
self.storageDirectory = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.STORAGE_DIRECTORY_KEY)
self.storageId = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.STORAGE_ID_KEY)
self.manageStoragePermissions = ValueUtility.toBoolean(cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.MANAGE_STORAGE_PERMISSIONS_KEY))
platformUtility = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.PLATFORM_UTILITY_KEY)
if platformUtility:
(moduleName,className,constructor) = cm.getModuleClassConstructorTuple(platformUtility)
self.logger.debug('Creating platform utility class %s' % className)
self.platformUtility = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.logger.debug('Manage storage permissions: %s' % self.manageStoragePermissions)
def __getExperimentStorageDataDirectory(self, experiment):
experimentTypeName = experiment.get('experimentType').get('name')
experimentName = experiment.get('name')
storageDirectory = '%s/%s/%s' % (self.storageDirectory, experimentTypeName, experimentName)
storageDirectory = os.path.normpath(storageDirectory)
return storageDirectory
@ThreadingUtility.synchronize
def updateExperimentWithStorageDataDirectory(self, experiment):
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
storageHost = ConfigurationManager.getInstance().getHost()
experiment['storageDirectory'] = storageDirectory
experiment['storageHost'] = storageHost
experiment['storageUrl'] = '%s://%s%s' % (self.storageId, storageHost, storageDirectory)
return storageDirectory
def addUserToGroup(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentWithUsers(experimentName)
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.addUserToGroup(username, experimentName)
def deleteUserFromGroup(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentWithUsers(experimentName)
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
experimentUsers = experiment.get('experimentUsernameList', [])
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
def createRsyncScript(self, username, experimentName):
fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
self.logger.debug('Creating rsync script %s' % (fileName))
f = open(fileName, 'w')
f.write('#!/bin/sh\n')
f.write('exec sg %s "rsync $*"\n' % experimentName)
f.close()
OsUtility.chmodPath(fileName, fileMode=self.RSYNC_SCRIPT_PERMISSIONS_MODE)
def deleteRsyncScript(self, username, experimentName):
fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
self.logger.debug('Removing rsync script %s' % (fileName))
OsUtility.removeFile(fileName)
def authorizeDownload(self, username, experiment):
experimentName = experiment['name']
self.logger.debug('Authorizing download for %s from experiment %s' % (username, experimentName))
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.addLocalUserToGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
self.createRsyncScript(username, experimentName)
return experiment
def deauthorizeDownload(self, username, experiment):
experimentName = experiment['name']
self.logger.debug('De-authorizing download for %s from experiment %s' % (username, experimentName))
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.deleteLocalUserFromGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
self.deleteRsyncScript(username, experimentName)
return experiment
def createExperimentGroup(self, experiment):
experimentName = experiment.get('name')
storageDirectory = experiment.get('storageDirectory')
# Create experiment group
self.platformUtility.createGroup(experimentName)
self.logger.debug('Setting permissions for %s to %s' % (storageDirectory, self.DIR_PERMISSIONS_MODE))
OsUtility.chmodPath(storageDirectory, dirMode=self.DIR_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (storageDirectory, experimentName))
self.platformUtility.changePathGroupOwner(storageDirectory, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(storageDirectory, ownerUpdateTime)
# Add users to group
experimentUsers = experiment.get('experimentUsernameList', [])
self.logger.debug('Found experiment users: %s', experimentUsers)
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
def updateExperimentGroupUsers(self, experiment):
experimentName = experiment.get('name')
experimentUsers = experiment.get('experimentUsernameList', [])
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
@ThreadingUtility.synchronize
def createExperimentDataDirectory(self, experiment):
experimentName = experiment.get('name')
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.logger.debug('Data directory %s for experiment %s already exists' % (storageDirectory, experimentName))
else:
self.logger.debug('Creating data directory for experiment %s: %s' % (experimentName, storageDirectory))
OsUtility.createDir(storageDirectory)
storageHost = ConfigurationManager.getInstance().getHost()
experiment['storageDirectory'] = storageDirectory
experiment['storageHost'] = storageHost
experiment['storageUrl'] = '%s://%s%s' % (self.storageId, storageHost, storageDirectory)
if self.manageStoragePermissions:
self.createExperimentGroup(experiment)
def processExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
filePath = os.path.join(storageDirectory, experimentFilePath)
fileInfo['filePath'] = filePath
fileInfo['experiment'] = experiment
if not os.path.exists(filePath):
self.logger.error('File path %s does not exist' % filePath)
return
if self.manageStoragePermissions:
self.logger.debug('Setting permissions for %s to %s' % (filePath,self.FILE_PERMISSIONS_MODE_STRING))
OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (filePath, experimentName))
self.platformUtility.changePathGroupOwner(filePath, experimentName)
# Recursively modify subdirectory permissions
dirPath = os.path.dirname(filePath)
while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
if self.pathTracker.get(dirPath) is None:
self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
self.platformUtility.changePathGroupOwner(dirPath, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(dirPath, ownerUpdateTime)
else:
self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
dirPath = os.path.dirname(dirPath)
self.logger.debug('Processing file path %s (fileInfo: %s)' % (filePath, fileInfo))
self.fileProcessingManager.processFile(fileInfo)
def statExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
filePath = os.path.join(storageDirectory, experimentFilePath)
fileInfo['filePath'] = filePath
if os.path.exists(filePath):
self.logger.debug('Getting stat for file path %s' % (filePath))
FileUtility.statFile(filePath, fileInfo)
FileUtility.getMd5Sum(filePath, fileInfo)
self.logger.debug('File info after stat: %s' % str(fileInfo))
else:
self.logger.debug('File path %s does not exist' % filePath)
raise ObjectNotFound('File %s does not exist' % filePath)
def processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
directoryPath = os.path.join(storageDirectory, experimentDirectoryPath)
directoryInfo['directoryPath'] = directoryPath
directoryInfo['experiment'] = experiment
if not self.manageStoragePermissions:
self.logger.error('Skipping permission management for directory path %s' % directoryPath)
return
self.logger.debug('Processing directory path %s in background' % (directoryPath))
timer = threading.Timer(self.DIRECTORY_PROCESSING_DELAY_IN_SECONDS, self.__processExperimentDirectory, args=[experimentDirectoryPath, experiment, directoryInfo])
timer.start()
def __processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
experimentName = experiment.get('name')
storageDirectory = experiment.get('storageDirectory')
directoryPath = directoryInfo.get('directoryPath')
if not os.path.exists(directoryPath):
self.logger.error('Directory path %s does not exist' % directoryPath)
return
# Modify ownership
self.logger.debug('Processing directory path %s (directoryInfo: %s)' % (directoryPath, directoryInfo))
self.logger.debug('Modifying permissions for directory %s to %s' % (directoryPath, self.DIR_PERMISSIONS_MODE))
OsUtility.chmodPath(directoryPath, dirMode=self.DIR_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (directoryPath, experimentName))
self.platformUtility.recursivelyChangePathGroupOwner(directoryPath, experimentName)
# Recursively modify subdirectory permissions
dirPath = os.path.dirname(directoryPath)
while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
if self.pathTracker.get(dirPath) is None:
self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
self.platformUtility.changePathGroupOwner(dirPath, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(dirPath, ownerUpdateTime)
else:
self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
dirPath = os.path.dirname(dirPath)
# Update file permissions
self.logger.debug('Changing permissions for all files in %s for experiment %s' % (directoryPath, experimentName))
self.platformUtility.chmodPathForFilesInDirectory(directoryPath, self.FILE_PERMISSIONS_MODE_STRING)
@ThreadingUtility.synchronize
def start(self):
self.logger.debug('Started experiment manager')
@ThreadingUtility.synchronize
def stop(self):
self.logger.debug('Stopped experiment manager')
####################################################################
# Testing
if __name__ == '__main__':
em = ExperimentManager.getInstance()
print em
#!/usr/bin/env python
#
# Implementation for experiment session controller.
#
import time
from dm.common.objects.experiment import Experiment
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
class ExperimentSessionControllerImpl(DmObjectManager):
""" Experiment session controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.experimentDbApi = ExperimentDbApi()
def getExperimentTypes(self):
experimentTypeList = self.experimentDbApi.getExperimentTypes()
return experimentTypeList
def getExperiments(self):
experimentList = self.experimentDbApi.getExperiments()
return experimentList
def getExperimentsByStation(self, stationName):
experimentList = self.experimentDbApi.getExperimentsByStation(stationName)
return experimentList
def getExperimentByName(self, name):
experiment = self.experimentDbApi.getExperimentWithUsers(name)
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def getExperimentById(self, id):
experiment = self.experimentDbApi.getExperimentById(id)
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def addExperiment(self, name, stationName, typeId, description, startDate, endDate):
experiment = self.experimentDbApi.addExperiment(name, stationName, typeId, description, startDate, endDate)
ExperimentManager.getInstance().createExperimentDataDirectory(experiment)
return experiment
def startExperiment(self, experiment):
experiment2 = self.experimentDbApi.setExperimentStartDateToNow(experiment['name'])
experiment['startDate'] = experiment2['startDate']
return experiment
def updateExperiment(self, experiment):
ExperimentManager.getInstance().updateExperimentGroupUsers(experiment)
return experiment
def stopExperiment(self, experiment):
experiment = self.experimentDbApi.setExperimentEndDateToNow(experiment['name'])
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def authorizeDownload(self, username, experiment):
return ExperimentManager.getInstance().authorizeDownload(username, experiment)
def deauthorizeDownload(self, username, experiment):
return ExperimentManager.getInstance().deauthorizeDownload(username, experiment)