Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • DM/dm-docs
  • hammonds/dm-docs
  • hparraga/dm-docs
3 results
Show changes
Showing
with 1175 additions and 0 deletions
#!/usr/bin/env python
#
# Experiment route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from experimentSessionController import ExperimentSessionController
class ExperimentRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
experimentSessionController = ExperimentSessionController()
# Define routes.
routes = [
# Get experiment types
{
'name' : 'getExperimentTypes',
'path' : '%s/experimentTypes' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentTypes',
'method' : ['GET']
},
# Get experiment type by id
{
'name' : 'getExperimentTypeById',
'path' : '%s/experimentTypesById/:(id)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentTypeById',
'method' : ['GET']
},
# Get experiments
{
'name' : 'getExperiments',
'path' : '%s/experiments' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperiments',
'method' : ['GET']
},
# Get experiments by station
{
'name' : 'getExperimentsByStation',
'path' : '%s/experimentsByStation/:(stationName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentsByStation',
'method' : ['GET']
},
# Get experiment
{
'name' : 'getExperimentById',
'path' : '%s/experimentsById/:(id)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentById',
'method' : ['GET']
},
# Get experiment
{
'name' : 'getExperimentByName',
'path' : '%s/experimentsByName/:(name)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentByName',
'method' : ['GET']
},
# Add experiment
{
'name' : 'addExperiment',
'path' : '%s/experiments' % contextRoot,
'controller' : experimentSessionController,
'action' : 'addExperiment',
'method' : ['POST']
},
# Start experiment
{
'name' : 'startExperiment',
'path' : '%s/experiments/start' % contextRoot,
'controller' : experimentSessionController,
'action' : 'startExperiment',
'method' : ['PUT']
},
# Update experiment
{
'name' : 'updateExperiment',
'path' : '%s/experiments/update' % contextRoot,
'controller' : experimentSessionController,
'action' : 'updateExperiment',
'method' : ['PUT']
},
# Stop experiment
{
'name' : 'stopExperiment',
'path' : '%s/experiments/stop' % contextRoot,
'controller' : experimentSessionController,
'action' : 'stopExperiment',
'method' : ['PUT']
},
# Authorize download for a given UNIX username
{
'name' : 'authorizeDownload',
'path' : '%s/downloadAuthorizations/:(username)/:(experimentName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'authorizeDownload',
'method' : ['POST']
},
# Revoke authorization download for a given UNIX username
{
'name' : 'deauthorizeDownload',
'path' : '%s/downloadAuthorizations/:(username)/:(experimentName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'deauthorizeDownload',
'method' : ['DELETE']
},
]
return routes
#!/usr/bin/env python
import cherrypy
from dm.common.service.dmSessionController import DmSessionController
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.exceptions.authorizationError import AuthorizationError
from dm.common.utility.encoder import Encoder
from dm.ds_web_service.service.impl.experimentSessionControllerImpl import ExperimentSessionControllerImpl
class ExperimentSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.experimentSessionControllerImpl = ExperimentSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentTypes(self, **kwargs):
return self.listToJson(self.experimentSessionControllerImpl.getExperimentTypes())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentTypeById(self, id, **kwargs):
experimentType = self.experimentSessionControllerImpl.getExperimentTypeById(id)
return experimentType.getFullJsonRep()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getExperiments(self, **kwargs):
return self.listToJson(self.experimentSessionControllerImpl.getExperiments())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentsByStation(self, stationName, **kwargs):
if not self.hasAdministratorRole() and not self.hasManagerRole(stationName):
raise AuthorizationError('User %s cannot retrieve experiments for station %s.' % (self.getSessionUsername(), stationName))
return self.listToJson(self.experimentSessionControllerImpl.getExperimentsByStation(stationName))
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentByName(self, name, **kwargs):
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot retrieve experiment %s.' % (self.getSessionUsername(), name))
return experiment.getFullJsonRep()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentById(self, id, **kwargs):
experiment = self.experimentSessionControllerImpl.getExperimentById(id)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot retrieve experiment %s.' % (self.getSessionUsername(), id))
return experiment.getFullJsonRep()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def addExperiment(self, **kwargs):
name = kwargs.get('name')
if not name:
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
stationName = kwargs.get('stationName')
if not stationName:
raise InvalidRequest('Missing experiment station name.')
stationName = Encoder.decode(stationName)
typeName = kwargs.get('typeName')
if not typeName:
raise InvalidRequest('Missing experiment type.')
typeName = Encoder.decode(typeName)
description = kwargs.get('description')
if description is not None:
description = Encoder.decode(description)
startDate = kwargs.get('startDate')
if startDate is not None:
startDate = Encoder.decode(startDate)
endDate = kwargs.get('endDate')
if endDate is not None:
endDate = Encoder.decode(endDate)
if not self.hasAdministratorRole() and not self.hasManagerRole(stationName):
raise AuthorizationError('User %s is not allowed to add experiment on station %s.' % (self.getSessionUsername(), stationName))
response = self.experimentSessionControllerImpl.addExperiment(name, stationName, typeName, description, startDate, endDate).getFullJsonRep()
self.logger.debug('Added experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def startExperiment(self, **kwargs):
name = kwargs.get('name')
if not name:
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot start experiment %s.' % (self.getSessionUsername(), name))
response = self.experimentSessionControllerImpl.startExperiment(experiment).getFullJsonRep()
self.logger.debug('Started experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def updateExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot update experiment %s.' % (self.getSessionUsername(), name))
response = self.experimentSessionControllerImpl.updateExperiment(experiment).getFullJsonRep()
self.logger.debug('Updated experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def stopExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
experiment = self.experimentSessionControllerImpl.getExperimentByName(name)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot stop experiment %s.' % (self.getSessionUsername(), name))
response = self.experimentSessionControllerImpl.stopExperiment(experiment).getFullJsonRep()
self.logger.debug('Stopped experiment: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def authorizeDownload(self, username, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
experiment = self.experimentSessionControllerImpl.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s is not allowed to authorize download for experiment %s.' % (self.getSessionUsername(), experimentName))
response = self.experimentSessionControllerImpl.authorizeDownload(username, experiment).getFullJsonRep()
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def deauthorizeDownload(self, username, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
experiment = self.experimentSessionControllerImpl.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s is not allowed to de-authorize download for experiment %s.' % (self.getSessionUsername(), experimentName))
response = self.experimentSessionControllerImpl.deauthorizeDownload(username, experiment).getFullJsonRep()
return response
#!/usr/bin/env python
#
# File route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from fileSessionController import FileSessionController
class FileRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
fileSessionController = FileSessionController()
# Define routes.
routes = [
# Process file
{
'name' : 'processFile',
'path' : '%s/files/processFile' % contextRoot,
'controller' : fileSessionController,
'action' : 'processFile',
'method' : ['POST']
},
# Stat file
{
'name' : 'statFile',
'path' : '%s/files/statFile' % contextRoot,
'controller' : fileSessionController,
'action' : 'statFile',
'method' : ['POST']
},
# Process directory
{
'name' : 'processDirectory',
'path' : '%s/files/processDirectory' % contextRoot,
'controller' : fileSessionController,
'action' : 'processDirectory',
'method' : ['POST']
},
]
return routes
#!/usr/bin/env python
import cherrypy
import json
from dm.common.service.dmSessionController import DmSessionController
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.exceptions.authorizationError import AuthorizationError
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.common.utility.encoder import Encoder
from dm.ds_web_service.service.impl.fileSessionControllerImpl import FileSessionControllerImpl
class FileSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.experimentDbApi = ExperimentDbApi()
self.fileSessionControllerImpl = FileSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def processFile(self, **kwargs):
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
if not fileInfo.has_key('experimentFilePath'):
raise InvalidRequest('Experiment file path is missing.')
if not fileInfo.has_key('experimentName'):
raise InvalidRequest('Experiment name is missing.')
response = self.fileSessionControllerImpl.processFile(fileInfo).getFullJsonRep()
self.logger.debug('Processed file: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def statFile(self, **kwargs):
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
if not fileInfo.has_key('experimentFilePath'):
raise InvalidRequest('Experiment file path is missing.')
experimentName = fileInfo.get('experimentName')
if not experimentName:
raise InvalidRequest('Experiment name is missing.')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']):
raise AuthorizationError('User %s cannot stat file for experiment %s.' % (self.getSessionUsername(),experimentName))
response = self.fileSessionControllerImpl.statFile(fileInfo, experiment).getFullJsonRep()
self.logger.debug('File stat: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def processDirectory(self, **kwargs):
encodedDirectoryInfo = kwargs.get('directoryInfo')
if not encodedDirectoryInfo:
raise InvalidRequest('Invalid directory info provided.')
directoryInfo = json.loads(Encoder.decode(encodedDirectoryInfo))
if not directoryInfo.has_key('experimentName'):
raise InvalidRequest('Experiment name is missing.')
response = self.fileSessionControllerImpl.processDirectory(directoryInfo).getFullJsonRep()
self.logger.debug('Processed directory: %s' % response)
return response
#!/usr/bin/env python
#
# Implementation for user info controller.
#
import datetime
import cherrypy
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.objects.dmSession import DmSession
from dm.common.db.api.userDbApi import UserDbApi
from dm.common.exceptions.objectNotFound import ObjectNotFound
class AuthSessionControllerImpl(DmObjectManager):
""" User info controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.userDbApi = UserDbApi()
def getAuthorizationPrincipal(self, username):
principal = None
try:
user = self.userDbApi.getUserWithPasswordByUsername(username)
principal = AuthorizationPrincipal(name=username, token=user.get('password'))
principal.setUserSystemRoleDict(user.get('userSystemRoleDict', {}))
principal.setUserExperimentRoleDict(user.get('userExperimentRoleDict', {}))
principal.setSessionRole(dmRole.DM_USER_SESSION_ROLE)
for userSystemRoleId in principal.get('userSystemRoleDict', {}).keys():
if userSystemRoleId == dmRole.DM_ADMIN_SYSTEM_ROLE_ID:
principal.setSessionRole(dmRole.DM_ADMIN_SESSION_ROLE)
break
except Exception, ex:
self.logger.debug(ex)
return principal
def addSession(self, sessionId, sessionInfo):
sessionCache = cherrypy.session.cache
self.logger.debug('Session cache length: %s' % (len(sessionCache)))
sessionCache[sessionId] = (sessionInfo, datetime.datetime.now())
self.logger.debug('Session cache: %s' % (sessionCache))
return DmSession(sessionInfo)
def checkSession(self, sessionId):
sessionCache = cherrypy.session.cache
sessionTuple = sessionCache.get(sessionId)
if not sessionTuple:
raise ObjectNotFound('Session %s not found in cache.' % sessionId)
sessionInfo = sessionTuple[0]
oldTimestamp = sessionTuple[1]
newTimestamp = datetime.datetime.now()
self.logger.debug('Updated timestamp from %s to %s for session id %s' % (oldTimestamp, newTimestamp, sessionId))
sessionCache[sessionId] = (sessionInfo, newTimestamp)
return DmSession(sessionInfo)
#!/usr/bin/env python
import threading
import time
import os
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.utility.singleton import Singleton
from dm.common.utility.osUtility import OsUtility
from dm.common.utility.valueUtility import ValueUtility
from dm.common.utility.objectUtility import ObjectUtility
from dm.common.utility.threadingUtility import ThreadingUtility
from dm.common.utility.fileUtility import FileUtility
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.common.processing.fileProcessingManager import FileProcessingManager
from dm.common.exceptions.objectNotFound import ObjectNotFound
from dm.common.exceptions.invalidRequest import InvalidRequest
from pathTracker import PathTracker
class ExperimentManager(Singleton):
CONFIG_SECTION_NAME = 'ExperimentManager'
STORAGE_DIRECTORY_KEY = 'storagedirectory'
STORAGE_ID_KEY = 'storageid'
MANAGE_STORAGE_PERMISSIONS_KEY = 'managestoragepermissions'
PLATFORM_UTILITY_KEY = 'platformutility'
RSYNC_SCRIPT_PERMISSIONS_MODE = 0755
FILE_PERMISSIONS_MODE = 0640
FILE_PERMISSIONS_MODE_STRING = '0640'
DIR_PERMISSIONS_MODE = 0750
DIRECTORY_PROCESSING_DELAY_IN_SECONDS = 1
# Singleton.
__instanceLock = threading.RLock()
__instance = None
def __init__(self):
ExperimentManager.__instanceLock.acquire()
try:
if ExperimentManager.__instance:
return
ExperimentManager.__instance = self
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
self.logger.debug('Initializing')
self.lock = threading.RLock()
self.experimentDbApi = ExperimentDbApi()
self.pathTracker = PathTracker()
self.platformUtility = None
self.__configure()
self.fileProcessingManager = FileProcessingManager.getInstance()
self.logger.debug('Initialization complete')
finally:
ExperimentManager.__instanceLock.release()
def __configure(self):
cm = ConfigurationManager.getInstance()
configItems = cm.getConfigItems(ExperimentManager.CONFIG_SECTION_NAME)
self.logger.debug('Got config items: %s' % configItems)
self.storageDirectory = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.STORAGE_DIRECTORY_KEY)
self.storageId = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.STORAGE_ID_KEY)
self.manageStoragePermissions = ValueUtility.toBoolean(cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.MANAGE_STORAGE_PERMISSIONS_KEY))
platformUtility = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.PLATFORM_UTILITY_KEY)
if platformUtility:
(moduleName,className,constructor) = cm.getModuleClassConstructorTuple(platformUtility)
self.logger.debug('Creating platform utility class %s' % className)
self.platformUtility = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.logger.debug('Manage storage permissions: %s' % self.manageStoragePermissions)
def __getExperimentStorageDataDirectory(self, experiment):
experimentTypeName = experiment.get('experimentType').get('name')
experimentName = experiment.get('name')
storageDirectory = '%s/%s/%s' % (self.storageDirectory, experimentTypeName, experimentName)
storageDirectory = os.path.normpath(storageDirectory)
return storageDirectory
@ThreadingUtility.synchronize
def updateExperimentWithStorageDataDirectory(self, experiment):
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
storageHost = ConfigurationManager.getInstance().getHost()
experiment['storageDirectory'] = storageDirectory
experiment['storageHost'] = storageHost
experiment['storageUrl'] = '%s://%s%s' % (self.storageId, storageHost, storageDirectory)
return storageDirectory
def addUserToGroup(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentWithUsers(experimentName)
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.addUserToGroup(username, experimentName)
def deleteUserFromGroup(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentWithUsers(experimentName)
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
experimentUsers = experiment.get('experimentUsernameList', [])
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
def createRsyncScript(self, username, experimentName):
fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
self.logger.debug('Creating rsync script %s' % (fileName))
f = open(fileName, 'w')
f.write('#!/bin/sh\n')
f.write('exec sg %s "rsync $*"\n' % experimentName)
f.close()
OsUtility.chmodPath(fileName, fileMode=self.RSYNC_SCRIPT_PERMISSIONS_MODE)
def deleteRsyncScript(self, username, experimentName):
fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
self.logger.debug('Removing rsync script %s' % (fileName))
OsUtility.removeFile(fileName)
def authorizeDownload(self, username, experiment):
experimentName = experiment['name']
self.logger.debug('Authorizing download for %s from experiment %s' % (username, experimentName))
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.addLocalUserToGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
self.createRsyncScript(username, experimentName)
return experiment
def deauthorizeDownload(self, username, experiment):
experimentName = experiment['name']
self.logger.debug('De-authorizing download for %s from experiment %s' % (username, experimentName))
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.deleteLocalUserFromGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
self.deleteRsyncScript(username, experimentName)
return experiment
def createExperimentGroup(self, experiment):
experimentName = experiment.get('name')
storageDirectory = experiment.get('storageDirectory')
# Create experiment group
self.platformUtility.createGroup(experimentName)
self.logger.debug('Setting permissions for %s to %s' % (storageDirectory, self.DIR_PERMISSIONS_MODE))
OsUtility.chmodPath(storageDirectory, dirMode=self.DIR_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (storageDirectory, experimentName))
self.platformUtility.changePathGroupOwner(storageDirectory, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(storageDirectory, ownerUpdateTime)
# Add users to group
experimentUsers = experiment.get('experimentUsernameList', [])
self.logger.debug('Found experiment users: %s', experimentUsers)
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
def updateExperimentGroupUsers(self, experiment):
experimentName = experiment.get('name')
experimentUsers = experiment.get('experimentUsernameList', [])
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
@ThreadingUtility.synchronize
def createExperimentDataDirectory(self, experiment):
experimentName = experiment.get('name')
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.logger.debug('Data directory %s for experiment %s already exists' % (storageDirectory, experimentName))
else:
self.logger.debug('Creating data directory for experiment %s: %s' % (experimentName, storageDirectory))
OsUtility.createDir(storageDirectory)
storageHost = ConfigurationManager.getInstance().getHost()
experiment['storageDirectory'] = storageDirectory
experiment['storageHost'] = storageHost
experiment['storageUrl'] = '%s://%s%s' % (self.storageId, storageHost, storageDirectory)
if self.manageStoragePermissions:
self.createExperimentGroup(experiment)
def processExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
filePath = os.path.join(storageDirectory, experimentFilePath)
fileInfo['filePath'] = filePath
fileInfo['experiment'] = experiment
if not os.path.exists(filePath):
self.logger.error('File path %s does not exist' % filePath)
return
if self.manageStoragePermissions:
self.logger.debug('Setting permissions for %s to %s' % (filePath,self.FILE_PERMISSIONS_MODE_STRING))
OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (filePath, experimentName))
self.platformUtility.changePathGroupOwner(filePath, experimentName)
# Recursively modify subdirectory permissions
dirPath = os.path.dirname(filePath)
while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
if self.pathTracker.get(dirPath) is None:
self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
self.platformUtility.changePathGroupOwner(dirPath, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(dirPath, ownerUpdateTime)
else:
self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
dirPath = os.path.dirname(dirPath)
self.logger.debug('Processing file path %s (fileInfo: %s)' % (filePath, fileInfo))
self.fileProcessingManager.processFile(fileInfo)
def statExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
filePath = os.path.join(storageDirectory, experimentFilePath)
fileInfo['filePath'] = filePath
if os.path.exists(filePath):
self.logger.debug('Getting stat for file path %s' % (filePath))
FileUtility.statFile(filePath, fileInfo)
FileUtility.getMd5Sum(filePath, fileInfo)
self.logger.debug('File info after stat: %s' % str(fileInfo))
else:
self.logger.debug('File path %s does not exist' % filePath)
raise ObjectNotFound('File %s does not exist' % filePath)
def processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
directoryPath = os.path.join(storageDirectory, experimentDirectoryPath)
directoryInfo['directoryPath'] = directoryPath
directoryInfo['experiment'] = experiment
if not self.manageStoragePermissions:
self.logger.error('Skipping permission management for directory path %s' % directoryPath)
return
self.logger.debug('Processing directory path %s in background' % (directoryPath))
timer = threading.Timer(self.DIRECTORY_PROCESSING_DELAY_IN_SECONDS, self.__processExperimentDirectory, args=[experimentDirectoryPath, experiment, directoryInfo])
timer.start()
def __processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
experimentName = experiment.get('name')
storageDirectory = experiment.get('storageDirectory')
directoryPath = directoryInfo.get('directoryPath')
if not os.path.exists(directoryPath):
self.logger.error('Directory path %s does not exist' % directoryPath)
return
# Modify ownership
self.logger.debug('Processing directory path %s (directoryInfo: %s)' % (directoryPath, directoryInfo))
self.logger.debug('Modifying permissions for directory %s to %s' % (directoryPath, self.DIR_PERMISSIONS_MODE))
OsUtility.chmodPath(directoryPath, dirMode=self.DIR_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (directoryPath, experimentName))
self.platformUtility.recursivelyChangePathGroupOwner(directoryPath, experimentName)
# Recursively modify subdirectory permissions
dirPath = os.path.dirname(directoryPath)
while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
if self.pathTracker.get(dirPath) is None:
self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
self.platformUtility.changePathGroupOwner(dirPath, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(dirPath, ownerUpdateTime)
else:
self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
dirPath = os.path.dirname(dirPath)
# Update file permissions
self.logger.debug('Changing permissions for all files in %s for experiment %s' % (directoryPath, experimentName))
self.platformUtility.chmodPathForFilesInDirectory(directoryPath, self.FILE_PERMISSIONS_MODE_STRING)
@ThreadingUtility.synchronize
def start(self):
self.logger.debug('Started experiment manager')
@ThreadingUtility.synchronize
def stop(self):
self.logger.debug('Stopped experiment manager')
####################################################################
# Testing
if __name__ == '__main__':
em = ExperimentManager.getInstance()
print em
#!/usr/bin/env python
#
# Implementation for experiment session controller.
#
import time
from dm.common.objects.experiment import Experiment
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
class ExperimentSessionControllerImpl(DmObjectManager):
""" Experiment session controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.experimentDbApi = ExperimentDbApi()
def getExperimentTypes(self):
experimentTypeList = self.experimentDbApi.getExperimentTypes()
return experimentTypeList
def getExperiments(self):
experimentList = self.experimentDbApi.getExperiments()
return experimentList
def getExperimentsByStation(self, stationName):
experimentList = self.experimentDbApi.getExperimentsByStation(stationName)
return experimentList
def getExperimentByName(self, name):
experiment = self.experimentDbApi.getExperimentWithUsers(name)
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def getExperimentById(self, id):
experiment = self.experimentDbApi.getExperimentById(id)
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def addExperiment(self, name, stationName, typeId, description, startDate, endDate):
experiment = self.experimentDbApi.addExperiment(name, stationName, typeId, description, startDate, endDate)
ExperimentManager.getInstance().createExperimentDataDirectory(experiment)
return experiment
def startExperiment(self, experiment):
experiment2 = self.experimentDbApi.setExperimentStartDateToNow(experiment['name'])
experiment['startDate'] = experiment2['startDate']
return experiment
def updateExperiment(self, experiment):
ExperimentManager.getInstance().updateExperimentGroupUsers(experiment)
return experiment
def stopExperiment(self, experiment):
experiment = self.experimentDbApi.setExperimentEndDateToNow(experiment['name'])
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def authorizeDownload(self, username, experiment):
return ExperimentManager.getInstance().authorizeDownload(username, experiment)
def deauthorizeDownload(self, username, experiment):
return ExperimentManager.getInstance().deauthorizeDownload(username, experiment)
#!/usr/bin/env python
#
# Implementation for file session controller.
#
import time
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.objects.fileMetadata import FileMetadata
from dm.common.objects.directoryMetadata import DirectoryMetadata
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
class FileSessionControllerImpl(DmObjectManager):
""" File session controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.experimentDbApi = ExperimentDbApi()
def processFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
ExperimentManager.getInstance().processExperimentFile(experimentFilePath, experiment, fileInfo)
return FileMetadata(fileInfo)
def statFile(self, fileInfo, experiment=None):
experimentFilePath = fileInfo.get('experimentFilePath')
if not experiment:
experimentName = fileInfo.get('experimentName')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
ExperimentManager.getInstance().statExperimentFile(experimentFilePath, experiment, fileInfo)
return FileMetadata(fileInfo)
def processDirectory(self, directoryInfo):
experimentDirectoryPath = directoryInfo.get('experimentDirectoryPath', '')
experimentName = directoryInfo.get('experimentName')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
ExperimentManager.getInstance().processExperimentDirectory(experimentDirectoryPath, experiment, directoryInfo)
return DirectoryMetadata(directoryInfo)
#!/usr/bin/env python
from dm.common.utility.objectTracker import ObjectTracker
class PathTracker(ObjectTracker):
# Cache configuration
pass
####################################################################
# Testing
if __name__ == '__main__':
et = PathTracker.getInstance()
print et
et2 = PathTracker.getInstance()
print et2
et.put('xyz', 1)
print et.get('xyz')
print et.get('xyz2')
#!/usr/bin/env python
#
# Implementation for user info controller.
#
from dm.common.objects.dmObject import DmObject
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.db.api.userDbApi import UserDbApi
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
class UserInfoSessionControllerImpl(DmObjectManager):
""" User info controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.userDbApi = UserDbApi()
def getUsers(self):
return self.userDbApi.getUsers()
def getUserById(self, id):
return self.userDbApi.getUserById(id)
def getUserByUsername(self, username):
return self.userDbApi.getUserByUsername(username)
def addUserSystemRole(self, username, roleName, experimentStationName):
userSystemRole = self.userDbApi.addUserSystemRole(username, roleName, experimentStationName)
return userSystemRole
def deleteUserSystemRole(self, username, roleName, experimentStationName):
userSystemRole = self.userDbApi.deleteUserSystemRole(username, roleName, experimentStationName)
return userSystemRole
def addUserExperimentRole(self, username, roleName, experimentName):
userExperimentRole = self.userDbApi.addUserExperimentRole(username, roleName, experimentName)
ExperimentManager.getInstance().addUserToGroup(username, experimentName)
return userExperimentRole
def deleteUserExperimentRole(self, username, roleName, experimentName):
userExperimentRole = self.userDbApi.deleteUserExperimentRole(username, roleName, experimentName)
ExperimentManager.getInstance().deleteUserFromGroup(username, experimentName)
return userExperimentRole
#!/usr/bin/env python
import cherrypy
from dm.common.service.dmSessionController import DmSessionController
from dm.ds_web_service.service.impl.userInfoSessionControllerImpl import UserInfoSessionControllerImpl
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.exceptions.authorizationError import AuthorizationError
class UserInfoSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.experimentDbApi = ExperimentDbApi()
self.userInfoSessionControllerImpl = UserInfoSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getUsers(self, **kwargs):
return self.listToJson(self.userInfoSessionControllerImpl.getUsers())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getUserById(self, id, **kwargs):
if not id:
raise InvalidRequest('Invalid id provided.')
response = self.userInfoSessionControllerImpl.getUserById(id).getFullJsonRep()
self.logger.debug('Returning user info for %s: %s' % (id,response))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getUserByUsername(self, username, **kwargs):
if not len(username):
raise InvalidRequest('Invalid username provided.')
response = self.userInfoSessionControllerImpl.getUserByUsername(username).getFullJsonRep()
self.logger.debug('Returning user info for %s: %s' % (username,response))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def addUserSystemRole(self, username, roleName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not roleName:
raise InvalidRequest('Invalid role name provided.')
experimentStationName = kwargs.get('experimentStationName')
if experimentStationName:
experimentStation = self.experimentDbApi.getExperimentStationByName(experimentStationName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experimentStation['id']):
raise AuthorizationError('User %s cannot modify system roles for experiment station %s.' % (self.getSessionUsername(),experimentStationName))
else:
if not self.hasAdministratorRole():
raise AuthorizationError('User %s cannot modify system roles.' % (self.getSessionUsername()))
response = self.userInfoSessionControllerImpl.addUserSystemRole(username, roleName, experimentStationName).getFullJsonRep()
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def deleteUserSystemRole(self, username, roleName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not roleName:
raise InvalidRequest('Invalid role name provided.')
experimentStationName = kwargs.get('experimentStationName')
if experimentStationName:
experimentStation = self.experimentDbApi.getExperimentStationByName(experimentStationName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experimentStation['id']):
raise AuthorizationError('User %s cannot modify system roles for experiment station %s.' % (self.getSessionUsername(),experimentStationName))
else:
if not self.hasAdministratorRole():
raise AuthorizationError('User %s cannot modify system roles.' % (self.getSessionUsername()))
response = self.userInfoSessionControllerImpl.deleteUserSystemRole(username, roleName, experimentStationName).getFullJsonRep()
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def addUserExperimentRole(self, username, roleName, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
if not roleName:
raise InvalidRequest('Invalid role name provided.')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']) and not self.hasPiRole(experiment['id']):
raise AuthorizationError('User %s cannot modify users for experiment %s.' % (self.getSessionUsername(),experimentName))
response = self.userInfoSessionControllerImpl.addUserExperimentRole(username, roleName, experimentName).getFullJsonRep()
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def deleteUserExperimentRole(self, username, roleName, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
if not roleName:
raise InvalidRequest('Invalid role name provided.')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']) and not self.hasPiRole(experiment['id']):
raise AuthorizationError('User %s cannot modify users for experiment %s.' % (self.getSessionUsername(),experimentName))
response = self.userInfoSessionControllerImpl.deleteUserExperimentRole(username, roleName, experimentName).getFullJsonRep()
return response
#!/usr/bin/env python
#
# User route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from userInfoSessionController import UserInfoSessionController
class UserRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
userInfoSessionController = UserInfoSessionController()
# Define routes.
routes = [
# Get user info list
{
'name' : 'getUsers',
'path' : '%s/users' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'getUsers',
'method' : ['GET']
},
# Get user by id
{
'name' : 'getUserById',
'path' : '%s/users/:(id)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'getUserById',
'method' : ['GET']
},
# Get user by username
{
'name' : 'getUserByUsername',
'path' : '%s/usersByUsername/:(username)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'getUserByUsername',
'method' : ['GET']
},
# Add user system role
{
'name' : 'addUserSystemRole',
'path' : '%s/userSystemRoles/:(username)/:(roleName)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'addUserSystemRole',
'method' : ['POST']
},
# Delete user system role
{
'name' : 'deleteUserSystemRole',
'path' : '%s/userSystemRoles/:(username)/:(roleName)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'deleteUserSystemRole',
'method' : ['DELETE']
},
# Add user experiment role
{
'name' : 'addUserExperimentRole',
'path' : '%s/userExperimentRoles/:(username)/:(roleName)/:(experimentName)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'addUserExperimentRole',
'method' : ['POST']
},
# Delete user experiment role
{
'name' : 'deleteUserExperimentRole',
'path' : '%s/userExperimentRoles/:(username)/:(roleName)/:(experimentName)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'deleteUserExperimentRole',
'method' : ['DELETE']
},
]
return routes
all:
for d in $(SUBDIRS); do $(MAKE) -C $$d $(ARCH); done
dist:
for d in $(SUBDIRS); do $(MAKE) -C $$d $(ARCH); done
clean:
for d in $(SUBDIRS); do $(MAKE) -C $$d clean; done
distclean:
for d in $(SUBDIRS); do $(MAKE) -C $$d distclean; done
for d in $(SUBDIRS); do rm -f `find $$d -name 'RELEASE.local'`; done
tidy: distclean