Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • DM/dm-docs
  • hammonds/dm-docs
  • hparraga/dm-docs
3 results
Show changes
Showing
with 1195 additions and 0 deletions
#authenticateUser!/usr/bin/env python
import json
import cherrypy
from dm.common.utility.encoder import Encoder
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.service.dmSessionController import DmSessionController
from dm.ds_web_service.service.impl.authSessionControllerImpl import AuthSessionControllerImpl
class AuthSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.authSessionControllerImpl = AuthSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getAuthorizationPrincipal(self, username, **kwargs):
if not len(username):
raise InvalidRequest('Invalid username provided.')
response = self.authSessionControllerImpl.getAuthorizationPrincipal(username).getFullJsonRep()
self.logger.debug('Returning authorization principal for %s' % (username))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def addSession(self, sessionId, **kwargs):
if not len(sessionId):
raise InvalidRequest('Invalid session id provided.')
sessionInfo = kwargs.get('sessionInfo')
if not sessionInfo:
raise InvalidRequest('Missing session object.')
sessionInfo = json.loads(Encoder.decode(sessionInfo))
response = self.authSessionControllerImpl.addSession(sessionId, sessionInfo).getFullJsonRep()
self.logger.debug('Session %s added to cache' % (sessionInfo))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def checkSession(self, sessionId, **kwargs):
if not len(sessionId):
raise InvalidRequest('Invalid session id provided.')
response = self.authSessionControllerImpl.checkSession(sessionId).getFullJsonRep()
self.logger.debug('Verified session %s' % (sessionId))
return response
#!/usr/bin/env python
#
# DM DS Web Service
#
####################################################################
from dm.common.service.dmRestWebServiceBase import DmRestWebServiceBase
from dm.common.utility.dmModuleManager import DmModuleManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.processing.fileProcessingManager import FileProcessingManager
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
from dsWebServiceRouteMapper import DsWebServiceRouteMapper
####################################################################
class DsWebService(DmRestWebServiceBase):
def __init__(self):
DmRestWebServiceBase.__init__(self, DsWebServiceRouteMapper)
def initDmModules(self):
self.logger.debug('Initializing dm modules')
# Add modules that will be started.
moduleManager = DmModuleManager.getInstance()
moduleManager.addModule(ExperimentManager.getInstance())
moduleManager.addModule(FileProcessingManager.getInstance())
self.logger.debug('Initialized dm modules')
def getDefaultServerHost(self):
return ConfigurationManager.getInstance().getServiceHost()
def getDefaultServerPort(self):
return ConfigurationManager.getInstance().getServicePort()
####################################################################
# Run service
if __name__ == '__main__':
ConfigurationManager.getInstance().setServiceName('ds-web-service')
service = DsWebService();
service.run()
#!/usr/bin/env python
#
# Route mapper for DM DS web service.
#
import sys
import os
import cherrypy
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.service.loginRouteDescriptor import LoginRouteDescriptor
from userRouteDescriptor import UserRouteDescriptor
from experimentRouteDescriptor import ExperimentRouteDescriptor
from fileRouteDescriptor import FileRouteDescriptor
from authRouteDescriptor import AuthRouteDescriptor
class DsWebServiceRouteMapper:
@classmethod
def setupRoutes(cls):
""" Setup RESTFul routes. """
logger = LoggingManager.getInstance().getLogger(cls.__name__)
contextRoot = ConfigurationManager.getInstance().getContextRoot()
logger.debug('Using context root: %s' % contextRoot)
# Get routes.
routes = LoginRouteDescriptor.getRoutes()
routes += AuthRouteDescriptor.getRoutes()
routes += UserRouteDescriptor.getRoutes()
routes += ExperimentRouteDescriptor.getRoutes()
routes += FileRouteDescriptor.getRoutes()
# Add routes to dispatcher.
d = cherrypy.dispatch.RoutesDispatcher()
for route in routes:
logger.debug('Connecting route: %s' % route)
d.connect(route['name'], route['path'], action=route['action'], controller=route['controller'], conditions=dict(method=route['method']))
return d
#!/usr/bin/env python
#
# Experiment route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from experimentSessionController import ExperimentSessionController
class ExperimentRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
experimentSessionController = ExperimentSessionController()
# Define routes.
routes = [
# Get experiment types
{
'name' : 'getExperimentTypes',
'path' : '%s/experimentTypes' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentTypes',
'method' : ['GET']
},
# Get experiments
{
'name' : 'getExperiments',
'path' : '%s/experiments' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperiments',
'method' : ['GET']
},
# Get experiment
{
'name' : 'getExperimentById',
'path' : '%s/experiments/:(id)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentById',
'method' : ['GET']
},
# Get experiment
{
'name' : 'getExperimentByName',
'path' : '%s/experimentsByName/:(name)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'getExperimentByName',
'method' : ['GET']
},
# Add experiment
{
'name' : 'addExperiment',
'path' : '%s/experiments' % contextRoot,
'controller' : experimentSessionController,
'action' : 'addExperiment',
'method' : ['POST']
},
# Start experiment
{
'name' : 'startExperiment',
'path' : '%s/experiments/start' % contextRoot,
'controller' : experimentSessionController,
'action' : 'startExperiment',
'method' : ['PUT']
},
# Update experiment
{
'name' : 'updateExperiment',
'path' : '%s/experiments/update' % contextRoot,
'controller' : experimentSessionController,
'action' : 'updateExperiment',
'method' : ['PUT']
},
# Stop experiment
{
'name' : 'stopExperiment',
'path' : '%s/experiments/stop' % contextRoot,
'controller' : experimentSessionController,
'action' : 'stopExperiment',
'method' : ['PUT']
},
# Authorize download for a given UNIX username
{
'name' : 'authorizeDownload',
'path' : '%s/downloadAuthorizations/:(username)/:(experimentName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'authorizeDownload',
'method' : ['POST']
},
# Revoke authorization download for a given UNIX username
{
'name' : 'deauthorizeDownload',
'path' : '%s/downloadAuthorizations/:(username)/:(experimentName)' % contextRoot,
'controller' : experimentSessionController,
'action' : 'deauthorizeDownload',
'method' : ['DELETE']
},
]
return routes
#!/usr/bin/env python
import cherrypy
from dm.common.service.dmSessionController import DmSessionController
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.utility.encoder import Encoder
from dm.ds_web_service.service.impl.experimentSessionControllerImpl import ExperimentSessionControllerImpl
class ExperimentSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.experimentSessionControllerImpl = ExperimentSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isLoggedIn())
@DmSessionController.execute
def getExperimentTypes(self, **kwargs):
return self.listToJson(self.experimentSessionControllerImpl.getExperimentTypes())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getExperiments(self, **kwargs):
return self.listToJson(self.experimentSessionControllerImpl.getExperiments())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getExperimentByName(self, name, **kwargs):
response = self.experimentSessionControllerImpl.getExperimentByName(name).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getExperimentById(self, id, **kwargs):
response = self.experimentSessionControllerImpl.getExperimentById(id).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def addExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
experimentTypeId = kwargs.get('experimentTypeId')
if experimentTypeId is None:
raise InvalidRequest('Missing experiment type id.')
description = kwargs.get('description')
if description is not None:
description = Encoder.decode(description)
startDate = kwargs.get('startDate')
if startDate is not None:
startDate = Encoder.decode(startDate)
endDate = kwargs.get('endDate')
if endDate is not None:
endDate = Encoder.decode(endDate)
response = self.experimentSessionControllerImpl.addExperiment(name, experimentTypeId, description, startDate, endDate).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def startExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
response = self.experimentSessionControllerImpl.startExperiment(name).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def updateExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
response = self.experimentSessionControllerImpl.updateExperiment(name).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def stopExperiment(self, **kwargs):
name = kwargs.get('name')
if name is None or not len(name):
raise InvalidRequest('Missing experiment name.')
name = Encoder.decode(name)
response = self.experimentSessionControllerImpl.stopExperiment(name).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def authorizeDownload(self, username, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
response = self.experimentSessionControllerImpl.authorizeDownload(username, experimentName).getFullJsonRep()
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def deauthorizeDownload(self, username, experimentName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
response = self.experimentSessionControllerImpl.deauthorizeDownload(username, experimentName).getFullJsonRep()
return response
#!/usr/bin/env python
#
# File route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from fileSessionController import FileSessionController
class FileRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
fileSessionController = FileSessionController()
# Define routes.
routes = [
# Process file
{
'name' : 'processFile',
'path' : '%s/files/processFile' % contextRoot,
'controller' : fileSessionController,
'action' : 'processFile',
'method' : ['POST']
},
# Stat file
{
'name' : 'statFile',
'path' : '%s/files/statFile' % contextRoot,
'controller' : fileSessionController,
'action' : 'statFile',
'method' : ['POST']
},
# Process directory
{
'name' : 'processDirectory',
'path' : '%s/files/processDirectory' % contextRoot,
'controller' : fileSessionController,
'action' : 'processDirectory',
'method' : ['POST']
},
]
return routes
#!/usr/bin/env python
import cherrypy
import json
from dm.common.service.dmSessionController import DmSessionController
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.utility.encoder import Encoder
from dm.ds_web_service.service.impl.fileSessionControllerImpl import FileSessionControllerImpl
class FileSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.fileSessionControllerImpl = FileSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def processFile(self, **kwargs):
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
if not fileInfo.has_key('experimentFilePath'):
raise InvalidRequest('Experiment file path is missing.')
if not fileInfo.has_key('experimentName'):
raise InvalidRequest('Experiment name is missing.')
response = self.fileSessionControllerImpl.processFile(fileInfo).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def statFile(self, **kwargs):
encodedFileInfo = kwargs.get('fileInfo')
if not encodedFileInfo:
raise InvalidRequest('Invalid file info provided.')
fileInfo = json.loads(Encoder.decode(encodedFileInfo))
if not fileInfo.has_key('experimentFilePath'):
raise InvalidRequest('Experiment file path is missing.')
if not fileInfo.has_key('experimentName'):
raise InvalidRequest('Experiment name is missing.')
response = self.fileSessionControllerImpl.statFile(fileInfo).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def processDirectory(self, **kwargs):
encodedDirectoryInfo = kwargs.get('directoryInfo')
if not encodedDirectoryInfo:
raise InvalidRequest('Invalid directory info provided.')
directoryInfo = json.loads(Encoder.decode(encodedDirectoryInfo))
if not directoryInfo.has_key('experimentName'):
raise InvalidRequest('Experiment name is missing.')
response = self.fileSessionControllerImpl.processDirectory(directoryInfo).getFullJsonRep()
self.logger.debug('Returning: %s' % response)
return response
#!/usr/bin/env python
#
# Implementation for user info controller.
#
import datetime
import cherrypy
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.objects.dmSession import DmSession
from dm.common.db.api.userDbApi import UserDbApi
from dm.common.exceptions.objectNotFound import ObjectNotFound
class AuthSessionControllerImpl(DmObjectManager):
""" User info controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.userDbApi = UserDbApi()
def getAuthorizationPrincipal(self, username):
user = self.userDbApi.getUserWithPasswordByUsername(username)
principal = AuthorizationPrincipal(name=username, token=user.get('password'))
principal.setRole(dmRole.DM_USER_ROLE)
principal.setUserInfo(user)
for userSystemRoleName in user.get('userSystemRoleNameList', []):
if userSystemRoleName == dmRole.DM_ADMIN_ROLE:
principal.setRole(dmRole.DM_ADMIN_ROLE)
return principal
def addSession(self, sessionId, sessionInfo):
sessionCache = cherrypy.session.cache
self.logger.debug('Session cache length: %s' % (len(sessionCache)))
sessionCache[sessionId] = (sessionInfo, datetime.datetime.now())
self.logger.debug('Session cache: %s' % (sessionCache))
return DmSession(sessionInfo)
def checkSession(self, sessionId):
sessionCache = cherrypy.session.cache
sessionTuple = sessionCache.get(sessionId)
if not sessionTuple:
raise ObjectNotFound('Session %s not found in cache.' % sessionId)
sessionInfo = sessionTuple[0]
oldTimestamp = sessionTuple[1]
newTimestamp = datetime.datetime.now()
self.logger.debug('Updated timestamp from %s to %s for session id %s' % (oldTimestamp, newTimestamp, sessionId))
sessionCache[sessionId] = (sessionInfo, newTimestamp)
return DmSession(sessionInfo)
#!/usr/bin/env python
import threading
import time
import os
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.utility.singleton import Singleton
from dm.common.utility.osUtility import OsUtility
from dm.common.utility.valueUtility import ValueUtility
from dm.common.utility.objectUtility import ObjectUtility
from dm.common.utility.threadingUtility import ThreadingUtility
from dm.common.utility.fileUtility import FileUtility
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.common.processing.fileProcessingManager import FileProcessingManager
from dm.common.exceptions.objectNotFound import ObjectNotFound
from dm.common.exceptions.invalidRequest import InvalidRequest
from pathTracker import PathTracker
class ExperimentManager(Singleton):
CONFIG_SECTION_NAME = 'ExperimentManager'
STORAGE_DIRECTORY_KEY = 'storagedirectory'
STORAGE_ID_KEY = 'storageid'
MANAGE_STORAGE_PERMISSIONS_KEY = 'managestoragepermissions'
PLATFORM_UTILITY_KEY = 'platformutility'
RSYNC_SCRIPT_PERMISSIONS_MODE = 0755
FILE_PERMISSIONS_MODE = 0640
FILE_PERMISSIONS_MODE_STRING = '0640'
DIR_PERMISSIONS_MODE = 0750
DIRECTORY_PROCESSING_DELAY_IN_SECONDS = 1
# Singleton.
__instanceLock = threading.RLock()
__instance = None
def __init__(self):
ExperimentManager.__instanceLock.acquire()
try:
if ExperimentManager.__instance:
return
ExperimentManager.__instance = self
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
self.logger.debug('Initializing')
self.lock = threading.RLock()
self.experimentDbApi = ExperimentDbApi()
self.pathTracker = PathTracker()
self.platformUtility = None
self.__configure()
self.fileProcessingManager = FileProcessingManager.getInstance()
self.logger.debug('Initialization complete')
finally:
ExperimentManager.__instanceLock.release()
def __configure(self):
cm = ConfigurationManager.getInstance()
configItems = cm.getConfigItems(ExperimentManager.CONFIG_SECTION_NAME)
self.logger.debug('Got config items: %s' % configItems)
self.storageDirectory = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.STORAGE_DIRECTORY_KEY)
self.storageId = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.STORAGE_ID_KEY)
self.manageStoragePermissions = ValueUtility.toBoolean(cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.MANAGE_STORAGE_PERMISSIONS_KEY))
platformUtility = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.PLATFORM_UTILITY_KEY)
if platformUtility:
(moduleName,className,constructor) = cm.getModuleClassConstructorTuple(platformUtility)
self.logger.debug('Creating platform utility class %s' % className)
self.platformUtility = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.logger.debug('Manage storage permissions: %s' % self.manageStoragePermissions)
def __getExperimentStorageDataDirectory(self, experiment):
experimentTypeName = experiment.get('experimentType').get('rootDataPath')
experimentName = experiment.get('name')
storageDirectory = '%s/%s/%s' % (self.storageDirectory, experimentTypeName, experimentName)
storageDirectory = os.path.normpath(storageDirectory)
return storageDirectory
@ThreadingUtility.synchronize
def updateExperimentWithStorageDataDirectory(self, experiment):
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
storageHost = ConfigurationManager.getInstance().getHost()
experiment['storageDirectory'] = storageDirectory
experiment['storageHost'] = storageHost
experiment['storageUrl'] = '%s://%s%s' % (self.storageId, storageHost, storageDirectory)
return storageDirectory
def addUserToGroup(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentWithUsers(experimentName)
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.addUserToGroup(username, experimentName)
def deleteUserFromGroup(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentWithUsers(experimentName)
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
experimentUsers = experiment.get('experimentUsernameList', [])
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
def createRsyncScript(self, username, experimentName):
fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
self.logger.debug('Creating rsync script %s' % (fileName))
f = open(fileName, 'w')
f.write('#!/bin/sh\n')
f.write('exec sg %s "rsync $*"\n' % experimentName)
f.close()
OsUtility.chmodPath(fileName, fileMode=self.RSYNC_SCRIPT_PERMISSIONS_MODE)
def deleteRsyncScript(self, username, experimentName):
fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
self.logger.debug('Removing rsync script %s' % (fileName))
OsUtility.removeFile(fileName)
def authorizeDownload(self, username, experimentName):
self.logger.debug('Authorizing download for %s from experiment %s' % (username, experimentName))
experiment = self.experimentDbApi.getExperimentByName(experimentName)
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.addLocalUserToGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
self.createRsyncScript(username, experimentName)
return experiment
def deauthorizeDownload(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentByName(experimentName)
self.logger.debug('De-authorizing download for %s from experiment %s' % (username, experimentName))
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.deleteLocalUserFromGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
self.deleteRsyncScript(username, experimentName)
return experiment
def createExperimentGroup(self, experiment):
experimentName = experiment.get('name')
storageDirectory = experiment.get('storageDirectory')
# Create experiment group
self.platformUtility.createGroup(experimentName)
self.logger.debug('Setting permissions for %s to %s' % (storageDirectory, self.DIR_PERMISSIONS_MODE))
OsUtility.chmodPath(storageDirectory, dirMode=self.DIR_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (storageDirectory, experimentName))
self.platformUtility.changePathGroupOwner(storageDirectory, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(storageDirectory, ownerUpdateTime)
# Add users to group
experimentUsers = experiment.get('experimentUsernameList', [])
self.logger.debug('Found experiment users: %s', experimentUsers)
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
def updateExperimentGroupUsers(self, experiment):
experimentName = experiment.get('name')
experimentUsers = experiment.get('experimentUsernameList', [])
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
@ThreadingUtility.synchronize
def createExperimentDataDirectory(self, experiment):
experimentName = experiment.get('name')
storageDirectory = self.__getExperimentStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.logger.debug('Data directory %s for experiment %s already exists' % (storageDirectory, experimentName))
else:
self.logger.debug('Creating data directory for experiment %s: %s' % (experimentName, storageDirectory))
OsUtility.createDir(storageDirectory)
storageHost = ConfigurationManager.getInstance().getHost()
experiment['storageDirectory'] = storageDirectory
experiment['storageHost'] = storageHost
experiment['storageUrl'] = '%s://%s%s' % (self.storageId, storageHost, storageDirectory)
if self.manageStoragePermissions:
self.createExperimentGroup(experiment)
def processExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
filePath = os.path.join(storageDirectory, experimentFilePath)
fileInfo['filePath'] = filePath
fileInfo['experiment'] = experiment
if not os.path.exists(filePath):
self.logger.error('File path %s does not exist' % filePath)
return
if self.manageStoragePermissions:
self.logger.debug('Setting permissions for %s to %s' % (filePath,self.FILE_PERMISSIONS_MODE_STRING))
OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (filePath, experimentName))
self.platformUtility.changePathGroupOwner(filePath, experimentName)
# Recursively modify subdirectory permissions
dirPath = os.path.dirname(filePath)
while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
if self.pathTracker.get(dirPath) is None:
self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
self.platformUtility.changePathGroupOwner(dirPath, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(dirPath, ownerUpdateTime)
else:
self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
dirPath = os.path.dirname(dirPath)
self.logger.debug('Processing file path %s (fileInfo: %s)' % (filePath, fileInfo))
self.fileProcessingManager.processFile(fileInfo)
def statExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
filePath = os.path.join(storageDirectory, experimentFilePath)
fileInfo['filePath'] = filePath
if os.path.exists(filePath):
self.logger.debug('Getting stat for file path %s' % (filePath))
FileUtility.statFile(filePath, fileInfo)
FileUtility.getMd5Sum(filePath, fileInfo)
self.logger.debug('File info after stat: %s' % str(fileInfo))
else:
self.logger.debug('File path %s does not exist' % filePath)
raise ObjectNotFound('File %s does not exist' % filePath)
def processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
storageDirectory = experiment.get('storageDirectory')
directoryPath = os.path.join(storageDirectory, experimentDirectoryPath)
directoryInfo['directoryPath'] = directoryPath
directoryInfo['experiment'] = experiment
if not self.manageStoragePermissions:
self.logger.error('Skipping permission management for directory path %s' % directoryPath)
return
self.logger.debug('Processing directory path %s in background' % (directoryPath))
timer = threading.Timer(self.DIRECTORY_PROCESSING_DELAY_IN_SECONDS, self.__processExperimentDirectory, args=[experimentDirectoryPath, experiment, directoryInfo])
timer.start()
def __processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
experimentName = experiment.get('name')
storageDirectory = experiment.get('storageDirectory')
directoryPath = directoryInfo.get('directoryPath')
if not os.path.exists(directoryPath):
self.logger.error('Directory path %s does not exist' % directoryPath)
return
# Modify ownership
self.logger.debug('Processing directory path %s (directoryInfo: %s)' % (directoryPath, directoryInfo))
self.logger.debug('Modifying permissions for directory %s to %s' % (directoryPath, self.DIR_PERMISSIONS_MODE))
OsUtility.chmodPath(directoryPath, dirMode=self.DIR_PERMISSIONS_MODE)
self.logger.debug('Changing group owner for %s to %s' % (directoryPath, experimentName))
self.platformUtility.recursivelyChangePathGroupOwner(directoryPath, experimentName)
# Recursively modify subdirectory permissions
dirPath = os.path.dirname(directoryPath)
while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
if self.pathTracker.get(dirPath) is None:
self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
self.platformUtility.changePathGroupOwner(dirPath, experimentName)
ownerUpdateTime = time.time()
self.pathTracker.put(dirPath, ownerUpdateTime)
else:
self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
dirPath = os.path.dirname(dirPath)
# Update file permissions
self.logger.debug('Changing permissions for all files in %s for experiment %s' % (directoryPath, experimentName))
self.platformUtility.chmodPathForFilesInDirectory(directoryPath, self.FILE_PERMISSIONS_MODE_STRING)
@ThreadingUtility.synchronize
def start(self):
self.logger.debug('Started experiment manager')
@ThreadingUtility.synchronize
def stop(self):
self.logger.debug('Stopped experiment manager')
####################################################################
# Testing
if __name__ == '__main__':
em = ExperimentManager.getInstance()
print em
#!/usr/bin/env python
#
# Implementation for experiment session controller.
#
import time
from dm.common.objects.experiment import Experiment
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
class ExperimentSessionControllerImpl(DmObjectManager):
""" Experiment session controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.experimentDbApi = ExperimentDbApi()
def getExperimentTypes(self):
experimentTypeList = self.experimentDbApi.getExperimentTypes()
return experimentTypeList
def getExperiments(self):
experimentList = self.experimentDbApi.getExperiments()
return experimentList
def getExperimentByName(self, name):
experiment = self.experimentDbApi.getExperimentWithUsers(name)
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def getExperimentById(self, id):
experiment = self.experimentDbApi.getExperimentById(id)
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def addExperiment(self, name, experimentTypeId, description, startDate, endDate):
experiment = self.experimentDbApi.addExperiment(name, experimentTypeId, description, startDate, endDate)
return experiment
def startExperiment(self, name):
experiment = self.experimentDbApi.getExperimentWithUsers(name)
if experiment.get('startDate') is None:
experiment2 = self.experimentDbApi.setExperimentStartDateToNow(name)
experiment['startDate'] = experiment2['startDate']
ExperimentManager.getInstance().createExperimentDataDirectory(experiment)
return experiment
def updateExperiment(self, name):
experiment = self.experimentDbApi.getExperimentWithUsers(name)
if experiment.get('startDate') is not None:
ExperimentManager.getInstance().updateExperimentGroupUsers(experiment)
return experiment
def stopExperiment(self, name):
experiment = self.experimentDbApi.getExperimentByName(name)
if experiment.get('endDate') is None:
experiment = self.experimentDbApi.setExperimentEndDateToNow(name)
ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
return experiment
def authorizeDownload(self, username, experimentName):
return ExperimentManager.getInstance().authorizeDownload(username, experimentName)
def deauthorizeDownload(self, username, experimentName):
return ExperimentManager.getInstance().deauthorizeDownload(username, experimentName)
#!/usr/bin/env python
#
# Implementation for file session controller.
#
import time
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.objects.fileMetadata import FileMetadata
from dm.common.objects.directoryMetadata import DirectoryMetadata
from dm.common.db.api.experimentDbApi import ExperimentDbApi
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
class FileSessionControllerImpl(DmObjectManager):
""" File session controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.experimentDbApi = ExperimentDbApi()
def processFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
ExperimentManager.getInstance().processExperimentFile(experimentFilePath, experiment, fileInfo)
return FileMetadata(fileInfo)
def statFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
ExperimentManager.getInstance().statExperimentFile(experimentFilePath, experiment, fileInfo)
return FileMetadata(fileInfo)
def processDirectory(self, directoryInfo):
experimentDirectoryPath = directoryInfo.get('experimentDirectoryPath', '')
experimentName = directoryInfo.get('experimentName')
experiment = self.experimentDbApi.getExperimentByName(experimentName)
ExperimentManager.getInstance().processExperimentDirectory(experimentDirectoryPath, experiment, directoryInfo)
return DirectoryMetadata(directoryInfo)
#!/usr/bin/env python
from dm.common.utility.objectTracker import ObjectTracker
class PathTracker(ObjectTracker):
# Cache configuration
pass
####################################################################
# Testing
if __name__ == '__main__':
et = PathTracker.getInstance()
print et
et2 = PathTracker.getInstance()
print et2
et.put('xyz', 1)
print et.get('xyz')
print et.get('xyz2')
#!/usr/bin/env python
#
# Implementation for user info controller.
#
from dm.common.objects.dmObject import DmObject
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.db.api.userDbApi import UserDbApi
from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
class UserInfoSessionControllerImpl(DmObjectManager):
""" User info controller implementation class. """
def __init__(self):
DmObjectManager.__init__(self)
self.userDbApi = UserDbApi()
def getUsers(self):
return self.userDbApi.getUsers()
def getUserById(self, id):
return self.userDbApi.getUserById(id)
def getUserByUsername(self, username):
return self.userDbApi.getUserByUsername(username)
def addUserExperimentRole(self, username, experimentName, roleName):
userInfo = self.userDbApi.addUserExperimentRole(username, experimentName, roleName)
ExperimentManager.getInstance().addUserToGroup(username, experimentName)
return userInfo
def deleteUserExperimentRole(self, username, experimentName, roleName):
userInfo = self.userDbApi.deleteUserExperimentRole(username, experimentName, roleName)
ExperimentManager.getInstance().deleteUserFromGroup(username, experimentName)
return userInfo
#!/usr/bin/env python
import cherrypy
from dm.common.service.dmSessionController import DmSessionController
from dm.ds_web_service.service.impl.userInfoSessionControllerImpl import UserInfoSessionControllerImpl
class UserInfoSessionController(DmSessionController):
def __init__(self):
DmSessionController.__init__(self)
self.userInfoSessionControllerImpl = UserInfoSessionControllerImpl()
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getUsers(self, **kwargs):
return self.listToJson(self.userInfoSessionControllerImpl.getUsers())
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getUserById(self, id, **kwargs):
if not id:
raise InvalidRequest('Invalid id provided.')
response = self.userInfoSessionControllerImpl.getUserById(id).getFullJsonRep()
self.logger.debug('Returning user info for %s: %s' % (id,response))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def getUserByUsername(self, username, **kwargs):
if not len(username):
raise InvalidRequest('Invalid username provided.')
response = self.userInfoSessionControllerImpl.getUserByUsername(username).getFullJsonRep()
self.logger.debug('Returning user info for %s: %s' % (username,response))
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def addUserExperimentRole(self, username, experimentName, roleName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
if not roleName:
raise InvalidRequest('Invalid role name provided.')
response = self.userInfoSessionControllerImpl.addUserExperimentRole(username, experimentName, roleName).getFullJsonRep()
return response
@cherrypy.expose
@DmSessionController.require(DmSessionController.isAdministrator())
@DmSessionController.execute
def deleteUserExperimentRole(self, username, experimentName, roleName, **kwargs):
if not username:
raise InvalidRequest('Invalid username provided.')
if not experimentName:
raise InvalidRequest('Invalid experiment name provided.')
if not roleName:
raise InvalidRequest('Invalid role name provided.')
response = self.userInfoSessionControllerImpl.deleteUserExperimentRole(username, experimentName, roleName).getFullJsonRep()
return response
#!/usr/bin/env python
#
# User route descriptor.
#
from dm.common.utility.configurationManager import ConfigurationManager
from userInfoSessionController import UserInfoSessionController
class UserRouteDescriptor:
@classmethod
def getRoutes(cls):
contextRoot = ConfigurationManager.getInstance().getContextRoot()
# Static instances shared between different routes
userInfoSessionController = UserInfoSessionController()
# Define routes.
routes = [
# Get user info list
{
'name' : 'getUsers',
'path' : '%s/users' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'getUsers',
'method' : ['GET']
},
# Get user by id
{
'name' : 'getUserById',
'path' : '%s/users/:(id)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'getUserById',
'method' : ['GET']
},
# Get user by username
{
'name' : 'getUserByUsername',
'path' : '%s/usersByUsername/:(username)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'getUserByUsername',
'method' : ['GET']
},
# Add user experiment role
{
'name' : 'addUserExperimentRole',
'path' : '%s/usersByExperiment/:(username)/:(experimentName)/:(roleName)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'addUserExperimentRole',
'method' : ['POST']
},
# Delete user experiment role
{
'name' : 'deleteUserExperimentRole',
'path' : '%s/usersByExperiment/:(username)/:(experimentName)/:(roleName)' % contextRoot,
'controller' : userInfoSessionController,
'action' : 'deleteUserExperimentRole',
'method' : ['DELETE']
},
]
return routes
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="src" path="resources"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="/home/phoebus/BFROSIK/trunk/tools/AccountSynchronizer/lib/ojdbc7.jar"/>
<classpathentry kind="lib" path="/home/phoebus/BFROSIK/trunk/tools/AccountSynchronizer/lib/postgresql-9.4-1201.jdbc4.jar"/>
<classpathentry kind="lib" path="/home/phoebus/BFROSIK/trunk/tools/AccountSynchronizer/lib/unboundid-ldapsdk-me.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>accountSynchronizer</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
Main-Class: gov.anl.aps.dm.sync.Synchronizer
Class-Path: . ../../lib/postgresql-9.4-1201.jdbc4.jar ../../lib/ojdbc7.jar
### log congiguration
log.file = accountSynchronizer%g.log
log.limit = 6000
log.count = 2
###
### Oracle database settings
###
oracle.database.connection = jdbc:oracle:thin:@ra.aps.anl.gov:1527:aps1
oracle.database.username = glob_conn
#oracle.database.password =
oracle.database.table = DCC.FL$03_BL_APV_VIEW_V2
dm.database.connection = jdbc:postgresql://127.0.0.1:11136/dm
dm.database.username = dm
### dm.database.password =
# The user name prefix that gets append to every badge number.
user.userid.prefix = d