src/python/dm/daq_web_service/service/impl/uploadTracker.py

+#!/usr/bin/env python
+
+import os
+from dm.common.constants import dmProcessingStatus
+from dm.common.objects.uploadInfo import UploadInfo
+from dm.common.utility.objectTracker import ObjectTracker
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+
+class UploadTracker(ObjectTracker):
+
+    # Cache configuration
+    objectClass = UploadInfo
+    cacheSize = 100
+
+    def __init__(self, *args, **kwargs):
+        ObjectTracker.__init__(self, args, kwargs)
+        self.activeUploadDict = {}
+
+    def checkForActiveUpload(self, experiment, dataDirectory):
+        experimentName = experiment.get('name')
+        dataDir = os.path.normpath(dataDirectory)
+        activeUploadKey = experimentName + dataDir
+        uploadId = self.activeUploadDict.get(activeUploadKey)
+        if uploadId:
+            uploadInfo = self.get(uploadId)
+            if uploadInfo is not None:
+                uploadInfo.updateStatus()
+                if uploadInfo.get('status') in dmProcessingStatus.DM_ACTIVE_PROCESSING_STATUS_LIST:
+                    raise ObjectAlreadyExists('Upload id %s is already active for experiment %s in data directory %s.' % (uploadId, experimentName, dataDir))
+                del self.activeUploadDict[activeUploadKey]
+
+    def startUpload(self, uploadId, uploadInfo):
+        experimentName = uploadInfo.get('experimentName') 
+        dataDirectory = uploadInfo.get('dataDirectory')
+        dataDir = os.path.normpath(dataDirectory)
+        activeUploadKey = experimentName + dataDir
+        self.activeUploadDict[activeUploadKey] = uploadId
+        self.put(uploadId, uploadInfo)
+
+    def getUploadInfo(self, id):
+        return self.get(id)
+
+    def getUploadInfos(self, status=None):
+        uploadInfoList = self.getAll()
+        filteredUploadInfoList = []
+        for uploadInfo in uploadInfoList:
+            uploadInfo.updateStatus()
+            if status is None or status == dmProcessingStatus.DM_PROCESSING_STATUS_ANY or uploadInfo.get('status', '') == status:
+                filteredUploadInfoList.append(uploadInfo)
+        return filteredUploadInfoList
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    pass
+
+
+

src/python/dm/daq_web_service/service/impl/userInfoControllerImpl.py

+#!/usr/bin/env python
+
+#
+# Implementation for user info controller.
+#
+
+#######################################################################
+
+from dm.common.objects.dmObject import DmObject
+from dm.common.objects.dmObjectManager import DmObjectManager
+from dm.common.db.api.userDbApi import UserDbApi
+
+#######################################################################
+
+class UserInfoControllerImpl(DmObjectManager):
+    """ User info controller implementation class. """
+
+    def __init__(self):
+        DmObjectManager.__init__(self)
+        self.userDbApi = UserDbApi()
+
+    def getUsers(self):
+        return self.userDbApi.getUsers()
+
+    def getUserById(self, id):
+        return self.userDbApi.getUserById(id)
+
+    def getUserByUsername(self, username):
+        return self.userDbApi.getUserByUsername(username)
+

src/python/dm/daq_web_service/service/impl/watchdogFileSystemObserverAgent.py

+#!/usr/bin/env python
+
+from dm.common.utility.osUtility import OsUtility
+
+from watchdog.observers.polling import PollingObserver
+from fileSystemObserverAgent import FileSystemObserverAgent
+from dmFileSystemEventHandler import DmFileSystemEventHandler
+
+class WatchdogFileSystemObserverAgent(FileSystemObserverAgent):
+
+    DAQ_PERMISSIONS_MODE = 0777
+
+    def __init__(self):
+        FileSystemObserverAgent.__init__(self)
+        self.observer = PollingObserver()
+        self.observedWatchDict = {}
+
+    def createDirectory(self, dataDirectory):
+        try:
+            OsUtility.createDir(dataDirectory, mode=self.DAQ_PERMISSIONS_MODE)
+        except Exception, ex:
+            self.logger.warn('Unable to create directory %s: %s' % (dataDirectory, ex))
+
+    def getFiles(self, dataDirectory):
+        return OsUtility.findFilesAsDict(dataDirectory)
+
+    def startObservingPath(self, dataDirectory, experiment):
+        self.logger.debug('Starting observer for %s' % dataDirectory)
+        eventHandler = DmFileSystemEventHandler(self.fileSystemObserver, dataDirectory, experiment)
+        observedWatch = self.observer.schedule(eventHandler, dataDirectory, recursive=True)
+        self.observedWatchDict[dataDirectory] = observedWatch
+
+    def stopObservingPath(self, dataDirectory, experiment):
+        observedWatch = self.observedWatchDict.get(dataDirectory)
+        if observedWatch:
+            self.logger.debug('Stopping observer for %s' % dataDirectory)
+            self.observer.unschedule(observedWatch)
+            del self.observedWatchDict[dataDirectory]
+        else:
+            self.logger.debug('Observer for %s is not active' % dataDirectory)
+
+    def start(self):
+        self.logger.debug('Starting watchdog observer agent')
+        self.observer.start()
+
+    def stop(self):
+        self.logger.debug('Stopping watchdog observer agent')
+        self.observer.stop()
+        self.observer.join()
+

src/python/dm/ds_web_service/api/authDsApi.py

+#!/usr/bin/env python
+
+import os
+import json
+import urllib
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
+from dm.common.objects.dmSession import DmSession
+from dsRestApi import DsRestApi
+
+class AuthDsApi(DsRestApi):
+   
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        DsRestApi.__init__(self, username, password, host, port, protocol)
+
+    @DsRestApi.execute
+    def getAuthorizationPrincipal(self, username):
+        if username is None:
+            raise InvalidRequest('Username must be provided.')
+        url = '%s/authorizationPrincipals/%s' % (self.getContextRoot(), username)
+        self.logger.debug('Retrieving principal for user %s' % (username))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return AuthorizationPrincipal(responseData)
+
+    @DsRestApi.execute
+    def addSession(self, sessionId, sessionInfo):
+        if sessionId is None:
+            raise InvalidRequest('Session id must be provided.')
+        if sessionInfo is None:
+            raise InvalidRequest('Session info must be provided.')
+        url = '%s/sessions/%s' % (self.getContextRoot(), sessionId)
+        url += '?sessionInfo=%s' % Encoder.encode(json.dumps(sessionInfo))
+        responseData = self.sendSessionRequest(url=url, method='POST')
+        return DmSession(responseData)
+
+    @DsRestApi.execute
+    def checkSession(self, sessionId):
+        if sessionId is None:
+            raise InvalidRequest('Session id must be provided.')
+        url = '%s/sessions/%s' % (self.getContextRoot(), sessionId)
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return DmSession(responseData)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = AuthDsApi('dm', 'dm', 'zagreb.svdev.net', 22236, 'http')
+    print api.authenticateUser('sveseli', 'sv')
+        
+
+
+

src/python/dm/ds_web_service/api/dsRestApi.py

+#!/usr/bin/env python
+
+from dm.common.api.dmRestApi import DmRestApi
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DsRestApi(DmRestApi):
+    """ Base DS DM REST api class. """
+
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        if host == None:
+            host = ConfigurationManager.getInstance().getDsWebServiceHost()
+        if port == None:
+            port = ConfigurationManager.getInstance().getDsWebServicePort()
+        DmRestApi.__init__(self, username, password, host, port, protocol)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass
+
+

src/python/dm/ds_web_service/api/dsRestApiFactory.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DsRestApiFactory:
+
+    CONFIG_SECTION_NAME = 'DsRestApiFactory'
+    USERNAME_KEY = 'username'
+    PASSWORD_FILE_KEY = 'passwordfile'
+    HOST_KEY = 'host'
+    PORT_KEY = 'port'
+    PROTOCOL_KEY = 'protocol'
+
+    __logger = None
+    __username = None
+    __password = None
+    __host = None
+    __port = None
+    __protocol = None
+
+    @classmethod
+    def getLogger(cls):
+        if cls.__logger is None:
+            cls.__logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return cls.__logger
+
+    @classmethod
+    def __getConfiguration(cls):
+        if cls.__username is None:
+            cls.__username = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.USERNAME_KEY)
+            cls.__password = open(ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.PASSWORD_FILE_KEY)).read().strip()
+            cls.__host = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.HOST_KEY)
+            cls.__port = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.PORT_KEY)
+            cls.__protocol = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.PROTOCOL_KEY)
+        return (cls.__username, cls.__password, cls.__host, cls.__port, cls.__protocol)
+
+    @classmethod
+    def getUserDsApi(cls):
+        from userDsApi import UserDsApi
+        (username, password, host, port, protocol) = cls.__getConfiguration()
+        api = UserDsApi(username, password, host, port, protocol)
+        return api
+
+    @classmethod
+    def getAuthDsApi(cls):
+        from authDsApi import AuthDsApi
+        (username, password, host, port, protocol) = cls.__getConfiguration()
+        api = AuthDsApi(username, password, host, port, protocol)
+        return api
+
+    @classmethod
+    def getExperimentDsApi(cls):
+        from experimentDsApi import ExperimentDsApi
+        try:
+            (username, password, host, port, protocol) = cls.__getConfiguration()
+        except Exception, ex:
+            print ex
+            raise
+        api = ExperimentDsApi(username, password, host, port, protocol)
+        return api
+
+    @classmethod
+    def getFileDsApi(cls):
+        from fileDsApi import FileDsApi
+        (username, password, host, port, protocol) = cls.__getConfiguration()
+        api = FileDsApi(username, password, host, port, protocol)
+        return api
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    pass
+

src/python/dm/ds_web_service/api/experimentDsApi.py

+#!/usr/bin/env python
+
+import os
+import urllib
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.objects.experiment import Experiment
+from dm.common.objects.experimentType import ExperimentType
+from dsRestApi import DsRestApi
+
+class ExperimentDsApi(DsRestApi):
+    
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        DsRestApi.__init__(self, username, password, host, port, protocol)
+
+    @DsRestApi.execute
+    def getExperimentTypes(self):
+        url = '%s/experimentTypes' % (self.getContextRoot())
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, ExperimentType)
+
+    @DsRestApi.execute
+    def getExperiments(self):
+        url = '%s/experiments' % (self.getContextRoot())
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, Experiment)
+
+    @DsRestApi.execute
+    def getExperimentsByStation(self, stationName):
+        url = '%s/experimentsByStation/%s' % (self.getContextRoot(), stationName)
+        if not stationName:
+            raise InvalidRequest('Experiment station name must be provided.')
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, Experiment)
+
+    @DsRestApi.execute
+    def getExperimentByName(self, name):
+        url = '%s/experimentsByName/%s' % (self.getContextRoot(), name)
+        if name is None or not len(name):
+            raise InvalidRequest('Experiment name must be provided.')
+        responseDict = self.sendSessionRequest(url=url, method='GET')
+        return Experiment(responseDict)
+
+    @DsRestApi.execute
+    def getExperimentById(self, id):
+        url = '%s/experimentsById/%s' % (self.getContextRoot(), id)
+        if id is None:
+            raise InvalidRequest('Experiment id must be provided.')
+        responseDict = self.sendSessionRequest(url=url, method='GET')
+        return Experiment(responseDict)
+
+    @DsRestApi.execute
+    def startExperiment(self, name):
+        url = '%s/experiments/start' % (self.getContextRoot())
+        if name is None or not len(name):
+            raise InvalidRequest('Experiment name must be provided.')
+        url += '?name=%s' % Encoder.encode(name)
+        responseDict = self.sendSessionRequest(url=url, method='PUT')
+        return Experiment(responseDict)
+
+    @DsRestApi.execute
+    def updateExperiment(self, name):
+        url = '%s/experiments/update' % (self.getContextRoot())
+        if name is None or not len(name):
+            raise InvalidRequest('Experiment name must be provided.')
+        url += '?name=%s' % Encoder.encode(name)
+        responseDict = self.sendSessionRequest(url=url, method='PUT')
+        return Experiment(responseDict)
+
+    @DsRestApi.execute
+    def stopExperiment(self, name):
+        url = '%s/experiments/stop' % (self.getContextRoot())
+        if name is None or not len(name):
+            raise InvalidRequest('Experiment name must be provided.')
+        url += '?name=%s' % Encoder.encode(name)
+        responseDict = self.sendSessionRequest(url=url, method='PUT')
+        return Experiment(responseDict)
+
+    @DsRestApi.execute
+    def addExperiment(self, name, stationName, typeName, description, startDate, endDate):
+        url = '%s/experiments' % (self.getContextRoot())
+        if not name:
+            raise InvalidRequest('Experiment name must be provided.')
+        url += '?name=%s' % Encoder.encode(name)
+        if not stationName:
+            raise InvalidRequest('Experiment station name must be provided.')
+        url += '&stationName=%s' % Encoder.encode(stationName)
+        if not typeName:
+            raise InvalidRequest('Experiment type must be provided.')
+        url += '&typeName=%s' % Encoder.encode(typeName)
+        if description is not None:
+            url += '&description=%s' % Encoder.encode(description)
+        if startDate is not None:
+            url += '&startDate=%s' % Encoder.encode(startDate)
+        if endDate is not None:
+            url += '&endDate=%s' % Encoder.encode(endDate)
+        responseDict = self.sendSessionRequest(url=url, method='POST')
+        return Experiment(responseDict)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = ExperimentDsApi('sveseli', 'sveseli', 'zagreb.svdev.net', 33336, 'http')
+    print api.startExperiment('experiment1')
+

src/python/dm/ds_web_service/api/fileDsApi.py

+#!/usr/bin/env python
+
+import os
+import urllib
+import json
+import getpass
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.fileMetadata import FileMetadata
+from dm.common.objects.directoryMetadata import DirectoryMetadata
+from dm.common.objects.experiment import Experiment
+from dm.common.utility.rsyncFileTransfer import RsyncFileTransfer
+from dsRestApi import DsRestApi
+
+class FileDsApi(DsRestApi):
+    
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        DsRestApi.__init__(self, username, password, host, port, protocol)
+
+    @DsRestApi.execute
+    def processFile(self, experimentFilePath, experimentName, fileInfo={}):
+        url = '%s/files/processFile' % (self.getContextRoot())
+        if not experimentFilePath:
+            raise InvalidRequest('Experiment file path must be provided.')
+        if not experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        fileInfo['experimentFilePath'] = experimentFilePath
+        fileInfo['experimentName'] = experimentName
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseDict = self.sendSessionRequest(url=url, method='POST')
+        return FileMetadata(responseDict)
+
+    @DsRestApi.execute
+    def statFile(self, experimentFilePath, experimentName, fileInfo={}):
+        url = '%s/files/statFile' % (self.getContextRoot())
+        if not experimentFilePath:
+            raise InvalidRequest('Experiment file path must be provided.')
+        if not experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        fileInfo['experimentFilePath'] = experimentFilePath
+        fileInfo['experimentName'] = experimentName
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseDict = self.sendSessionRequest(url=url, method='POST')
+        return FileMetadata(responseDict)
+
+    @DsRestApi.execute
+    def processDirectory(self, experimentDirectoryPath, experimentName, directoryInfo={}):
+        url = '%s/files/processDirectory' % (self.getContextRoot())
+        if not experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        directoryInfo['experimentDirectoryPath'] = experimentDirectoryPath
+        directoryInfo['experimentName'] = experimentName
+        url += '?directoryInfo=%s' % (Encoder.encode(json.dumps(directoryInfo)))
+        responseDict = self.sendSessionRequest(url=url, method='POST')
+        return DirectoryMetadata(responseDict)
+
+    @DsRestApi.execute
+    def download(self, experimentName, experimentFilePath='',  destDirectory='.'):
+        username = getpass.getuser()
+
+        # Initialize download
+        url = '%s/downloadAuthorizations/%s/%s' % (self.getContextRoot(), username, experimentName)
+        if not experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        self.logger.info('Authorizing download for user %s (experiment: %s)' % (username, experimentName))
+        responseDict = self.sendSessionRequest(url=url, method='POST')
+        experiment = Experiment(responseDict)
+
+        # Download
+        try:
+            storageDirectory = experiment.get('storageDirectory')
+            storageHost = experiment.get('storageHost')
+            src = '%s@%s:%s' % (username, storageHost, storageDirectory)
+            if experimentFilePath:
+                src = '%s/%s' % (src, experimentFilePath)
+            dest = destDirectory
+            
+            rsyncPath = '/tmp/rsync.%s.%s' % (username, experimentName)
+            flags = '-arvlP --rsync-path="%s"' % rsyncPath
+            fileTransfer = RsyncFileTransfer(src=src, dest=dest, flags=flags)
+            self.logger.info('Executing file download on behalf of %s (experiment: %s)' % (username, experimentName))
+            fileTransfer.execute()
+        finally:
+            # Finalize download
+            self.logger.info('Deleting download authorization for user %s (experiment: %s)' % (username, experimentName))
+            self.sendSessionRequest(url=url, method='DELETE')
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = FileDsApi('sveseli', 'sveseli', 'zagreb.svdev.net', 22236, 'http')
+    print api.processFile('file1', '/ESAF/exp1', 'exp1')
+

src/python/dm/ds_web_service/api/userDsApi.py

+#!/usr/bin/env python
+
+import os
+import urllib
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.userInfo import UserInfo
+from dm.common.objects.userExperimentRole import UserExperimentRole
+from dm.common.objects.userSystemRole import UserSystemRole
+from dsRestApi import DsRestApi
+
+class UserDsApi(DsRestApi):
+    
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        DsRestApi.__init__(self, username, password, host, port, protocol)
+
+    @DsRestApi.execute
+    def getUsers(self):
+        url = '%s/users' % (self.getContextRoot())
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, UserInfo)
+
+    @DsRestApi.execute
+    def getUserById(self, id):
+        if id is None:
+            raise InvalidRequest('User id must be provided.')
+        url = '%s/users/%s' % (self.getContextRoot(), id)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return UserInfo(responseData)
+
+    @DsRestApi.execute
+    def getUserByUsername(self, username):
+        if username is None:
+            raise InvalidRequest('Username must be provided.')
+        url = '%s/usersByUsername/%s' % (self.getContextRoot(), username)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return UserInfo(responseData)
+
+    @DsRestApi.execute
+    def addUserSystemRole(self, username, roleName, experimentStationName=None):
+        if not username:
+            raise InvalidRequest('Username must be provided.')
+        if not roleName:
+            raise InvalidRequest('Role name must be provided.')
+        url = '%s/userSystemRoles/%s/%s' % (self.getContextRoot(), username, roleName)
+        if experimentStationName:
+            url += '?experimentStationName=%s' % (experimentStationName)
+        responseData = self.sendSessionRequest(url=url, method='POST')
+        return UserSystemRole(responseData)
+
+    @DsRestApi.execute
+    def deleteUserSystemRole(self, username, roleName, experimentStationName=None):
+        if not username:
+            raise InvalidRequest('Username must be provided.')
+        if not roleName:
+            raise InvalidRequest('Role name must be provided.')
+        url = '%s/userSystemRoles/%s/%s' % (self.getContextRoot(), username, roleName)
+        if experimentStationName:
+            url += '?experimentStationName=%s' % (experimentStationName)
+        responseData = self.sendSessionRequest(url=url, method='DELETE')
+        return UserSystemRole(responseData)
+
+    @DsRestApi.execute
+    def addUserExperimentRole(self, username, roleName, experimentName):
+        if not username:
+            raise InvalidRequest('Username must be provided.')
+        if not experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        if not roleName:
+            raise InvalidRequest('Role name must be provided.')
+        url = '%s/userExperimentRoles/%s/%s/%s' % (self.getContextRoot(), username, roleName, experimentName)
+        responseData = self.sendSessionRequest(url=url, method='POST')
+        return UserExperimentRole(responseData)
+
+    @DsRestApi.execute
+    def deleteUserExperimentRole(self, username, roleName, experimentName):
+        if not username:
+            raise InvalidRequest('Username must be provided.')
+        if not experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        if not roleName:
+            raise InvalidRequest('Role name must be provided.')
+        url = '%s/userExperimentRoles/%s/%s/%s' % (self.getContextRoot(), username, roleName, experimentName)
+        responseData = self.sendSessionRequest(url=url, method='DELETE')
+        return UserExperimentRole(responseData)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass
+
+
+

src/python/dm/ds_web_service/cli/addExperimentCli.py

+#!/usr/bin/env python
+
+from dm.aps_bss.api.apsBssApi import ApsBssApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
+
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.utility.configurationManager import ConfigurationManager
+from dsWebServiceSessionCli import DsWebServiceSessionCli
+
+class AddExperimentCli(DsWebServiceSessionCli):
+    def __init__(self):
+        DsWebServiceSessionCli.__init__(self)
+        configManager = ConfigurationManager.getInstance()
+        self.allowedExperimentTypes = configManager.getAllowedExperimentTypes()
+        allowedExperimentTypesHelp = ''
+        if self.allowedExperimentTypes:
+            allowedExperimentTypesHelp = ' Allowed types: %s' % self.allowedExperimentTypes
+            
+
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+        self.addOption('', '--station', dest='stationName', help='Experiment station name, can also be set via DM_STATION_NAME environment variable.')
+        self.addOption('', '--type', dest='typeName', help='Experiment type name.%s' % allowedExperimentTypesHelp)
+        self.addOption('', '--type-id', dest='typeId', help='Experiment type id (may be given instead of type name; type id is ignored if both type name and id are provided).')
+        self.addOption('', '--description', dest='description', help='Experiment description.')
+        self.addOption('', '--start-date', dest='startDate', help='Experiment start date in format DD-MMM-YY.')
+        self.addOption('', '--end-date', dest='endDate', help='Experiment end date in format DD-MMM-YY.')
+        self.addOption('', '--users', dest='users', help='Comma-separated list of DM usernames to be added to the new experiment as users.')
+        self.addOption('', '--proposal-id', dest='proposalId', help='Beamline proposal id. If specified, all users listed on the proposal will be added to the new experiment.')
+        self.addOption('', '--run', dest='runName', help='Run name. If not specified, current run name is assumed for beamline proposal.')
+        self.addOption('', '--bss-login-file', dest='bssLoginFile', help='BSS login file. Login file may also be specified via environment variable DM_BSS_LOGIN_FILE.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.getTypeName() is None:
+            raise InvalidRequest('Experiment type name must be provided.')
+        if self.getStationName() is None:
+            raise InvalidRequest('Experiment station name must be provided.')
+        # If allowed experiment types is not set, there are no restrictions
+        if self.allowedExperimentTypes:
+            if self.getTypeName() not in self.allowedExperimentTypes.split(','):
+                raise InvalidRequest('Experiment type %s is not allowed on this station. Allowed types are: %s.' % (self.getTypeName(), self.allowedExperimentTypes))
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getStationName(self):
+        stationName = self.options.stationName
+        if not stationName:
+            configManager = ConfigurationManager.getInstance()
+            stationName = configManager.getStationName()
+        return stationName
+
+    def getTypeName(self):
+        typeName = self.options.typeName
+        if not typeName:
+            if self.options.typeId:
+                api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+                experimentType = api.getExperimentTypeById(typeId)
+                typeName = experimentType.get('name')
+                self.options.typeName = typeName
+        return typeName
+
+    def getDescription(self):
+        return self.options.description
+
+    def getStartDate(self):
+        return self.options.startDate
+
+    def getEndDate(self):
+        return self.options.endDate
+
+    def getProposalId(self):
+        proposalId = self.options.proposalId
+        if proposalId:
+            proposalId = int(proposalId)
+        return proposalId
+
+    def getUsers(self):
+        # Return list of users and beamline managers that can access data
+        users = self.options.users
+        if users:
+            users = users.split(',')
+        else:
+            users = []
+        beamlineManagers = ConfigurationManager.getInstance().getBeamlineManagers()
+        if beamlineManagers:
+            beamlineManagers = beamlineManagers.split(',')
+        else:
+            beamlineManagers = []
+        # Remove duplicates by converting into set
+        return list(set(users+beamlineManagers))
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-add-experiment 
+        --experiment=EXPERIMENTNAME --station=STATIONNAME --type=TYPENAME|--type-id=TYPEID
+        [--description=DESCRIPTION] 
+        [--start-date=STARTDATE] 
+        [--end-date=ENDDATE]
+        [--users=USERS]
+        [--proposal-id=PROPOSALID]
+        [--run=RUNNAME]
+        [--bss-login-file=BSSLOGINFILE]
+
+Description:
+    Add new experiment to the DM database. If list of users or proposal id is specified, this command will also add roles for all users listed on the proposal.
+        """)
+        self.checkArgs()
+        dsExperimentApi = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        dsUserApi = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+
+        description = self.getDescription()
+        proposalId = self.getProposalId()
+        experimenters = []
+        if proposalId:
+            bssApi = ApsBssApi(loginFile=self.options.bssLoginFile)
+            proposal = bssApi.getBeamlineProposal(proposalId=proposalId, runName=self.options.runName)
+            experimenters = proposal.get('experimenters', [])
+            if not description:
+                description = '%s (Proposal id: %s)' % (proposal['title'], proposalId)
+
+
+        users = self.getUsers()
+        pis = []
+        for experimenter in experimenters:
+            badge = int(experimenter['badge'])
+            if not badge:
+                #print 'Skipping user %s due to invalid badge.' % lastName 
+                continue
+            username = 'd%s' % badge
+
+            # Clasify user
+            if experimenter.get('piFlag') == 'Y':
+                if not pis.count(username):
+                    pis.append(username)
+                if users.count(username):
+                    users.remove(username)
+            else:
+                if not users.count(username):
+                    users.append(username)
+
+        for username in users+pis:
+            # Check that user exists
+            dsUserApi.getUserByUsername(username)
+           
+
+        # Everything looks good, add experiment and users
+        experiment = dsExperimentApi.addExperiment(self.getExperimentName(), self.getStationName(), self.getTypeName(), description, self.getStartDate(), self.getEndDate())
+
+        # Add pis.
+        experimentName = experiment['name']
+        roleName = 'PI'
+        for username in pis:
+            dsUserApi.addUserExperimentRole(username, roleName, experimentName)
+        roleName = 'User'
+        for username in users:
+            dsUserApi.addUserExperimentRole(username, roleName, experimentName)
+
+        if len(users+pis):
+            experiment = dsExperimentApi.getExperimentByName(experimentName)
+        print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = AddExperimentCli()
+    cli.run()
+

src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py

+#!/usr/bin/env python
+
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.ds_web_service.api.userDsApi import UserDsApi
+from dsWebServiceSessionCli import DsWebServiceSessionCli
+
+class AddUserExperimentRoleCli(DsWebServiceSessionCli):
+    def __init__(self):
+        DsWebServiceSessionCli.__init__(self)
+        self.addOption('', '--username', dest='username', help='User username.')
+        self.addOption('', '--role', dest='roleName', help='Role name can be either \'PI\' or \'User\'.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if not self.options.username:
+            raise InvalidRequest('Username must be provided.')
+        if not self.options.experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        if not self.options.roleName:
+            raise InvalidRequest('Role name must be provided.')
+
+    def getUsername(self):
+        return self.options.username
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getRoleName(self):
+        return self.options.roleName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-add-user-experiment-role --username=USERNAME 
+        --role=ROLENAME
+        --experiment=EXPERIMENTNAME
+
+Description:
+    Assigns experiment role to the given user.
+        """)
+        self.checkArgs()
+        api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        userExperimentRole = api.addUserExperimentRole(self.getUsername(), self.getRoleName(), self.getExperimentName())
+        print userExperimentRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = AddUserExperimentRoleCli()
+    cli.run()
+

src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py

+#!/usr/bin/env python
+
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.ds_web_service.api.userDsApi import UserDsApi
+from dsWebServiceSessionCli import DsWebServiceSessionCli
+
+class AddUserSystemRoleCli(DsWebServiceSessionCli):
+    def __init__(self):
+        DsWebServiceSessionCli.__init__(self)
+        self.addOption('', '--username', dest='username', help='User username.')
+        self.addOption('', '--role', dest='roleName', help='Role name can be either \'Administrator\' or \'Manager\'.')
+        self.addOption('', '--station', dest='experimentStationName', help='Experiment station name (should not be provided for administrator system role).')
+
+    def checkArgs(self):
+        if not self.options.username:
+            raise InvalidRequest('Username must be provided.')
+        if not self.options.roleName:
+            raise InvalidRequest('Role name must be provided.')
+
+    def getUsername(self):
+        return self.options.username
+
+    def getExperimentStationName(self):
+        return self.options.experimentStationName
+
+    def getRoleName(self):
+        return self.options.roleName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-add-user-experiment-role --username=USERNAME 
+        --role=ROLENAME
+        [--station=EXPERIMENTSTATIONNAME]
+
+Description:
+    Assigns system role to the given user. The 'Administrator' role cannot be accompanied with experiment station, while the 'Manager' role requires it.
+        """)
+        self.checkArgs()
+        api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        userSystemRole = api.addUserSystemRole(self.getUsername(), self.getRoleName(), self.getExperimentStationName())
+        print userSystemRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = AddUserSystemRoleCli()
+    cli.run()
+

src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py

+#!/usr/bin/env python
+
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.ds_web_service.api.userDsApi import UserDsApi
+from dsWebServiceSessionCli import DsWebServiceSessionCli
+
+class DeleteUserExperimentRoleCli(DsWebServiceSessionCli):
+    def __init__(self):
+        DsWebServiceSessionCli.__init__(self)
+        self.addOption('', '--username', dest='username', help='User username.')
+        self.addOption('', '--role', dest='roleName', help='Role name can be either \'PI\' or \'User\'.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if not self.options.username:
+            raise InvalidRequest('Username must be provided.')
+        if not self.options.experimentName:
+            raise InvalidRequest('Experiment name must be provided.')
+        if not self.options.roleName:
+            raise InvalidRequest('Role name must be provided.')
+
+    def getUsername(self):
+        return self.options.username
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getRoleName(self):
+        return self.options.roleName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-delete-user-experiment-role --username=USERNAME 
+        --role=ROLENAME
+        --experiment=EXPERIMENTNAME
+
+Description:
+    Deletes experiment role from the given user.
+        """)
+        self.checkArgs()
+        api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        userExperimentRole = api.deleteUserExperimentRole(self.getUsername(), self.getRoleName(), self.getExperimentName())
+        #print 'Deleted User Experiment Role:\n\t%s' % userExperimentRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = DeleteUserExperimentRoleCli()
+    cli.run()
+

src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py

+#!/usr/bin/env python
+
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.ds_web_service.api.userDsApi import UserDsApi
+from dsWebServiceSessionCli import DsWebServiceSessionCli
+
+class DeleteUserSystemRoleCli(DsWebServiceSessionCli):
+    def __init__(self):
+        DsWebServiceSessionCli.__init__(self)
+        self.addOption('', '--username', dest='username', help='User username.')
+        self.addOption('', '--role', dest='roleName', help='Role name can be either \'Administrator\' or \'Manager\'.')
+        self.addOption('', '--station', dest='experimentStationName', help='Experiment station name (should not be provided for administrator system role).')
+
+    def checkArgs(self):
+        if not self.options.username:
+            raise InvalidRequest('Username must be provided.')
+        if not self.options.roleName:
+            raise InvalidRequest('Role name must be provided.')
+
+    def getUsername(self):
+        return self.options.username
+
+    def getExperimentStationName(self):
+        return self.options.experimentStationName
+
+    def getRoleName(self):
+        return self.options.roleName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-delete-user-experiment-role --username=USERNAME 
+        --role=ROLENAME
+        [--station=EXPERIMENTSTATIONNAME]
+
+Description:
+    Removes system role from the given user. The 'Administrator' role cannot be accompanied with experiment station, while the 'Manager' role requires it.
+        """)
+        self.checkArgs()
+        api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        userSystemRole = api.deleteUserSystemRole(self.getUsername(), self.getRoleName(), self.getExperimentStationName())
+        #print userSystemRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = DeleteUserSystemRoleCli()
+    cli.run()
+

src/python/dm/ds_web_service/cli/downloadCli.py

+#!/usr/bin/env python
+
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.ds_web_service.api.fileDsApi import FileDsApi
+from dsWebServiceSessionCli import DsWebServiceSessionCli
+
+class DownloadCli(DsWebServiceSessionCli):
+    def __init__(self):
+        DsWebServiceSessionCli.__init__(self)
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+        self.addOption('', '--relative-path', dest='experimentFilePath', default='', help='Experiment (relative) file path. If omitted, all experiment data will be downloaded.')
+        self.addOption('', '--destination-directory', dest='destinationDirectory', default='.', help='Destination directory. If omitted, files will be downloaded to current directory.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getExperimentFilePath(self):
+        return self.options.experimentFilePath
+
+    def getDestinationDirectory(self):
+        return self.options.destinationDirectory
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-download --experiment=EXPERIMENTNAME 
+        [--relative-path=EXPERIMENTFILEPATH]
+        [--destination-directory=DESTINATIONDIRECTORY]
+
+Description:
+    Downloads experiment files.
+        """)
+        self.checkArgs()
+        api = FileDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        api.download(self.getExperimentName(), self.getExperimentFilePath(), self.getDestinationDirectory())
+        #print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = DownloadCli()
+    cli.run()
+

src/python/dm/ds_web_service/cli/dsWebServiceCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmRestCli import DmRestCli
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DsWebServiceCli(DmRestCli):
+    """ DM DS web service cli class. """
+
+    def __init__(self, validArgCount=0):
+        DmRestCli.__init__(self, validArgCount)
+
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getDsWebServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getDsWebServicePort()
+                        `

src/python/dm/ds_web_service/cli/dsWebServiceSessionCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmRestSessionCli import DmRestSessionCli
+from dm.common.utility.osUtility import OsUtility
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class DsWebServiceSessionCli(DmRestSessionCli):
+    """ DM DS web service session cli class. """
+
+    DEFAULT_SESSION_CACHE_FILE = OsUtility.getUserHomeDir() + '/.dm/.ds.session.cache'
+
+    def __init__(self, validArgCount=0):
+        DmRestSessionCli.__init__(self, validArgCount)
+        ConfigurationManager.getInstance().setSessionCacheFile(DsWebServiceSessionCli.DEFAULT_SESSION_CACHE_FILE)
+
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getDsWebServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getDsWebServicePort()
+