Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • DM/dm-docs
  • hammonds/dm-docs
  • hparraga/dm-docs
3 results
Show changes
Showing
with 940 additions and 0 deletions
#!/usr/bin/env python
import abc
class FileProcessor:
DEFAULT_NUMBER_OF_RETRIES = 0
DEFAULT_RETRY_WAIT_PERIOD_IN_SECONDS = 60
def __init__(self):
self.configDict = {}
@abc.abstractmethod
def processFile(self, fileInfo):
return NotImplemented
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
return {}
def configure(self):
# Use this method for processor configuration
pass
def setConfigKeyValue(self, key, value):
self.configDict[key] = value
def getConfigKeyValue(self, key):
return self.configDict.get(key)
def setNumberOfRetries(self, nRetries):
self.configDict['numberOfRetries'] = nRetries
def getNumberOfRetries(self):
return self.configDict.get('numberOfRetries', self.DEFAULT_NUMBER_OF_RETRIES)
def setRetryWaitPeriodInSeconds(self, waitPeriod):
self.configDict['retryWaitPeriodInSeconds'] = waitPeriod
def getRetryWaitPeriodInSeconds(self):
return self.configDict.get('retryWaitPeriodInSeconds', self.DEFAULT_RETRY_WAIT_PERIOD_IN_SECONDS)
#!/usr/bin/env python
import os
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.dmSubprocess import DmSubprocess
from dm.common.exceptions.invalidArgument import InvalidArgument
from dm.common.exceptions.invalidRequest import InvalidRequest
from fileProcessor import FileProcessor
class FileTransferPlugin(FileProcessor):
def __init__(self, command, src=None, dest=None):
FileProcessor.__init__(self)
self.src = src
self.dest = dest
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
if command is None or not len(command):
raise InvalidArgument('File transfer command must be non-empty string.')
self.command = command
self.subprocess = None
def checkUploadFilesForProcessing(self, filePaths, uploadInfo):
pass
def processFile(self, fileInfo):
filePath = fileInfo.get('filePath')
dataDirectory = fileInfo.get('dataDirectory')
storageHost = fileInfo.get('storageHost')
storageDirectory = fileInfo.get('storageDirectory')
destUrl = self.getDestUrl(filePath, dataDirectory, storageHost, storageDirectory)
srcUrl = self.getSrcUrl(filePath, dataDirectory)
self.start(srcUrl, destUrl)
def getSrcUrl(self, filePath, dataDirectory):
# Use relative path with respect to data directory as a source
os.chdir(dataDirectory)
srcUrl = os.path.relpath(filePath, dataDirectory)
return srcUrl
def getDestUrl(self, filePath, dataDirectory, storageHost, storageDirectory):
destUrl = '%s:%s' % (storageHost, storageDirectory)
return destUrl
def getFullCommand(self, src, dest):
return '%s %s %s' % (self.command, src, dest)
def setSrc(self, src):
self.src = src
def setDest(self, dest):
self.dest = dest
def start(self, src=None, dest=None):
# Use preconfigured source if provided source is None
fileSrc = src
if src is None:
fileSrc = self.src
# Use provided destination only if preconfigured destination is None
# Plugins may have desired destination preconfigured for all files
fileDest = self.dest
if self.dest is None:
fileDest = dest
if not fileSrc or not fileDest:
raise InvalidRequest('Both source and destination must be non-empty strings.')
self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand(fileSrc, fileDest))
return self.subprocess.run()
def wait(self):
if self.subprocess:
return self.subprocess.wait()
return None
def poll(self):
if self.subprocess:
return self.subprocess.poll()
return None
def getStdOut(self):
if self.subprocess:
return self.subprocess.getStdOut()
return None
def getStdErr(self):
if self.subprocess:
return self.subprocess.getStdErr()
return None
def getExitStatus(self):
if self.subprocess:
return self.subprocess.getExitStatus()
return None
def reset(self):
self.subprocess = None
#######################################################################
# Testing.
if __name__ == '__main__':
ft = FileTransfer('rsync -arv', '/tmp/xyz', '/tmp/xyz2')
ft.start()
print 'StdOut: ', ft.getStdOut()
print 'StdErr: ', ft.getStdErr()
print 'Exit Status: ', ft.getExitStatus()
#!/usr/bin/env python
import os
from fileTransferPlugin import FileTransferPlugin
from dm.common.utility.fileUtility import FileUtility
from dm.common.utility.ftpUtility import FtpUtility
from dm.common.exceptions.fileProcessingError import FileProcessingError
from dm.common.utility.dmSubprocess import DmSubprocess
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
class GridftpFileTransferPlugin(FileTransferPlugin):
DEFAULT_COMMAND = 'globus-url-copy -c -cd -r -vb -tcp-bs 512K -p 1 -sync -sync-level 2'
DEFAULT_PORT = 2811
def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False):
FileTransferPlugin.__init__(self, command, src, dest)
self.dsFileApi = DsRestApiFactory.getFileRestApi()
self.localMd5Sum = localMd5Sum
self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal
def getSrcUrl(self, filePath, dataDirectory):
(scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
if scheme:
srcUrl = '%s://%s:%s/%s' % (scheme, host, port, filePath)
elif self.src is None:
srcUrl = 'file://%s' % filePath
else:
srcUrl = '%s/%s' % (self.src,filePath)
return srcUrl
def getDestUrl(self, filePath, dataDirectory, storageHost, storageDirectory):
(scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
dirName = os.path.dirname(os.path.relpath(filePath, dirPath)).strip()
fileName = os.path.basename(filePath)
destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName)
return destUrl
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
storageHost = uploadInfo['storageHost']
storageDirectory = uploadInfo['storageDirectory']
dataDirectory = uploadInfo['dataDirectory']
#(scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(storageUrl, defaultPort=self.DEFAULT_PORT)
ftpUtility = FtpUtility(storageHost, self.DEFAULT_PORT)
storageFilePathsDict = ftpUtility.getFiles(storageDirectory, {})
pluginFilePathsDict = {}
filePaths = filePathsDict.keys()
for filePath in filePaths:
filePathDict = filePathsDict.get(filePath)
experimentFilePath = os.path.relpath(filePath, dataDirectory)
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
storageFilePathDict = storageFilePathsDict.get(storageFilePath)
if not storageFilePathDict:
# remote directory does not have the file
pluginFilePathsDict[filePath] = filePathDict
else:
fSize = filePathDict.get('Size')
sfSize = storageFilePathDict.get('Size')
# check size
if not fSize or not sfSize or fSize != sfSize:
pluginFilePathsDict[filePath] = filePathDict
else:
# sizes are the same, check modify time
mTime = filePathDict.get('Modify')
sfTime = storageFilePathDict.get('Modify')
if not mTime or not sfTime or mTime > sfTime:
pluginFilePathsDict[filePath] = filePathDict
self.logger.debug('Number of original files: %s, number of plugin files: %s', len(filePathsDict), len(pluginFilePathsDict))
return pluginFilePathsDict
def processFile(self, fileInfo):
filePath = fileInfo.get('filePath')
dataDirectory = fileInfo.get('dataDirectory')
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
storageHost = fileInfo.get('storageHost')
storageDirectory = fileInfo.get('storageDirectory')
destUrl = self.getDestUrl(filePath, dataDirectory, storageHost, storageDirectory)
srcUrl = self.getSrcUrl(filePath, dataDirectory)
# Calculate checksum
if self.localMd5Sum:
FileUtility.statFile(filePath, fileInfo)
FileUtility.getMd5Sum(filePath, fileInfo)
# Transfer file
self.logger.debug('Starting transfer: %s' % fileInfo)
self.start(srcUrl, destUrl)
# Get remote checksum
if self.remoteMd5Sum:
fileInfo2 = {}
fileInfo2['experimentFilePath'] = experimentFilePath
fileInfo2['experimentName'] = experimentName
fileMetadata = self.dsFileApi.statFile(experimentFilePath, experimentName, fileInfo2)
# If we have not done md5 locally, update file info
if not self.md5Sum:
fileInfo['md5Sum'] = fileMetadata.get('md5Sum')
# Verify checksum
if fileMetadata.get('md5Sum') != fileInfo.get('md5Sum'):
self.logger.error('Checksum mismatch for file: %s' % filePath)
raise FileProcessingError('Checksum mismatch detected for file: %s' % filePath)
self.logger.debug('Checksum test passed for file %s' % filePath)
# Remove file
if self.deleteOriginal:
self.logger.debug('Deleting file %s' % filePath)
OsUtility.removeFile(srcUrl)
#######################################################################
# Testing.
if __name__ == '__main__':
ft = GridftpFileTransferPlugin('/tmp/xyz', '/tmp/xyz2')
ft.start()
print 'StdOut: ', ft.getStdOut()
print 'StdErr: ', ft.getStdErr()
print 'Exit Status: ', ft.getExitStatus()
#!/usr/bin/env python
import os
import time
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi
class MongoDbFileCatalogPlugin(FileProcessor):
def __init__(self):
FileProcessor.__init__(self)
self.fileMongoDbApi = FileMongoDbApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
self.logger.debug('Processing file %s for experiment %s' % (fileInfo, experimentName))
daqInfo = fileInfo.get('daqInfo')
storageDirectory = daqInfo.get('storageDirectory')
storageHost = daqInfo.get('storageHost')
storageUrl = daqInfo.get('storageUrl')
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
fileProcessingTime = time.time()
fileProcessingTimeStamp = TimeUtility.formatLocalTimeStamp(fileProcessingTime)
# Prepare catalogging entry
fileInfo2 = {}
for key in ['md5Sum', 'fileSize', 'fileCreationTime', 'fileCreationTimeStamp']:
if fileInfo.has_key(key):
fileInfo2[key] = fileInfo.get(key, '')
fileLocations = fileInfo.get('fileLocations', [])
fileLocations.append('%s/%s' % (storageUrl, experimentFilePath))
fileInfo2['fileName'] = os.path.basename(experimentFilePath)
fileInfo2['storageDirectory'] = storageDirectory
fileInfo2['storageUrl'] = storageUrl
fileInfo2['storageHost'] = storageHost
fileInfo2['storageFilePath'] = storageFilePath
fileInfo2['experimentFilePath'] = experimentFilePath
fileInfo2['experimentName'] = experimentName
fileInfo2['fileProcessingTime'] = fileProcessingTime
fileInfo2['fileProcessingTimeStamp'] = fileProcessingTimeStamp
fileInfo2['fileLocations'] = fileLocations
self.logger.debug('Daq info: %s' % (daqInfo))
fileInfo2.update(daqInfo)
if daqInfo.has_key('id'):
fileInfo2['daqId'] = daqInfo.get('id')
del fileInfo2['id']
self.logger.debug('File %s catalog entry: %s' % (experimentFilePath, str(fileInfo2)))
self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
from fileTransferPlugin import FileTransferPlugin
from dm.common.utility.osUtility import OsUtility
from dm.common.utility.fileUtility import FileUtility
from dm.common.exceptions.fileProcessingError import FileProcessingError
from dm.common.utility.dmSubprocess import DmSubprocess
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
class RsyncFileTransferPlugin(FileTransferPlugin):
DEFAULT_COMMAND = 'rsync -arvlPR'
DRY_RUN_COMMAND = 'rsync -arvlP'
def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False):
FileTransferPlugin.__init__(self, command, src, dest)
self.dsFileApi = DsRestApiFactory.getFileRestApi()
self.localMd5Sum = localMd5Sum
self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
storageDirectory = uploadInfo['storageDirectory']
storageHost = uploadInfo['storageHost']
dataDirectory = uploadInfo['dataDirectory']
dryRunCommand = '%s --dry-run %s/ %s:%s' % (self.DRY_RUN_COMMAND, dataDirectory, storageHost, storageDirectory)
subprocess = DmSubprocess.getSubprocess(dryRunCommand)
subprocess.run()
lines = subprocess.getStdOut().split('\n')
pluginFilePathsDict = {}
pathBase = dataDirectory
filePaths = filePathsDict.keys()
for line in lines:
if line.endswith(os.sep):
continue
filePath = os.path.join(pathBase, line)
if filePath in filePaths:
pluginFilePathsDict[filePath] = filePathsDict.get(filePath)
self.logger.debug('Number of original files: %s, number of plugin files: %s', len(filePathsDict), len(pluginFilePathsDict))
return pluginFilePathsDict
def processFile(self, fileInfo):
filePath = fileInfo.get('filePath')
dataDirectory = fileInfo.get('dataDirectory')
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
storageHost = fileInfo.get('storageHost')
storageDirectory = fileInfo.get('storageDirectory')
destUrl = self.getDestUrl(filePath, dataDirectory, storageHost, storageDirectory)
# Use relative path with respect to data directory as a source
os.chdir(dataDirectory)
srcUrl = self.getSrcUrl(filePath, dataDirectory)
# Calculate checksum
if self.localMd5Sum:
FileUtility.statFile(filePath, fileInfo)
FileUtility.getMd5Sum(filePath, fileInfo)
# Transfer file
self.logger.debug('Starting transfer: %s' % fileInfo)
self.start(srcUrl, destUrl)
# Get remote checksum
if self.remoteMd5Sum:
fileInfo2 = {}
fileInfo2['experimentFilePath'] = experimentFilePath
fileInfo2['experimentName'] = experimentName
fileMetadata = self.dsFileApi.statFile(experimentFilePath, experimentName, fileInfo2)
# If we have not done md5 locally, update file info
if not self.md5Sum:
fileInfo['md5Sum'] = fileMetadata.get('md5Sum')
# Verify checksum
if fileMetadata.get('md5Sum') != fileInfo.get('md5Sum'):
self.logger.error('Checksum mismatch for file: %s' % filePath)
raise FileProcessingError('Checksum mismatch detected for file: %s' % filePath)
self.logger.debug('Checksum test passed for file %s' % filePath)
# Remove file
if self.deleteOriginal:
self.logger.debug('Deleting file %s' % filePath)
OsUtility.removeFile(srcUrl)
#######################################################################
# Testing.
if __name__ == '__main__':
ft = RsyncFileTransferPlugin(command='rsync -arvlPR --exclude "*.svn" --exclude "*.pyc"')
ft.checkUploadFilesForProcessing(['/home/sveseli/Work/DM/dev/src/python/dm/common/processing/plugins/fileProcessor.py'], {'storageDirectory' : '/opt/DM/dev', 'storageHost' : 'dm@dmstorage', 'dataDirectory' : '/home/sveseli/Work/DM/dev/src'})
#!/usr/bin/env python
import os
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.osUtility import OsUtility
from dm.common.utility.fileUtility import FileUtility
from dm.common.exceptions.fileProcessingError import FileProcessingError
from dm.common.processing.plugins.fileTransferPlugin import FileTransferPlugin
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
class RsyncWithChecksumAndDeleteFileTransferPlugin(FileTransferPlugin):
COMMAND = 'rsync -arvlPR'
def __init__(self, src=None, dest=None):
FileTransferPlugin.__init__(self, self.COMMAND, src, dest)
self.dsFileApi = DsRestApiFactory.getFileRestApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
filePath = fileInfo.get('filePath')
dataDirectory = fileInfo.get('dataDirectory')
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
storageHost = fileInfo.get('storageHost')
storageDirectory = fileInfo.get('storageDirectory')
destUrl = self.getDestUrl(filePath, dataDirectory, storageHost, storageDirectory)
# Use relative path with respect to data directory as a source
os.chdir(dataDirectory)
srcUrl = self.getSrcUrl(filePath, dataDirectory)
# Calculate checksum
FileUtility.statFile(filePath, fileInfo)
FileUtility.getMd5Sum(filePath, fileInfo)
self.logger.debug('File info before transfer: %s' % fileInfo)
# Transfer file
self.start(srcUrl, destUrl)
# Get remote checksum
fileInfo2 = {}
fileInfo2['experimentFilePath'] = experimentFilePath
fileInfo2['experimentName'] = experimentName
fileMetadata = self.dsFileApi.statFile(experimentFilePath, experimentName, fileInfo2)
self.logger.debug('File stat after transfer: %s' % fileMetadata)
# Verify checksum
if fileMetadata.get('md5Sum') != fileInfo.get('md5Sum'):
self.logger.error('Checksum mismatch for file: %s' % filePath)
raise FileProcessingError('Checksum mismatch detected for file: %s' % filePath)
# Remove file
self.logger.debug('Checksum test passed, deleting file %s' % filePath)
OsUtility.removeFile(srcUrl)
#######################################################################
# Testing.
if __name__ == '__main__':
ft = RsyncWithChecksumAndDeleteFileTransferPlugin('/tmp/xyz', '/tmp/xyz2')
ft.start()
print 'StdOut: ', ft.getStdOut()
print 'StdErr: ', ft.getStdErr()
print 'Exit Status: ', ft.getExitStatus()
#!/usr/bin/env python
import os
import time
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi
from dm.common.utility.dmSubprocess import DmSubprocess
class ScriptProcessingPlugin(FileProcessor):
PROCESSING_SCRIPT_KEY = 'processingScript'
def __init__(self):
FileProcessor.__init__(self)
self.fileMongoDbApi = FileMongoDbApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
experimentName = fileInfo.get('experimentName')
experimentFilePath = fileInfo.get('experimentFilePath')
daqInfo = fileInfo.get('daqInfo', {})
processingScript = daqInfo.get(self.PROCESSING_SCRIPT_KEY)
if not processingScript:
self.logger.debug('Ignoring file %s for experiment %s' % (experimentFilePath, experimentName))
return
self.logger.debug('Processing file %s for experiment %s' % (experimentFilePath, experimentName))
storageDirectory = fileInfo.get('storageDirectory')
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Process file
try:
p = DmSubprocess('%s %s' % (processingScript, storageFilePath))
p.run()
stdOut = p.getStdOut()
except Exception, ex:
self.logger.error('Cannot process file %s for experiment %s: %s' % (experimentFilePath, experimentName, ex))
return
# Prepare catalogging entry
fileInfo2 = {}
fileInfo2['fileName'] = os.path.basename(experimentFilePath)
fileInfo2['experimentName'] = experimentName
fileInfo2['processingScriptOutput'] = '%s' % stdOut.strip()
self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
import time
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.mongodb.api.fileMongoDbApi import FileMongoDbApi
class SddsParameterProcessingPlugin(FileProcessor):
PROCESS_SDDS_PARAMETERS_KEY = 'processSddsParameters'
def __init__(self):
FileProcessor.__init__(self)
self.fileMongoDbApi = FileMongoDbApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
experimentFilePath = fileInfo.get('experimentFilePath')
experimentName = fileInfo.get('experimentName')
daqInfo = fileInfo.get('daqInfo', {})
processSddsParameters = daqInfo.get(self.PROCESS_SDDS_PARAMETERS_KEY)
if not processSddsParameters:
self.logger.debug('Ignoring file %s for experiment %s' % (experimentFilePath, experimentName))
return
self.logger.debug('Processing file %s for experiment %s' % (fileInfo, experimentName))
storageDirectory = fileInfo.get('storageDirectory')
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Load file
try:
import sddsdata
from sdds import SDDS
s = SDDS(0)
self.logger.error('Loading file %s for experiment %s' % (experimentFilePath, experimentName))
s.load(storageFilePath)
except Exception, ex:
self.logger.error('Cannot process file %s for experiment %s: %s' % (experimentFilePath, experimentName, ex))
return
# Prepare catalogging entry
fileInfo2 = {}
fileInfo2['fileName'] = os.path.basename(experimentFilePath)
fileInfo2['experimentName'] = experimentName
for i in range(0,len(s.parameterName)):
parameterName = s.parameterName[i]
parameterData = s.parameterData[i][0]
fileInfo2[parameterName] = parameterData
self.fileMongoDbApi.updateOrAddExperimentFile(fileInfo2)
self.logger.error('SDDS terminate file %s for experiment %s' % (experimentFilePath, experimentName))
sddsdata.Terminate(0)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
import time
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.timeUtility import TimeUtility
from dm.common.processing.plugins.fileProcessor import FileProcessor
from dm.common.utility.dmSubprocess import DmSubprocess
class SgeJobSubmissionPlugin(FileProcessor):
SGE_JOB_SCRIPT_KEY = 'sgeJobScript'
def __init__(self, sgeRoot):
FileProcessor.__init__(self)
self.sgeRoot = sgeRoot
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
experimentName = fileInfo.get('experimentName')
experimentFilePath = fileInfo.get('experimentFilePath')
daqInfo = fileInfo.get('daqInfo', {})
sgeJobScript = daqInfo.get(self.SGE_JOB_SCRIPT_KEY)
if not sgeJobScript:
self.logger.debug('Ignoring file %s for experiment %s' % (experimentFilePath, experimentName))
return
self.logger.debug('Processing file %s for experiment %s' % (experimentFilePath, experimentName))
storageDirectory = fileInfo.get('storageDirectory')
storageFilePath = os.path.join(storageDirectory, experimentFilePath)
# Process file
try:
p = DmSubprocess('. %s/default/common/settings.sh ; qsub -v DM_EXPERIMENT_NAME=%s %s %s' % (self.sgeRoot, experimentName, sgeJobScript, storageFilePath))
p.run()
except Exception, ex:
self.logger.error('Cannot process file %s for experiment %s: %s' % (experimentFilePath, experimentName, ex))
return
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.loggingManager import LoggingManager
class AuthorizationPrincipalAuthenticator:
def __init__(self, name=None):
self.name = name
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def getName(self):
return self.name
def authenticatePrincipal(self, principal, password):
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
import os
from dm.common.exceptions.authorizationError import AuthorizationError
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.utility.objectCache import ObjectCache
from dm.common.utility.cryptUtility import CryptUtility
from dm.common.utility.objectUtility import ObjectUtility
from dm.common.utility.singleton import Singleton
class AuthorizationPrincipalManager(DmObjectManager, Singleton):
DEFAULT_CACHE_SIZE = 10000 # number of items
DEFAULT_CACHE_OBJECT_LIFETIME = 3600 # seconds
CONFIG_SECTION_NAME = 'AuthorizationPrincipalManager'
ADMIN_ROLE_NAME_KEY = 'adminrolename'
PRINCIPAL_RETRIEVER_KEY = 'principalretriever'
PRINCIPAL_AUTHENTICATOR_KEY = 'principalauthenticator'
# Singleton instance.
__instance = None
def __init__(self):
if AuthorizationPrincipalManager.__instance:
return
AuthorizationPrincipalManager.__instance = self
DmObjectManager.__init__(self)
self.configurationManager = ConfigurationManager.getInstance()
self.principalRetriever = None
self.principalAuthenticatorList = []
self.objectCache = ObjectCache(AuthorizationPrincipalManager.DEFAULT_CACHE_SIZE, AuthorizationPrincipalManager.DEFAULT_CACHE_OBJECT_LIFETIME)
self.configure()
@classmethod
def cryptPassword(cls, cleartext):
return CryptUtility.cryptPassword(cleartext)
@classmethod
def cryptPasswordWithPbkdf2(cls, cleartext):
return CryptUtility.cryptPasswordWithPbkdf2(cleartext)
def configure(self):
configItems = self.configurationManager.getConfigItems(AuthorizationPrincipalManager.CONFIG_SECTION_NAME)
self.logger.debug('Got config items: %s' % configItems)
adminRoleName = self.configurationManager.getConfigOption(AuthorizationPrincipalManager.CONFIG_SECTION_NAME, AuthorizationPrincipalManager.ADMIN_ROLE_NAME_KEY)
# Create principal retriever
principalRetriever = self.configurationManager.getConfigOption(AuthorizationPrincipalManager.CONFIG_SECTION_NAME, AuthorizationPrincipalManager.PRINCIPAL_RETRIEVER_KEY)
(moduleName,className,constructor) = self.configurationManager.getModuleClassConstructorTuple(principalRetriever, AuthorizationPrincipalManager)
self.logger.debug('Creating principal retriever class: %s' % className)
self.principalRetriever = ObjectUtility.createObjectInstance(moduleName, className, constructor)
if adminRoleName is not None:
self.principalRetriever.setAdminRoleName(adminRoleName)
self.logger.debug('Authorization principal retriever: %s' % (self.principalRetriever))
# Create principal authenticators
for (key,value) in configItems:
if key.startswith(AuthorizationPrincipalManager.PRINCIPAL_AUTHENTICATOR_KEY):
(moduleName,className,constructor) = self.configurationManager.getModuleClassConstructorTuple(value, AuthorizationPrincipalManager)
self.logger.debug('Creating principal authenticator class: %s' % className)
principalAuthenticator = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.addAuthorizationPrincipalAuthenticator(principalAuthenticator)
self.logger.debug('Authorization principal authenticator: %s' % (principalAuthenticator))
def addAuthorizationPrincipalAuthenticator(self, principalAuthenticator):
self.principalAuthenticatorList.append(principalAuthenticator)
def getAuthenticatedAuthorizationPrincipal(self, username, password):
""" Get principal based on a username and password """
# First try cache.
#self.logger.debug('Trying username %s from the cache' % username)
principal = None
principal = self.objectCache.get(username)
if principal is None:
# Try principal retriever
principal = self.principalRetriever.getAuthorizationPrincipal(username)
if principal is None:
self.logger.debug('No principal for username: %s' % username)
return
# Try all authorization principal authenticators.
for principalAuthenticator in self.principalAuthenticatorList:
self.logger.debug('Attempting to authenticate %s by %s' % (username, principalAuthenticator.getName()))
authenticatedPrincipal = principalAuthenticator.authenticatePrincipal(principal, password)
if authenticatedPrincipal is not None:
self.logger.debug('Adding authorization principal %s to the cache, authenticated by %s' % (principal.getName(),principalAuthenticator.getName()))
self.objectCache.put(username, authenticatedPrincipal)
return authenticatedPrincipal
return None
def removeAuthorizationPrincipal(self, username):
""" Clear principal from the cache. """
self.objectCache.remove(username)
#######################################################################
# Testing.
if __name__ == '__main__':
am = AuthorizationPrincipalManager.getInstance()
authPrincipal = am.getAuthorizationPrincipal('sveseli', 'sv')
print 'Auth principal: ', authPrincipal
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.utility.loggingManager import LoggingManager
class AuthorizationPrincipalRetriever:
def __init__(self, name=None):
self.adminRoleName = dmRole.DM_ADMIN_ROLE
self.name = name
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def getName(self):
return self.name
def setAdminRoleName(self, adminRoleName):
self.adminRoleName = adminRoleName
def getAuthorizationPrincipal(self, username):
return None
def checkAutorizationPrincipalRole(self, principal):
if principal is None or self.adminRoleName is None:
return
userInfo = principal.getUserInfo()
if userInfo is None:
return
for userSystemRoleName in userInfo.get('userSystemRoleNameList', []):
if userSystemRoleName == self.adminRoleName:
principal.setRole(dmRole.DM_ADMIN_ROLE)
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.cryptUtility import CryptUtility
from authorizationPrincipalAuthenticator import AuthorizationPrincipalAuthenticator
class CryptedPasswordPrincipalAuthenticator(AuthorizationPrincipalAuthenticator):
def __init__(self):
AuthorizationPrincipalAuthenticator.__init__(self, self.__class__.__name__)
def authenticatePrincipal(self, principal, password):
if principal is not None:
principalToken = principal.getToken()
if principalToken is not None and len(principalToken):
if CryptUtility.verifyPasswordWithPbkdf2(password, principalToken):
self.logger.debug('Authentication successful for %s' % principal.getName())
return principal
else:
self.logger.debug('Authentication failed for %s' % principal.getName())
else:
self.logger.debug('Token is empty for %s, authentication not performed' % principal.getName())
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.db.api.userDbApi import UserDbApi
from authorizationPrincipalRetriever import AuthorizationPrincipalRetriever
class DbPrincipalRetriever(AuthorizationPrincipalRetriever):
def __init__(self):
AuthorizationPrincipalRetriever.__init__(self, self.__class__.__name__)
self.dbApi = UserDbApi()
def getAuthorizationPrincipal(self, username):
principal = None
try:
user = self.dbApi.getUserWithPasswordByUsername(username)
principal = AuthorizationPrincipal(name=username, token=user.get('password'))
principal.setRole(dmRole.DM_USER_ROLE)
principal.setUserInfo(user)
self.checkAutorizationPrincipalRole(principal)
except Exception, ex:
self.logger.debug(ex)
return principal
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.ldapUtility import LdapUtility
from authorizationPrincipalAuthenticator import AuthorizationPrincipalAuthenticator
class LdapPasswordPrincipalAuthenticator(AuthorizationPrincipalAuthenticator):
def __init__(self, serverUrl, dnFormat):
AuthorizationPrincipalAuthenticator.__init__(self, self.__class__.__name__)
self.ldapUtility = LdapUtility(serverUrl, dnFormat)
def authenticatePrincipal(self, principal, password):
if principal is not None:
try:
self.logger.debug('Checking credentials for %s' % principal.getName())
self.ldapUtility.checkCredentials(principal.getName(), password)
return principal
except Exception, ex:
self.logger.debug(ex)
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.constants import dmRole
from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.utility.cryptUtility import CryptUtility
from authorizationPrincipalRetriever import AuthorizationPrincipalRetriever
class NoOpPrincipalRetriever(AuthorizationPrincipalRetriever):
def __init__(self):
AuthorizationPrincipalRetriever.__init__(self, self.__class__.__name__)
def getAuthorizationPrincipal(self, username):
# Set password to be the same as username
noOpPassword = CryptUtility.cryptPasswordWithPbkdf2(username)
principal = AuthorizationPrincipal(name=username, token=noOpPassword)
principal.setRole(dmRole.DM_USER_ROLE)
if self.adminRoleName is not None:
principal.setRole(dmRole.DM_ADMIN_ROLE)
return principal
#######################################################################
# Testing.
if __name__ == '__main__':
pass
#!/usr/bin/env python
from dm.common.utility.loggingManager import LoggingManager
class SessionManager:
def __init__(self, name=None):
self.name = name
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def getName(self):
return self.name
def addSession(self, sessionId, sessionInfo):
return None
def checkSession(self, sessionId):
return None
#######################################################################
# Testing.
if __name__ == '__main__':
pass