src/python/dm/ds_web_service/service/impl/fileSessionControllerImpl.py

+#!/usr/bin/env python
+
+#
+# Implementation for file session controller.
+#
+
+import time
+
+from dm.common.objects.dmObjectManager import DmObjectManager
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.fileMetadata import FileMetadata
+from dm.common.objects.directoryMetadata import DirectoryMetadata
+from dm.common.db.api.experimentDbApi import ExperimentDbApi
+from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
+
+class FileSessionControllerImpl(DmObjectManager):
+    """ File session controller implementation class. """
+
+    def __init__(self):
+        DmObjectManager.__init__(self)
+        self.experimentDbApi = ExperimentDbApi()
+
+    def processFile(self, fileInfo):
+        experimentFilePath = fileInfo.get('experimentFilePath')
+        experimentName = fileInfo.get('experimentName')
+        experiment = self.experimentDbApi.getExperimentByName(experimentName)
+        ExperimentManager.getInstance().processExperimentFile(experimentFilePath, experiment, fileInfo)
+        return FileMetadata(fileInfo)
+
+    def statFile(self, fileInfo, experiment=None):
+        experimentFilePath = fileInfo.get('experimentFilePath')
+        if not experiment:
+            experimentName = fileInfo.get('experimentName')
+            experiment = self.experimentDbApi.getExperimentByName(experimentName)
+        ExperimentManager.getInstance().statExperimentFile(experimentFilePath, experiment, fileInfo)
+        return FileMetadata(fileInfo)
+
+    def processDirectory(self, directoryInfo):
+        experimentDirectoryPath = directoryInfo.get('experimentDirectoryPath', '')
+        experimentName = directoryInfo.get('experimentName')
+        experiment = self.experimentDbApi.getExperimentByName(experimentName)
+        ExperimentManager.getInstance().processExperimentDirectory(experimentDirectoryPath, experiment, directoryInfo)
+        return DirectoryMetadata(directoryInfo)
+
+

src/python/dm/ds_web_service/service/impl/pathTracker.py

+#!/usr/bin/env python
+
+from dm.common.utility.objectTracker import ObjectTracker
+
+class PathTracker(ObjectTracker):
+
+    # Cache configuration
+    pass
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    et = PathTracker.getInstance()
+    print et
+    et2 = PathTracker.getInstance()
+    print et2
+    et.put('xyz', 1)
+    print et.get('xyz')
+    print et.get('xyz2')
+
+
+

src/python/dm/ds_web_service/service/impl/userInfoSessionControllerImpl.py

+#!/usr/bin/env python
+
+#
+# Implementation for user info controller.
+#
+
+from dm.common.objects.dmObject import DmObject
+from dm.common.objects.dmObjectManager import DmObjectManager
+from dm.common.db.api.userDbApi import UserDbApi
+from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
+
+class UserInfoSessionControllerImpl(DmObjectManager):
+    """ User info controller implementation class. """
+
+    def __init__(self):
+        DmObjectManager.__init__(self)
+        self.userDbApi = UserDbApi()
+
+    def getUsers(self):
+        return self.userDbApi.getUsers()
+
+    def getUserById(self, id):
+        return self.userDbApi.getUserById(id)
+
+    def getUserByUsername(self, username):
+        return self.userDbApi.getUserByUsername(username)
+
+    def addUserSystemRole(self, username, roleName, experimentStationName):
+        userSystemRole = self.userDbApi.addUserSystemRole(username, roleName, experimentStationName)
+        return userSystemRole
+
+    def deleteUserSystemRole(self, username, roleName, experimentStationName):
+        userSystemRole = self.userDbApi.deleteUserSystemRole(username, roleName, experimentStationName)
+        return userSystemRole
+
+    def addUserExperimentRole(self, username, roleName, experimentName):
+        userExperimentRole = self.userDbApi.addUserExperimentRole(username, roleName, experimentName)
+        ExperimentManager.getInstance().addUserToGroup(username, experimentName)
+        return userExperimentRole 
+
+    def deleteUserExperimentRole(self, username, roleName, experimentName):
+        userExperimentRole = self.userDbApi.deleteUserExperimentRole(username, roleName, experimentName)
+        ExperimentManager.getInstance().deleteUserFromGroup(username, experimentName)
+        return userExperimentRole
+

src/python/dm/ds_web_service/service/userInfoSessionController.py

+#!/usr/bin/env python
+
+import cherrypy
+from dm.common.service.dmSessionController import DmSessionController
+from dm.ds_web_service.service.impl.userInfoSessionControllerImpl import UserInfoSessionControllerImpl
+from dm.common.db.api.experimentDbApi import ExperimentDbApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.exceptions.authorizationError import AuthorizationError
+
+class UserInfoSessionController(DmSessionController):
+
+    def __init__(self):
+        DmSessionController.__init__(self)
+        self.experimentDbApi = ExperimentDbApi()
+        self.userInfoSessionControllerImpl = UserInfoSessionControllerImpl()
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isLoggedIn())
+    @DmSessionController.execute
+    def getUsers(self, **kwargs):
+        return self.listToJson(self.userInfoSessionControllerImpl.getUsers())
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isLoggedIn())
+    @DmSessionController.execute
+    def getUserById(self, id, **kwargs):
+        if not id:
+            raise InvalidRequest('Invalid id provided.')
+        response = self.userInfoSessionControllerImpl.getUserById(id).getFullJsonRep()
+        self.logger.debug('Returning user info for %s: %s' % (id,response))
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isLoggedIn())
+    @DmSessionController.execute
+    def getUserByUsername(self, username, **kwargs):
+        if not len(username):
+            raise InvalidRequest('Invalid username provided.')
+        response = self.userInfoSessionControllerImpl.getUserByUsername(username).getFullJsonRep()
+        self.logger.debug('Returning user info for %s: %s' % (username,response))
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isLoggedIn())
+    @DmSessionController.execute
+    def addUserSystemRole(self, username, roleName, **kwargs):
+        if not username:
+            raise InvalidRequest('Invalid username provided.')
+        if not roleName:
+            raise InvalidRequest('Invalid role name provided.')
+        experimentStationName = kwargs.get('experimentStationName')
+        if experimentStationName:
+            experimentStation = self.experimentDbApi.getExperimentStationByName(experimentStationName)
+            if not self.hasAdministratorRole() and not self.hasManagerRole(experimentStation['id']):
+                raise AuthorizationError('User %s cannot modify system roles for experiment station %s.' % (self.getSessionUsername(),experimentStationName))
+        else:
+            if not self.hasAdministratorRole():
+                raise AuthorizationError('User %s cannot modify system roles.' % (self.getSessionUsername()))
+
+        response = self.userInfoSessionControllerImpl.addUserSystemRole(username, roleName, experimentStationName).getFullJsonRep()
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isLoggedIn())
+    @DmSessionController.execute
+    def deleteUserSystemRole(self, username, roleName, **kwargs):
+        if not username:
+            raise InvalidRequest('Invalid username provided.')
+        if not roleName:
+            raise InvalidRequest('Invalid role name provided.')
+        experimentStationName = kwargs.get('experimentStationName')
+        if experimentStationName:
+            experimentStation = self.experimentDbApi.getExperimentStationByName(experimentStationName)
+            if not self.hasAdministratorRole() and not self.hasManagerRole(experimentStation['id']):
+                raise AuthorizationError('User %s cannot modify system roles for experiment station %s.' % (self.getSessionUsername(),experimentStationName))
+        else:
+            if not self.hasAdministratorRole():
+                raise AuthorizationError('User %s cannot modify system roles.' % (self.getSessionUsername()))
+
+        response = self.userInfoSessionControllerImpl.deleteUserSystemRole(username, roleName, experimentStationName).getFullJsonRep()
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isLoggedIn())
+    @DmSessionController.execute
+    def addUserExperimentRole(self, username, roleName, experimentName, **kwargs):
+        if not username:
+            raise InvalidRequest('Invalid username provided.')
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not roleName:
+            raise InvalidRequest('Invalid role name provided.')
+ 
+        experiment = self.experimentDbApi.getExperimentByName(experimentName)
+        if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']) and not self.hasPiRole(experiment['id']):
+            raise AuthorizationError('User %s cannot modify users for experiment %s.' % (self.getSessionUsername(),experimentName))
+
+        response = self.userInfoSessionControllerImpl.addUserExperimentRole(username, roleName, experimentName).getFullJsonRep()
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isLoggedIn())
+    @DmSessionController.execute
+    def deleteUserExperimentRole(self, username, roleName, experimentName, **kwargs):
+        if not username:
+            raise InvalidRequest('Invalid username provided.')
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not roleName:
+            raise InvalidRequest('Invalid role name provided.')
+
+        experiment = self.experimentDbApi.getExperimentByName(experimentName)
+        if not self.hasAdministratorRole() and not self.hasManagerRole(experiment['experimentStationId']) and not self.hasPiRole(experiment['id']):
+            raise AuthorizationError('User %s cannot modify users for experiment %s.' % (self.getSessionUsername(),experimentName))
+
+        response = self.userInfoSessionControllerImpl.deleteUserExperimentRole(username, roleName, experimentName).getFullJsonRep()
+        return response
+

src/python/dm/ds_web_service/service/userRouteDescriptor.py

+#!/usr/bin/env python
+
+#
+# User route descriptor.
+#
+
+from dm.common.utility.configurationManager import ConfigurationManager
+from userInfoSessionController import UserInfoSessionController
+
+class UserRouteDescriptor:
+
+    @classmethod
+    def getRoutes(cls):
+        contextRoot = ConfigurationManager.getInstance().getContextRoot()
+
+        # Static instances shared between different routes
+        userInfoSessionController = UserInfoSessionController()
+
+        # Define routes.
+        routes = [
+
+            # Get user info list
+            {
+                'name' : 'getUsers',
+                'path' : '%s/users' % contextRoot,
+                'controller' : userInfoSessionController,
+                'action' : 'getUsers', 
+                'method' : ['GET']
+            },
+
+            # Get user by id
+            {
+                'name' : 'getUserById',
+                'path' : '%s/users/:(id)' % contextRoot,
+                'controller' : userInfoSessionController,
+                'action' : 'getUserById', 
+                'method' : ['GET']
+            },
+
+            # Get user by username
+            {
+                'name' : 'getUserByUsername',
+                'path' : '%s/usersByUsername/:(username)' % contextRoot,
+                'controller' : userInfoSessionController,
+                'action' : 'getUserByUsername', 
+                'method' : ['GET']
+            },
+
+            # Add user system role
+            {
+                'name' : 'addUserSystemRole',
+                'path' : '%s/userSystemRoles/:(username)/:(roleName)' % contextRoot,
+                'controller' : userInfoSessionController,
+                'action' : 'addUserSystemRole',
+                'method' : ['POST']
+            },
+
+            # Delete user system role
+            {
+                'name' : 'deleteUserSystemRole',
+                'path' : '%s/userSystemRoles/:(username)/:(roleName)' % contextRoot,
+                'controller' : userInfoSessionController,
+                'action' : 'deleteUserSystemRole',
+                'method' : ['DELETE']
+            },
+
+            # Add user experiment role
+            {
+                'name' : 'addUserExperimentRole',
+                'path' : '%s/userExperimentRoles/:(username)/:(roleName)/:(experimentName)' % contextRoot,
+                'controller' : userInfoSessionController,
+                'action' : 'addUserExperimentRole',
+                'method' : ['POST']
+            },
+
+            # Delete user experiment role
+            {
+                'name' : 'deleteUserExperimentRole',
+                'path' : '%s/userExperimentRoles/:(username)/:(roleName)/:(experimentName)' % contextRoot,
+                'controller' : userInfoSessionController,
+                'action' : 'deleteUserExperimentRole',
+                'method' : ['DELETE']
+            },
+
+        ]
+       
+        return routes
+
+

src/python/dm/proc_web_service/api/procRestApi.py

+#!/usr/bin/env python
+
+from dm.common.api.dmRestApi import DmRestApi
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class ProcRestApi(DmRestApi):
+    """ Base CAT DM REST api class. """
+
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        if host == None:
+            host = ConfigurationManager.getInstance().getProcWebServiceHost()
+        if port == None:
+            port = ConfigurationManager.getInstance().getProcWebServicePort()
+        DmRestApi.__init__(self, username, password, host, port, protocol)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass
+
+

src/python/dm/proc_web_service/api/workflowRestApi.py

+#!/usr/bin/env python
+
+import os
+import urllib
+import json
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.workflow import Workflow
+from procRestApi import ProcRestApi
+
+class WorkflowRestApi(ProcRestApi):
+    
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        ProcRestApi.__init__(self, username, password, host, port, protocol)
+
+    @ProcRestApi.execute
+    def addWorkflow(self, fileInfo):
+        owner = fileInfo.get('owner')
+        if not owner:
+            raise InvalidRequest('Workflow metadata must contain owner key.')
+        experimentWorkflowPath = fileInfo.get('experimentWorkflowPath')
+        if not experimentWorkflowPath:
+            raise InvalidRequest('Workflow metadata must contain experimentWorkflowPath key.')
+        url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), owner, Encoder.encode(experimentWorkflowPath))
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseData = self.sendSessionRequest(url=url, method='POST')
+        return Workflow(responseData)
+
+    @ProcRestApi.execute
+    def updateWorkflow(self, fileInfo):
+        owner = fileInfo.get('owner')
+        if not owner:
+            raise InvalidRequest('Workflow metadata must contain owner key.')
+        experimentWorkflowPath = fileInfo.get('experimentWorkflowPath')
+        if not experimentWorkflowPath:
+            raise InvalidRequest('Workflow metadata must contain experimentWorkflowPath key.')
+        url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), owner, Encoder.encode(experimentWorkflowPath))
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return Workflow(responseData)
+
+    @ProcRestApi.execute
+    def updateWorkflowById(self, fileInfo):
+        owner = fileInfo.get('owner')
+        if not owner:
+            raise InvalidRequest('Workflow metadata must contain owner key.')
+        id = fileInfo.get('id')
+        if not id:
+            raise InvalidRequest('Workflow metadata must contain id key.')
+        url = '%s/filesByExperimentAndId/%s/%s' % (self.getContextRoot(), owner, id)
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return Workflow(responseData)
+
+    @ProcRestApi.execute
+    def getWorkflows(self, owner, queryDict={}):
+        if not owner:
+            raise InvalidRequest('Invalid experiment name provided.')
+        url = '%s/filesByExperiment/%s' % (self.getContextRoot(), owner)
+        url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, Workflow)
+
+    @ProcRestApi.execute
+    def getWorkflowById(self, owner, id):
+        if not owner:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not id:
+            raise InvalidRequest('Invalid file id provided.')
+        url = '%s/filesByExperimentAndId/%s/%s' % (self.getContextRoot(), owner, id)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return Workflow(responseData)
+
+    @ProcRestApi.execute
+    def getWorkflowByName(self, owner, name):
+        if not owner:
+            raise InvalidRequest('Invalid owner name provided.')
+        if not name:
+            raise InvalidRequest('Invalid workflow name provided.')
+        url = '%s/workflowsByOwner/%s/%s' % (self.getContextRoot(), owner, Encoder.encode(name))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return Workflow(responseData)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = WorkflowRestApi()
+
+

src/python/dm/proc_web_service/cli/getWorkflowCli.py

+#!/usr/bin/env python
+
+from dm.proc_web_service.api.workflowRestApi import WorkflowRestApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from procWebServiceSessionCli import ProcWebServiceSessionCli
+
+class GetWorkflowCli(ProcWebServiceSessionCli):
+    def __init__(self):
+        ProcWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--owner', dest='owner', help='Owner name.')
+        self.addOption('', '--workflow', dest='workflow', help='Workflow name.')
+
+    def checkArgs(self):
+        if self.options.owner is None:
+            raise InvalidRequest('Owner name must be provided.')
+        if self.options.workflow is None:
+            raise InvalidRequest('Workflow name must be provided.')
+
+    def getOwner(self):
+        return self.options.owner
+
+    def getWorkflow(self):
+        return self.options.workflow
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-get-workflow --workflow=WORKFLOW --owner=OWNER
+
+Description:
+    Get workflow details.
+        """)
+        self.checkArgs()
+        api = WorkflowRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        workflow = api.getWorkflowByName(self.getOwner(), self.getWorkflow())
+        print workflow.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = GetWorkflowCli()
+    cli.run()
+

src/python/dm/proc_web_service/cli/procWebServiceCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmRestCli import DmRestCli
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class ProcWebServiceCli(DmRestCli):
+    """ DM PROC web service cli class. """
+
+    def __init__(self, validArgCount=0):
+        DmRestCli.__init__(self, validArgCount)
+
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getProcWebServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getProcWebServicePort()
+                        

src/python/dm/proc_web_service/cli/procWebServiceSessionCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmRestSessionCli import DmRestSessionCli
+from dm.common.utility.osUtility import OsUtility
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class ProcWebServiceSessionCli(DmRestSessionCli):
+    """ DM PROC web service session cli class. """
+
+    DEFAULT_SESSION_CACHE_FILE = OsUtility.getUserHomeDir() + '/.dm/.proc.session.cache'
+
+    def __init__(self, validArgCount=0):
+        DmRestSessionCli.__init__(self, validArgCount)
+        ConfigurationManager.getInstance().setSessionCacheFile(ProcWebServiceSessionCli.DEFAULT_SESSION_CACHE_FILE)
+
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getProcWebServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getProcWebServicePort()
+

src/python/dm/proc_web_service/service/impl/executionManager.py

+#!/usr/bin/env python
+
+import threading
+import time
+import os
+import uuid
+import glob
+import copy
+
+from dm.common.constants import dmProcessingStatus
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.childProcess import ChildProcess
+from dm.common.objects.processingJob import ProcessingJob
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.utility.objectUtility import ObjectUtility
+from dm.common.utility.valueUtility import ValueUtility
+from dm.common.utility.timeUtility import TimeUtility
+from dm.common.utility.threadingUtility import ThreadingUtility
+from dm.common.utility.timeBasedProcessingQueue import TimeBasedProcessingQueue
+from dm.common.utility.singleton import Singleton
+from executionThread import ExecutionThread
+from processingJobTracker import ProcessingJobTracker
+
+class ExecutionManager(Singleton):
+
+    CONFIG_SECTION_NAME = 'ExecutionManager'
+    NUMBER_OF_EXECUTION_THREADS_KEY = 'numberofexecutionthreads'
+    DEFAULT_NUMBER_OF_EXECUTION_THREADS = 3
+
+    FILE_PATH_LIST_KEY = 'filePathList'
+    FILE_PATH_PATTERN_KEY = 'filePathPattern'
+    FILE_QUERY_DICT_KEY = 'fileQueryDict'
+
+    PROCESSING_JOB_START_DELAY_IN_SECONDS = 10
+    MAX_NUMBER_OF_CHILD_PROCESSES = 10
+    CHILD_PROCESSES_EVENT_TIMEOUT_IN_SECONDS = 10
+
+    # Singleton.
+    __instanceLock = threading.RLock()
+    __instance = None
+
+    def __init__(self):
+        ExecutionManager.__instanceLock.acquire()
+        try:
+            if ExecutionManager.__instance:
+                return
+            ExecutionManager.__instance = self
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+
+            self.logger.debug('Initializing')
+            self.lock = threading.RLock()
+            self.executionThreadDict = {}
+            self.eventFlag = threading.Event()
+            self.executionQueue = TimeBasedProcessingQueue()
+            self.__configure()
+            self.logger.debug('Initialization complete')
+        finally:
+            ExecutionManager.__instanceLock.release()
+
+    def __configure(self):
+        cm = ConfigurationManager.getInstance()
+        configItems = cm.getConfigItems(ExecutionManager.CONFIG_SECTION_NAME)
+        self.logger.debug('Got config items: %s' % configItems)
+        self.nExecutionThreads = int(cm.getConfigOption(ExecutionManager.CONFIG_SECTION_NAME, ExecutionManager.NUMBER_OF_EXECUTION_THREADS_KEY, ExecutionManager.DEFAULT_NUMBER_OF_EXECUTION_THREADS))
+
+    def resolveFileSpec(self, processingJob):
+        if processingJob.has_key('filePathList'):
+            return self.resolveFilePathList(processingJob)
+        elif processingJob.has_key('filePathPattern'):
+            return self.resolveFilePathPattern(processingJob)
+        elif processingJob.has_key('fileQueryDict'):
+            return self.resolveFileQueryDict(processingJob)
+        else:
+            raise InvalidRequest('Invalid input file specification.')
+
+    def resolveFilePathList(self, processingJob):
+        return processingJob.get('filePathList')
+
+    def resolveFilePathPattern(self, processingJob):
+        if not processingJob.has_key('filePathList'):
+             globPattern = processingJob.get('filePathPattern')
+             dataDir = processingJob.get('dataDir')
+             currentDir = os.getcwd()
+             if dataDir:
+                 os.chdir(dataDir)     
+             processingJob['filePathList'] = glob.glob(globPattern)
+             os.chdir(currentDir)     
+        return processingJob.get('filePathList', [])
+
+    def resolveFileQueryDict(self, processingJob):
+        return []
+
+    def startProcessingJob(self, processingJob):
+        workflow = processingJob.get('workflow')
+        self.logger.debug('Submitted processing job for workflow %s' % workflow.get('name'))
+        jobId = str(uuid.uuid4())
+        processingJob['id'] =  jobId
+        startTime = time.time()
+        processingJob['startTime'] = startTime
+        processingJob['startTimestamp'] = TimeUtility.formatLocalTimestamp(startTime)
+
+        processingJob['nFiles'] = 0
+        processingJob['status'] = dmProcessingStatus.DM_PROCESSING_STATUS_PENDING
+        ProcessingJobTracker.getInstance().put(jobId, processingJob)
+        self.logger.debug('Starting processing job %s timer' % jobId)
+
+        timer = threading.Timer(self.PROCESSING_JOB_START_DELAY_IN_SECONDS, self.runProcessingJob, args=[processingJob])
+        timer.start()
+        return processingJob
+
+    def runProcessingJob(self, processingJob):
+        try:
+            childProcessEvent = threading.Event()
+            processingJob.childProcessEvent = childProcessEvent
+            jobId = processingJob['id']
+            self.logger.debug('Timer started for processing job %s.' % jobId)
+
+            # Resolve files
+            filePathList = self.resolveFileSpec(processingJob)
+            if not filePathList:
+                raise InvalidRequest('There are no input files available for processing.')
+            self.logger.debug('There are %s files for processing job %s.' % (len(filePathList), jobId))
+            processingJob['nFiles'] = len(filePathList)
+            workflowStages = processingJob.get('workflow').get('stages')
+            workflowStageKeys = workflowStages.keys()
+            workflowStageKeys.sort()
+            processingJob['status'] = dmProcessingStatus.DM_PROCESSING_STATUS_RUNNING
+            childProcessNumber = 0
+            workingDir = processingJob.get('workingDir')
+            for stageKey in workflowStageKeys:
+                if processingJob['status'] != dmProcessingStatus.DM_PROCESSING_STATUS_RUNNING:
+                    self.logger.debug('Processing job %s status is %s, exiting workflow at stage %s' % (jobId, processingJob['status'], stageKey))
+                    break
+
+                self.logger.debug('Starting stage %s for processing job %s.' % (stageKey,jobId))
+                workflowStage = workflowStages.get(stageKey)
+                workflowStage['childProcesses'] = {}
+                workflowStage['nRunningChildProcesses'] = 0
+                workflowStage['nQueuedChildProcesses'] = 0
+                command = workflowStage.get('command')
+                processingJob['activeStage'] = stageKey
+                variableNames = [word for word in command.split() if word.startswith('$')]
+                self.logger.debug('Variables found for stage %s: ' % variableNames)
+                iterateOverFiles = False
+                for v in variableNames:
+                    variableName = v[1:] # remove '$'
+                    variableValue = processingJob.get(variableName)
+                    if variableName == 'filePathList':
+                        variableValue = ' '.join(filePathList)
+                    if variableValue is not None:
+                        command = command.replace(v, variableValue)
+                    if variableName == 'filePath':
+                        iterateOverFiles = True
+
+                if iterateOverFiles:
+                    # We iterate over all files
+                    for f in filePathList:
+                        childProcessEvent.clear()
+                        if processingJob['status'] != dmProcessingStatus.DM_PROCESSING_STATUS_RUNNING:
+                            self.logger.debug('Processing job %s status is %s, exiting workflow at stage %s' % (jobId, processingJob['status'], stageKey))
+                            break
+
+                        # Wait until we can queue child process
+                        while True:
+                            nQueuedChildProcesses = workflowStage.get('nQueuedChildProcesses')
+                            nRunningChildProcesses = workflowStage.get('nRunningChildProcesses')
+                            nChildProcesses = nQueuedChildProcesses + nRunningChildProcesses
+                            if nChildProcesses < self.MAX_NUMBER_OF_CHILD_PROCESSES:
+                                break
+                            else:
+                                self.logger.debug('There are %s child processes for stage %s (processing job %s), cannot create new one' % (nChildProcesses, stageKey, jobId))
+                            childProcessEvent.wait(self.CHILD_PROCESSES_EVENT_TIMEOUT_IN_SECONDS)
+
+
+                        childCommand = command.replace('$filePath', f)
+                        self.logger.debug('Execution command for stage %s, child process %s: %s' % (stageKey, childProcessNumber, childCommand))
+                        childProcess = self.createChildProcess(stageKey, childProcessNumber, childCommand, workingDir)
+                        self.pushToExecutionQueue(processingJob, childProcess)
+                        childProcessNumber += 1
+
+                else:
+                    childCommand = command
+                    self.logger.debug('Execution command for stage %s, child process %s: %s' % (stageKey, childProcessNumber, childCommand))
+                    childProcess = self.createChildProcess(stageKey, childProcessNumber, childCommand, workingDir)
+                    self.pushToExecutionQueue(processingJob, childProcess)
+                    childProcessNumber += 1
+
+                # Child processed queued, wait until they finish
+                while True:
+                    childProcessEvent.clear()
+                    nQueuedChildProcesses = workflowStage.get('nQueuedChildProcesses')
+                    nRunningChildProcesses = workflowStage.get('nRunningChildProcesses')
+                    nChildProcesses = nQueuedChildProcesses + nRunningChildProcesses
+                    if nChildProcesses == 0:
+                        self.logger.debug('No more queued/running child processes for stage %s (processing job %s)' % (stageKey, jobId))
+                        break
+                    else:
+                        self.logger.debug('There are %s queued/running child processes for stage %s (processing job %s)' % (nChildProcesses, stageKey, jobId))
+                        childProcessEvent.wait(self.CHILD_PROCESSES_EVENT_TIMEOUT_IN_SECONDS)
+                 
+                
+        except Exception, ex:
+            processingJob['status'] = dmProcessingStatus.DM_PROCESSING_STATUS_FAILED
+            processingJob['errorMessage'] = str(ex)
+            self.logger.error('Processing job %s failed: %s' % (jobId,str(ex)))
+        if processingJob['status'] == dmProcessingStatus.DM_PROCESSING_STATUS_RUNNING:
+            processingJob['status'] = dmProcessingStatus.DM_PROCESSING_STATUS_DONE
+            self.logger.debug('Processing job %s is done.' % jobId)
+        self.logger.debug('Processing job: %s' % processingJob)
+
+    def pushToExecutionQueue(self, processingJob, childProcess):
+        self.executionQueue.push((processingJob, childProcess))
+        processingJob.childProcessQueued(childProcess)
+        self.setEvent()
+   
+    def createChildProcess(self, stageId, childProcessNumber, command, workingDir):
+        childProcess = ChildProcess({
+            'stageId' : stageId,
+            'childProcessNumber' : childProcessNumber,
+            'command' : command,
+            'workingDir' : workingDir,
+        })
+        return childProcess
+
+    @ThreadingUtility.synchronize
+    def start(self):
+        self.logger.debug('Starting execution threads')
+        for i in range(0, self.nExecutionThreads):
+            tName = 'ExecutionThread-%s' % i
+            t = ExecutionThread(tName, self, self.executionQueue)
+            t.start()
+            self.executionThreadDict[tName] = t
+
+    def stop(self):
+        self.logger.debug('Stopping execution threads')
+        for (tName, t) in self.executionThreadDict.items():
+            t.stop()
+        self.setEvent()
+        for (tName, t) in self.executionThreadDict.items():
+            t.join()
+
+    @ThreadingUtility.synchronize
+    def setEvent(self):
+         self.eventFlag.set()
+
+    @ThreadingUtility.synchronize
+    def clearEvent(self):
+        self.eventFlag.clear()
+
+    def waitOnEvent(self, timeoutInSeconds=None):
+        self.eventFlag.wait(timeoutInSeconds)
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    LoggingManager.getInstance().configure()
+    LoggingManager.getInstance().setConsoleLogLevel('DEBUG')
+    em = ExecutionManager.getInstance()
+    print em
+    em.start()
+    # Stage Keys
+    #   command (required)
+    #   execOutFormat (optional, to be parsed for variables like $jobId )
+    #   workingDir (optional, default .)
+    #   dataDir (optional, default $workingDir)
+    #   monitorCmd (optional, if exists, will execute monitoring in a timer)
+    #   parallelExec (optional, default False)
+    #   filePathList, filePathPattern, or fileQueryDict (one must be present)
+
+    # If $filePathList or $filePath is in the argument list, and
+    # job spec contains filePathPattern or fileQueryDict, files will be
+    # resolved into filePathList
+    # If $filePath is in the argument list, filePathList is iterated over
+    # for a particular stage 
+
+    # Variables that get filled automatically
+    #     $username : DM username
+    #     $execHost : execution host
+    #     $id : job id
+    #     $startTime : procesing start time
+    #     $endTime : procesing end time
+     
+    workflow = {u'owner': u'sveseli', u'stages': {
+        '1' : {u'command': u'/bin/date', 'stopOnFailure' : False}, 
+        '2' : {u'command': u'/bin/ls -l $filePath', 'parallelExec' : True},
+        '3' : {u'command': u'/bin/ls -c1 $filePathList'},
+        '4' : {u'command': u'/bin/echo Energy is $energyArg current is $CURRENT_ARG'},
+        }, u'description': u'My first workflow', u'name': u'workflow-01', u'id': u'589396e0e20bf2465e3a6bf0'}
+
+    processingJob = ProcessingJob({ 
+        'workflow' : workflow,
+        'experimentName' : 'exp-01',
+        'energyArg' : '2TeV', 
+        'CURRENT_ARG' : '2mA',
+        'filePathList' : [ 'testfile01', 'testfile02', 'testfile03' ],
+        'dataDir' : '/tmp/data',
+        'workingDir' : '/tmp/data',
+    })
+    em.startProcessingJob(processingJob)
+
+    time.sleep(180)
+    em.stop()
+
+

src/python/dm/proc_web_service/service/impl/executionThread.py

+#!/usr/bin/env python
+
+import threading
+import time
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.dmSubprocess import DmSubprocess
+
+class ExecutionThread(threading.Thread):
+
+    THREAD_EVENT_TIMEOUT_IN_SECONDS = 10.0
+
+    def __init__ (self, name, executionManager, executionQueue):
+    
+        threading.Thread.__init__(self)
+        self.setName(name)
+        self.exitFlag = False
+        self.executionManager = executionManager
+        self.executionQueue = executionQueue
+        self.logger = LoggingManager.getInstance().getLogger(name)
+
+    def processFile(self, fileInfo):
+
+        try:
+            pass
+        except Exception, ex:
+            self.logger.exception(ex)
+
+    def run(self):
+        self.logger.debug('Starting thread: %s' % self.getName())
+        while True:
+            self.executionManager.clearEvent()
+            if self.exitFlag:
+                self.logger.debug('Exit flag is set')
+                break
+
+            while True:
+                try:
+                    queuedTuple = self.executionQueue.pop()
+                    if queuedTuple is None:
+                        break
+                    processingJob, childProcess = queuedTuple
+                    self.logger.debug('Execution queue depth after pop: %s' % self.executionQueue.getLength())
+                    self.logger.debug('Starting child process: %s' % childProcess)
+                    command = childProcess.get('command')
+                    workingDir = childProcess.get('workingDir')
+                    processingJob.childProcessStarted(childProcess)
+                    p = DmSubprocess.executeCommandAndIgnoreFailure(command, workingDir)
+                    processInfo = p.toDict()
+                    if p.getExitStatus() == 0:
+                        processingJob.childProcessCompleted(childProcess, processInfo)
+                    else:
+                        processingJob.childProcessFailed(childProcess, processInfo)
+                except Exception, ex:
+                    self.logger.exception(ex)
+                    processingJob.childProcessFailed(childProcess, errorMessage=str(ex))
+
+            self.executionManager.waitOnEvent(self.THREAD_EVENT_TIMEOUT_IN_SECONDS)
+        self.logger.debug('%s is done' % self.getName())
+
+    def stop(self):
+        self.exitFlag = True
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    pass
+
+

src/python/dm/proc_web_service/service/impl/processingJobTracker.py

+#!/usr/bin/env python
+
+import os
+import uuid
+import time
+
+from dm.common.objects.processingJob import ProcessingJob
+from dm.common.utility.objectTracker import ObjectTracker
+
+class ProcessingJobTracker(ObjectTracker):
+
+    # Cache configuration
+    objectClass = ProcessingJob
+    cacheSize = 100
+
+    def __init__(self, *args, **kwargs):
+        ObjectTracker.__init__(self, args, kwargs)
+
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    tracker = ProcessingJobTracker.getInstance()
+    print tracker
+
+
+

src/python/dm/proc_web_service/service/impl/workflowSessionControllerImpl.py

+#!/usr/bin/env python
+
+#
+# Implementation for workflow controller.
+#
+
+from dm.common.objects.dmObject import DmObject
+from dm.common.objects.dmObjectManager import DmObjectManager
+from dm.common.mongodb.api.workflowMongoDbApi import WorkflowMongoDbApi
+
+class WorkflowSessionControllerImpl(DmObjectManager):
+    """ Workflow controller implementation class. """
+
+    def __init__(self):
+        DmObjectManager.__init__(self)
+        self.workflowMongoDbApi = WorkflowMongoDbApi()
+
+    def getWorkflowByName(self, owner, name):
+        return self.workflowMongoDbApi.getWorkflowByName(owner, name)
+

src/python/dm/proc_web_service/service/procWebService.py

+#!/usr/bin/env python
+
+#
+# DM PROC Web Service
+#
+
+from dm.common.service.dmRestWebServiceBase import DmRestWebServiceBase
+from dm.common.utility.dmModuleManager import DmModuleManager
+from dm.common.utility.configurationManager import ConfigurationManager
+
+from procWebServiceRouteMapper import ProcWebServiceRouteMapper
+
+class ProcWebService(DmRestWebServiceBase):
+
+    def __init__(self):
+        DmRestWebServiceBase.__init__(self, ProcWebServiceRouteMapper)
+
+    def initDmModules(self):
+        self.logger.debug('Initializing dm modules')
+
+        # Add modules that will be started.
+        moduleManager = DmModuleManager.getInstance()
+
+        self.logger.debug('Initialized dm modules')
+
+    def getDefaultServerHost(self):
+        return ConfigurationManager.getInstance().getServiceHost()
+
+    def getDefaultServerPort(self):
+        return ConfigurationManager.getInstance().getServicePort()
+
+####################################################################
+# Run service
+
+if __name__ == '__main__':
+    ConfigurationManager.getInstance().setServiceName('proc-web-service')
+    service = ProcWebService();
+    service.run()