add skip plugin options to daq commands; add beamline specific scripts; add...

add skip plugin options to daq commands; add beamline specific scripts; add command that combines add/update experiment with upload

bin/1ID/dm-1id-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/1ID/dm-1id-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/33ID/dm-33id-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/33ID/dm-33id-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/34IDE/dm-34ide-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/34IDE/dm-34ide-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/7ID/dm-7id-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/7ID/dm-7id-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

bin/TEST/dm-test-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/daqCli.py $DM_COMMAND_ARGS"
+
+

bin/TEST/dm-test-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source $DM_ROOT_DIR/bin/dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/aps_beamline_tools/cli/uploadCli.py $DM_COMMAND_ARGS"
+
+

doc/RELEASE_NOTES.txt

@@ -3,9 +3,9 @@ Release 1.1 (03/10/2017)
 
 - Introduced integration with Beamline Scheduling System:
   - New commands: 
-        - list-runs
-        - list-proposals
-        - get-proposal
+    - list-runs
+    - list-proposals
+    - get-proposal
   - Modified add-experiment command to automatically add users associated
     with a given beamline proposal
 - Added the following functionality for managing DAQs:
@@ -18,6 +18,8 @@ Release 1.1 (03/10/2017)
   - Destination directory specification causes files to be uploaded into
     a specific directory relative to experiment root path
 - Introduced framework for higher level beamline specific tools 
+  - New commands that combine adding experiment and running DAQs or
+    uploads
 - Introduced sphinx as python API documentation framework
 - Resolved possible timeout issue when starting DAQ or directory upload 
   with a directory containing large number of files

src/python/dm/aps_beamline_tools/cli/daqCli.py

@@ -45,6 +45,7 @@ class DaqCli(ApsBeamlineCli):
         self.addOptionToGroup(daqGroup, '', '--upload-data-directory-on-exit', dest='uploadDataDirectoryOnExit', help='Data directory that will be uploaded automatically after DAQ is stopped.')
         self.addOptionToGroup(daqGroup, '', '--upload-dest-directory-on-exit', dest='uploadDestDirectoryOnExit', help='Destination directory relative to experiment root path for automatic upload after DAQ is stopped. Requires upload data directory to be specified.')
         self.addOptionToGroup(daqGroup, '', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.')
+        self.addOptionToGroup(daqGroup, '', '--skip-plugins', dest='skipPlugins', help='Comma-separated list of plugins which should not process files.')
 
     def checkArgs(self):
         if self.options.experimentName is None:
@@ -58,6 +59,8 @@ class DaqCli(ApsBeamlineCli):
     def updateDaqInfoFromOptions(self, daqInfo):
         if self.options.processHidden:
             daqInfo['processHiddenFiles'] = True
+        if self.options.skipPlugins:
+            daqInfo['skipPlugins'] = self.options.skipPlugins
         if self.options.duration:
             duration = self.options.duration
             if duration.endswith('h'):
@@ -215,8 +218,7 @@ class DaqCli(ApsBeamlineCli):
         [key1:value1, key2:value2, ...]
 
 Description:
-    Run DAQ for experiment on station %s. If experiment does not exist, it will be
-    added to the DM database. If list of users or proposal id is specified, this command will also add roles for all users listed on the proposal.
+    Run DAQ for experiment on station %s. If experiment does not exist, it will be added to the DM database. If list of users or proposal id is specified, this command will also add roles for all users listed on the proposal.
         """ % (self.getStationName().lower(), self.getStationName()))
         self.checkArgs()
         self.checkCredentials()

src/python/dm/aps_beamline_tools/cli/uploadCli.py

+#!/usr/bin/env python
+
+import os
+from dm.aps_bss.api.apsBssApi import ApsBssApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
+from dm.daq_web_service.api.experimentDaqApi import ExperimentDaqApi
+from dm.common.constants import dmProcessingMode
+
+from dm.common.utility.ftpUtility import FtpUtility
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.aps_beamline_tools.cli.apsBeamlineCli import ApsBeamlineCli
+
+class UploadCli(ApsBeamlineCli):
+    def __init__(self, validArgCount=ApsBeamlineCli.ANY_NUMBER_OF_POSITIONAL_ARGS):
+        ApsBeamlineCli.__init__(self, validArgCount)
+        configManager = ConfigurationManager.getInstance()
+        self.allowedExperimentTypes = configManager.getAllowedExperimentTypes()
+        allowedExperimentTypesHelp = ''
+	self.defaultExperimentType = None
+        if self.allowedExperimentTypes:
+            allowedExperimentTypesHelp = ' Allowed types: %s' % self.allowedExperimentTypes
+	    self.defaultExperimentType = self.allowedExperimentTypes.split(',')[0]
+
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+        self.addOption('', '--data-directory', dest='dataDirectory', help='Experiment data directory.')
+
+        # Experiment options.
+        expGroup = 'Add/Update Experiment Options'
+        self.addOptionGroup(expGroup, prepend=True)
+        self.addOptionToGroup(expGroup, '', '--type', dest='typeName', default=self.defaultExperimentType, help='Experiment type name.%s' % allowedExperimentTypesHelp)
+        self.addOptionToGroup(expGroup, '', '--description', dest='description', help='Experiment description.')
+        self.addOptionToGroup(expGroup, '', '--start-date', dest='startDate', help='Experiment start date in format DD-MMM-YY.')
+        self.addOptionToGroup(expGroup, '', '--end-date', dest='endDate', help='Experiment end date in format DD-MMM-YY.')
+        self.addOptionToGroup(expGroup, '', '--users', dest='users', help='Comma-separated list of DM usernames to be added to the new experiment as users.')
+        self.addOptionToGroup(expGroup, '', '--proposal-id', dest='proposalId', help='Beamline proposal id. If specified, all users listed on the proposal will be added to the new experiment.')
+        self.addOptionToGroup(expGroup, '', '--run', dest='runName', help='Run name. If not specified, current run name is assumed for beamline proposal.')
+
+        # Upload Options
+        uploadGroup = 'Upload Options'
+        self.addOptionGroup(uploadGroup, prepend=True)
+        self.addOptionToGroup(uploadGroup, '', '--dest-directory', dest='destDirectory', help='Destination directory relative to experiment root path.')
+        self.addOptionToGroup(uploadGroup, '', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.')
+        self.addOptionToGroup(uploadGroup, '', '--reprocess', dest='reprocess', action='store_true', default=False, help='Reprocess source files that are already in storage, even if they have not been modified.')
+        self.addOptionToGroup(uploadGroup, '', '--processing-mode', dest='processingMode', default=dmProcessingMode.DM_PROCESSING_MODE_FILES, help='Processing mode can be one of %s (default: %s). In the "%s" mode files are processed individually, while in the "%s" mode processing plugins work on directories (if possible).' % (dmProcessingMode.DM_ALLOWED_PROCESSING_MODE_LIST, dmProcessingMode.DM_PROCESSING_MODE_FILES, dmProcessingMode.DM_PROCESSING_MODE_FILES, dmProcessingMode.DM_PROCESSING_MODE_DIRECTORY))
+        self.addOptionToGroup(uploadGroup, '', '--skip-plugins', dest='skipPlugins', help='Comma-separated list of plugins which should not process files.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.options.dataDirectory is None:
+            raise InvalidRequest('Experiment data directory must be provided.')
+        if self.getTypeName() and self.allowedExperimentTypes:
+            if self.getTypeName() not in self.allowedExperimentTypes.split(','):
+                raise InvalidRequest('Experiment type %s is not allowed on this station. Allowed types are: %s.' % (self.getTypeName(), self.allowedExperimentTypes))
+        if self.options.processingMode not in dmProcessingMode.DM_ALLOWED_PROCESSING_MODE_LIST:
+            raise InvalidRequest('Processing mode must be one of %s.' % dmProcessingMode.DM_ALLOWED_PROCESSING_MODE_LIST)
+
+    def updateDaqInfoFromOptions(self, daqInfo):
+        if self.options.reprocess:
+            daqInfo['reprocessFiles'] = True
+        if self.options.processHidden:
+            daqInfo['processHiddenFiles'] = True
+        if self.options.skipPlugins:
+            daqInfo['skipPlugins'] = self.options.skipPlugins
+        daqInfo['processingMode'] = self.options.processingMode
+        if self.options.destDirectory:
+            daqInfo['destDirectory'] = self.options.destDirectory
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getTypeName(self):
+        return self.options.typeName
+
+    def getDescription(self):
+        return self.options.description
+
+    def getStartDate(self):
+        return self.options.startDate
+
+    def getEndDate(self):
+        return self.options.endDate
+
+    def getProposalId(self):
+        proposalId = self.options.proposalId
+        if proposalId:
+            proposalId = int(proposalId)
+        return proposalId
+
+    def getUsers(self):
+        # Return list of users and beamline managers that can access data
+        users = self.options.users
+        if users:
+            users = users.split(',')
+        else:
+            users = []
+        beamlineManagers = self.beamlineManagers
+        if beamlineManagers:
+            beamlineManagers = beamlineManagers.split(',')
+        else:
+            beamlineManagers = []
+        # Remove duplicates by converting into set
+        return list(set(users+beamlineManagers))
+
+    def getDataDirectory(self):
+        dataDirectory = self.options.dataDirectory
+        replacementMap = os.environ.get('DM_DATA_DIRECTORY_MAP', '')
+        (scheme, host, port, dirPath) = FtpUtility.parseUrl(dataDirectory)
+        if dirPath and replacementMap:
+            # Map entries are expected to be in the form
+            # <original>|<replacement>;<original>|<replacement>;...
+            for entry in replacementMap.split(';'):
+                original = entry.split('|')[0]
+                replacement = entry.split('|')[1]
+                dirPath = dataDirectory.replace(original,replacement)
+        return FtpUtility.assembleUrl(scheme, host, port, dirPath)
+
+    def addOrUpdateExperiment(self):
+        dsExperimentApi = ExperimentDsApi(self.loginUsername, self.loginPassword, self.dsServiceHost, self.dsServicePort, self.serviceProtocol)
+        dsUserApi = UserDsApi(self.loginUsername, self.loginPassword, self.dsServiceHost, self.dsServicePort, self.serviceProtocol)
+
+        description = self.getDescription()
+        proposalId = self.getProposalId()
+        experimenters = []
+        if proposalId:
+            bssApi = ApsBssApi(loginFile=self.options.bssLoginFile)
+            proposal = bssApi.getBeamlineProposal(proposalId=proposalId, runName=self.options.runName)
+            experimenters = proposal.get('experimenters', [])
+            if not description:
+                description = '%s (Proposal id: %s)' % (proposal['title'], proposalId)
+
+
+        users = self.getUsers()
+        pis = []
+        for experimenter in experimenters:
+            badge = int(experimenter['badge'])
+            if not badge:
+                #print 'Skipping user %s due to invalid badge.' % lastName 
+                continue
+            username = 'd%s' % badge
+
+            # Clasify user
+            if experimenter.get('piFlag') == 'Y':
+                if not pis.count(username):
+                    pis.append(username)
+                if users.count(username):
+                    users.remove(username)
+            else:
+                if not users.count(username):
+                    users.append(username)
+
+        for username in users+pis:
+            # Check that user exists
+            dsUserApi.getUserByUsername(username)
+           
+
+        # Everything looks good, add experiment
+        try:
+            experiment = dsExperimentApi.getExperimentByName(self.getExperimentName())
+            experimentStation = experiment.get('experimentStation')
+            stationName = ''
+            if experimentStation:
+                stationName = experimentStation.get('name')
+            if stationName != self.getStationName():
+                raise InvalidRequest('Experiment %s already exists for station %s.' % (self.getExperimentName(), stationName))
+        except ObjectNotFound, ex:
+            experiment = dsExperimentApi.addExperiment(self.getExperimentName(), self.getStationName(), self.getTypeName(), description, self.getStartDate(), self.getEndDate())
+
+        # Add pis.
+        experimentUsernameList = experiment.get('experimentUsernameList', [])
+        experimentName = experiment['name']
+        roleName = 'PI'
+        for username in pis:
+            if username not in experimentUsernameList: 
+                dsUserApi.addUserExperimentRole(username, roleName, experimentName)
+        roleName = 'User'
+        for username in users:
+            if username not in experimentUsernameList: 
+                dsUserApi.addUserExperimentRole(username, roleName, experimentName)
+        if len(users+pis):
+            experiment = dsExperimentApi.getExperimentByName(experimentName)
+        return experiment
+
+    def startUpload(self):
+        daqExperimentApi = ExperimentDaqApi(self.loginUsername, self.loginPassword, self.daqServiceHost, self.daqServicePort, self.serviceProtocol)
+        daqInfo = self.splitArgsIntoDict()
+        self.updateDaqInfoFromOptions(daqInfo)
+        uploadInfo = daqExperimentApi.upload(self.getExperimentName(), self.getDataDirectory(), daqInfo=daqInfo)
+        return uploadInfo
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-%s-daq --experiment=EXPERIMENTNAME --data-directory=DATADIRECTORY
+        [--dest-directory=DESTDIRECTORY]
+        [--reprocess]
+        [--process-hidden]
+        [--processing-mode=PROCESSINGMODE]
+        [--skip-plugins=SKIPPLUGINS]
+        [--type=TYPENAME]
+        [--description=DESCRIPTION] 
+        [--start-date=STARTDATE] 
+        [--end-date=ENDDATE]
+        [--users=USERS]
+        [--proposal-id=PROPOSALID]
+        [--run=RUNNAME]
+        [key1:value1, key2:value2, ...]
+
+Description:
+    Run upload for experiment on station %s. If experiment does not exist, it will be added to the DM database. If list of users or proposal id is specified, this command will also add roles for all users listed on the proposal.
+        """ % (self.getStationName().lower(), self.getStationName()))
+        self.checkArgs()
+        self.checkCredentials()
+        experiment = self.addOrUpdateExperiment()
+        print 'EXPERIMENT INFO'
+        print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+        print 
+        uploadInfo = self.startUpload()
+        print 'UPLOAD INFO'
+        print uploadInfo.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = UploadCli()
+    cli.run()
+

src/python/dm/daq_web_service/cli/startDaqCli.py

@@ -18,6 +18,7 @@ class StartDaqCli(DaqWebServiceSessionCli):
         self.addOption('', '--upload-data-directory-on-exit', dest='uploadDataDirectoryOnExit', help='Data directory that will be uploaded automatically after DAQ is stopped.')
         self.addOption('', '--upload-dest-directory-on-exit', dest='uploadDestDirectoryOnExit', help='Destination directory relative to experiment root path for automatic upload after DAQ is stopped. Requires upload data directory to be specified.')
         self.addOption('', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.')
+        self.addOption('', '--skip-plugins', dest='skipPlugins', help='Comma-separated list of plugins which should not process files.')
 
     def checkArgs(self):
         if self.options.experimentName is None:
@@ -28,6 +29,8 @@ class StartDaqCli(DaqWebServiceSessionCli):
     def updateDaqInfoFromOptions(self, daqInfo):
         if self.options.processHidden:
             daqInfo['processHiddenFiles'] = True
+        if self.options.skipPlugins:
+            daqInfo['skipPlugins'] = self.options.skipPlugins
         if self.options.duration:
             duration = self.options.duration
             if duration.endswith('h'):

src/python/dm/daq_web_service/cli/uploadCli.py

@@ -14,7 +14,7 @@ class UploadCli(DaqWebServiceSessionCli):
         self.addOption('', '--reprocess', dest='reprocess', action='store_true', default=False, help='Reprocess source files that are already in storage, even if they have not been modified.')
         self.addOption('', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.')
         self.addOption('', '--processing-mode', dest='processingMode', default=dmProcessingMode.DM_PROCESSING_MODE_FILES, help='Processing mode can be one of %s (default: %s). In the "%s" mode files are processed individually, while in the "%s" mode processing plugins work on directories (if possible).' % (dmProcessingMode.DM_ALLOWED_PROCESSING_MODE_LIST, dmProcessingMode.DM_PROCESSING_MODE_FILES, dmProcessingMode.DM_PROCESSING_MODE_FILES, dmProcessingMode.DM_PROCESSING_MODE_DIRECTORY))
-        self.addOption('', '--skip-plugins', dest='skipPlugins', help='Comma-separated list of plugins which should not process the given directory.')
+        self.addOption('', '--skip-plugins', dest='skipPlugins', help='Comma-separated list of plugins which should not process files.')
 
     def checkArgs(self):
         if self.options.experimentName is None: