added reprocess and process hidden options to startDaq and upload commands

src/python/dm/common/processing/fileProcessingManager.py

@@ -84,8 +84,8 @@ class FileProcessingManager(threading.Thread,Singleton):
     # Each plugin calculates list of files that need to be processed
     # Final result is union of all plugins
     def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
-        if ValueUtility.toBoolean(uploadInfo.get('processAllFiles')):
-            del uploadInfo['processAllFiles']
+        if ValueUtility.toBoolean(uploadInfo.get('reprocessFiles')):
+            del uploadInfo['reprocessFiles']
             return filePathsDict
         checkedFilePathsDict = {}
         for processorKey in self.fileProcessorKeyList:

src/python/dm/daq_web_service/cli/startDaqCli.py

@@ -9,6 +9,7 @@ class StartDaqCli(DaqWebServiceSessionCli):
         DaqWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
         self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
         self.addOption('', '--data-directory', dest='dataDirectory', help='Experiment data directory. If specified string does not already contain file server URL, value of the %s environment variable will be prepended to it.' % self.DM_FILE_SERVER_URL_ENV_VAR)
+        self.addOption('', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.')
 
     def checkArgs(self):
         if self.options.experimentName is None:
@@ -16,9 +17,14 @@ class StartDaqCli(DaqWebServiceSessionCli):
         if self.options.dataDirectory is None:
             raise InvalidRequest('Experiment data directory must be provided.')
 
+    def updateDaqInfoFromOptions(self, daqInfo):
+        if self.options.processHidden:
+            daqInfo['processHiddenFiles'] = True
+
     def runCommand(self):
         self.parseArgs(usage="""
     dm-start-daq --experiment=EXPERIMENTNAME --data-directory=DATADIRECTORY
+        [--process-hidden]
         [key1:value1, key2:value2, ...]
 
 Description:
@@ -29,6 +35,7 @@ Description:
         self.checkArgs()
         api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
         daqInfo = self.splitArgsIntoDict()
+        self.updateDaqInfoFromOptions(daqInfo)
         daqInfo = api.startDaq(self.getExperimentName(), self.getDataDirectory(), daqInfo=daqInfo)
         print daqInfo.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
 

src/python/dm/daq_web_service/cli/uploadCli.py

@@ -9,6 +9,8 @@ class UploadCli(DaqWebServiceSessionCli):
         DaqWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
         self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
         self.addOption('', '--data-directory', dest='dataDirectory', help='Experiment data directory. If specified string does not already contain file server URL, value of the %s environment variable will be prepended to it.' % self.DM_FILE_SERVER_URL_ENV_VAR)
+        self.addOption('', '--reprocess', dest='reprocess', action='store_true', default=False, help='Reprocess source files that are already in storage, even if they have not been modified.')
+        self.addOption('', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.')
 
     def checkArgs(self):
         if self.options.experimentName is None:
@@ -16,9 +18,17 @@ class UploadCli(DaqWebServiceSessionCli):
         if self.options.dataDirectory is None:
             raise InvalidRequest('Experiment data directory must be provided.')
 
+    def updateDaqInfoFromOptions(self, daqInfo):
+        if self.options.reprocess:
+            daqInfo['reprocessFiles'] = True
+        if self.options.processHidden:
+            daqInfo['processHiddenFiles'] = True
+        
     def runCommand(self):
         self.parseArgs(usage="""
     dm-upload --experiment=EXPERIMENTNAME --data-directory=DATADIRECTORY
+        [--reprocess]
+        [--process-hidden]
         [key1:value1, key2:value2, ...]
 
 Description:
@@ -30,6 +40,7 @@ Description:
         self.checkArgs()
         api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
         daqInfo = self.splitArgsIntoDict()
+        self.updateDaqInfoFromOptions(daqInfo)
         uploadInfo = api.upload(self.getExperimentName(), self.getDataDirectory(), daqInfo=daqInfo)
         print uploadInfo.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())