diff --git a/src/python/dm/common/processing/fileProcessingManager.py b/src/python/dm/common/processing/fileProcessingManager.py index 74dc397d26385f116f213501d83b6bccb8e16469..8a5ea3db991ec5d050d90535ff3323530677b28a 100755 --- a/src/python/dm/common/processing/fileProcessingManager.py +++ b/src/python/dm/common/processing/fileProcessingManager.py @@ -84,8 +84,8 @@ class FileProcessingManager(threading.Thread,Singleton): # Each plugin calculates list of files that need to be processed # Final result is union of all plugins def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo): - if ValueUtility.toBoolean(uploadInfo.get('processAllFiles')): - del uploadInfo['processAllFiles'] + if ValueUtility.toBoolean(uploadInfo.get('reprocessFiles')): + del uploadInfo['reprocessFiles'] return filePathsDict checkedFilePathsDict = {} for processorKey in self.fileProcessorKeyList: diff --git a/src/python/dm/daq_web_service/cli/startDaqCli.py b/src/python/dm/daq_web_service/cli/startDaqCli.py index ea7cf18d615c97c70ea1d41ffd8db6d5dfecd7ab..7570f412c498416c7f6327ee41dac5da0b0ca608 100755 --- a/src/python/dm/daq_web_service/cli/startDaqCli.py +++ b/src/python/dm/daq_web_service/cli/startDaqCli.py @@ -9,6 +9,7 @@ class StartDaqCli(DaqWebServiceSessionCli): DaqWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS) self.addOption('', '--experiment', dest='experimentName', help='Experiment name.') self.addOption('', '--data-directory', dest='dataDirectory', help='Experiment data directory. If specified string does not already contain file server URL, value of the %s environment variable will be prepended to it.' % self.DM_FILE_SERVER_URL_ENV_VAR) + self.addOption('', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.') def checkArgs(self): if self.options.experimentName is None: @@ -16,9 +17,14 @@ class StartDaqCli(DaqWebServiceSessionCli): if self.options.dataDirectory is None: raise InvalidRequest('Experiment data directory must be provided.') + def updateDaqInfoFromOptions(self, daqInfo): + if self.options.processHidden: + daqInfo['processHiddenFiles'] = True + def runCommand(self): self.parseArgs(usage=""" dm-start-daq --experiment=EXPERIMENTNAME --data-directory=DATADIRECTORY + [--process-hidden] [key1:value1, key2:value2, ...] Description: @@ -29,6 +35,7 @@ Description: self.checkArgs() api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol()) daqInfo = self.splitArgsIntoDict() + self.updateDaqInfoFromOptions(daqInfo) daqInfo = api.startDaq(self.getExperimentName(), self.getDataDirectory(), daqInfo=daqInfo) print daqInfo.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat()) diff --git a/src/python/dm/daq_web_service/cli/uploadCli.py b/src/python/dm/daq_web_service/cli/uploadCli.py index 80f5c3caa3ef07482eb7bcc8fc62505b32d9edc9..dae08440cc84b062fdacf857b8ce1a7f8158d8d2 100755 --- a/src/python/dm/daq_web_service/cli/uploadCli.py +++ b/src/python/dm/daq_web_service/cli/uploadCli.py @@ -9,6 +9,8 @@ class UploadCli(DaqWebServiceSessionCli): DaqWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS) self.addOption('', '--experiment', dest='experimentName', help='Experiment name.') self.addOption('', '--data-directory', dest='dataDirectory', help='Experiment data directory. If specified string does not already contain file server URL, value of the %s environment variable will be prepended to it.' % self.DM_FILE_SERVER_URL_ENV_VAR) + self.addOption('', '--reprocess', dest='reprocess', action='store_true', default=False, help='Reprocess source files that are already in storage, even if they have not been modified.') + self.addOption('', '--process-hidden', dest='processHidden', action='store_true', default=False, help='Process hidden source files.') def checkArgs(self): if self.options.experimentName is None: @@ -16,9 +18,17 @@ class UploadCli(DaqWebServiceSessionCli): if self.options.dataDirectory is None: raise InvalidRequest('Experiment data directory must be provided.') + def updateDaqInfoFromOptions(self, daqInfo): + if self.options.reprocess: + daqInfo['reprocessFiles'] = True + if self.options.processHidden: + daqInfo['processHiddenFiles'] = True + def runCommand(self): self.parseArgs(usage=""" dm-upload --experiment=EXPERIMENTNAME --data-directory=DATADIRECTORY + [--reprocess] + [--process-hidden] [key1:value1, key2:value2, ...] Description: @@ -30,6 +40,7 @@ Description: self.checkArgs() api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol()) daqInfo = self.splitArgsIntoDict() + self.updateDaqInfoFromOptions(daqInfo) uploadInfo = api.upload(self.getExperimentName(), self.getDataDirectory(), daqInfo=daqInfo) print uploadInfo.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())