From f8b03320207bd2f430c5edfd3bd9ed8165750436 Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Tue, 14 Apr 2015 19:10:54 +0000
Subject: [PATCH] first stab at file upload functionality
---
bin/{dm-upload-data => dm-start-upload} | 2 +-
.../dm/daq_web_service/api/experimentRestApi.py | 12 ++++++++++++
.../cli/{uploadDataCli.py => startUploadCli.py} | 10 +++++-----
.../service/experimentRouteDescriptor.py | 9 +++++++++
.../service/experimentSessionController.py | 16 ++++++++++++++++
.../impl/experimentSessionControllerImpl.py | 17 +++++++++++++++++
6 files changed, 60 insertions(+), 6 deletions(-)
rename bin/{dm-upload-data => dm-start-upload} (79%)
rename src/python/dm/daq_web_service/cli/{uploadDataCli.py => startUploadCli.py} (83%)
diff --git a/bin/dm-upload-data b/bin/dm-start-upload
similarity index 79%
rename from bin/dm-upload-data
rename to bin/dm-start-upload
index 5c48db6f..982fde12 100755
--- a/bin/dm-upload-data
+++ b/bin/dm-start-upload
@@ -12,6 +12,6 @@ if [ -z $DM_ROOT_DIR ]; then
source $setupFile > /dev/null
fi
-$DM_ROOT_DIR/src/python/dm/daq_web_service/cli/uploadDataCli.py $@
+$DM_ROOT_DIR/src/python/dm/daq_web_service/cli/startUploadCli.py $@
diff --git a/src/python/dm/daq_web_service/api/experimentRestApi.py b/src/python/dm/daq_web_service/api/experimentRestApi.py
index 036f9403..07222dfd 100755
--- a/src/python/dm/daq_web_service/api/experimentRestApi.py
+++ b/src/python/dm/daq_web_service/api/experimentRestApi.py
@@ -34,6 +34,18 @@ class ExperimentRestApi(DaqRestApi):
responseDict = self.sendSessionRequest(url=url, method='POST')
return Experiment(responseDict)
+ @DaqRestApi.execute
+ def startUpload(self, name, dataDirectory):
+ url = '%s/experiments/startUpload' % (self.getContextRoot())
+ if name is None or not len(name):
+ raise InvalidRequest('Experiment name must be provided.')
+ url += '?name=%s' % Encoder.encode(name)
+ if dataDirectory is None or not len(dataDirectory):
+ raise InvalidRequest('Experiment data directory must be provided.')
+ url += '&dataDirectory=%s' % Encoder.encode(dataDirectory)
+ responseDict = self.sendSessionRequest(url=url, method='POST')
+ return Experiment(responseDict)
+
#######################################################################
# Testing.
diff --git a/src/python/dm/daq_web_service/cli/uploadDataCli.py b/src/python/dm/daq_web_service/cli/startUploadCli.py
similarity index 83%
rename from src/python/dm/daq_web_service/cli/uploadDataCli.py
rename to src/python/dm/daq_web_service/cli/startUploadCli.py
index 79371f68..ad11e36d 100755
--- a/src/python/dm/daq_web_service/cli/uploadDataCli.py
+++ b/src/python/dm/daq_web_service/cli/startUploadCli.py
@@ -4,9 +4,9 @@ from daqWebServiceSessionCli import DaqWebServiceSessionCli
from dm.daq_web_service.api.experimentRestApi import ExperimentRestApi
from dm.common.exceptions.invalidRequest import InvalidRequest
-class UploadDataCli(DaqWebServiceSessionCli):
+class StartUploadCli(DaqWebServiceSessionCli):
def __init__(self):
- DaqWebServiceCli.__init__(self)
+ DaqWebServiceSessionCli.__init__(self)
self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
self.addOption('', '--data-directory', dest='dataDirectory', help='Experiment data directory.')
@@ -24,19 +24,19 @@ class UploadDataCli(DaqWebServiceSessionCli):
def runCommand(self):
self.parseArgs(usage="""
- dm-upload-data --name=EXPERIMENTNAME --data-directory=DATADIRECTORY
+ dm-start-upload --name=EXPERIMENTNAME --data-directory=DATADIRECTORY
Description:
Schedules data upload for a given experiment.
""")
self.checkArgs()
api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
- experiment = api.uploadData(self.getExperimentName(), self.getDataDirectory())
+ experiment = api.startUpload(self.getExperimentName(), self.getDataDirectory())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
#######################################################################
# Run command.
if __name__ == '__main__':
- cli = UploadDataCli()
+ cli = StartUploadCli()
cli.run()
diff --git a/src/python/dm/daq_web_service/service/experimentRouteDescriptor.py b/src/python/dm/daq_web_service/service/experimentRouteDescriptor.py
index 615e072c..982d0f50 100755
--- a/src/python/dm/daq_web_service/service/experimentRouteDescriptor.py
+++ b/src/python/dm/daq_web_service/service/experimentRouteDescriptor.py
@@ -37,6 +37,15 @@ class ExperimentRouteDescriptor:
'method' : ['POST']
},
+ # Start experiment data upload
+ {
+ 'name' : 'startUpload',
+ 'path' : '%s/experiments/startUpload' % contextRoot,
+ 'controller' : experimentSessionController,
+ 'action' : 'startUpload',
+ 'method' : ['POST']
+ },
+
]
return routes
diff --git a/src/python/dm/daq_web_service/service/experimentSessionController.py b/src/python/dm/daq_web_service/service/experimentSessionController.py
index bf36488e..532e663c 100755
--- a/src/python/dm/daq_web_service/service/experimentSessionController.py
+++ b/src/python/dm/daq_web_service/service/experimentSessionController.py
@@ -43,3 +43,19 @@ class ExperimentSessionController(DmSessionController):
self.logger.debug('Returning: %s' % response)
return response
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isLoggedIn())
+ @DmSessionController.execute
+ def startUpload(self, **kwargs):
+ name = kwargs.get('name')
+ if name is None or not len(name):
+ raise InvalidRequest('Missing experiment name.')
+ name = Encoder.decode(name)
+ dataDirectory = kwargs.get('dataDirectory')
+ if dataDirectory is None or not len(dataDirectory):
+ raise InvalidRequest('Missing experiment data directory.')
+ dataDirectory = Encoder.decode(dataDirectory)
+ response = self.experimentSessionControllerImpl.startUpload(name, dataDirectory).getFullJsonRep()
+ self.logger.debug('Returning: %s' % response)
+ return response
+
diff --git a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
index e5d73a5a..83242c0a 100755
--- a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
+++ b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
@@ -9,9 +9,12 @@ import time
from dm.common.objects.experiment import Experiment
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.utility.osUtility import OsUtility
+from dm.common.objects.observedFile import ObservedFile
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
from experimentTracker import ExperimentTracker
from fileSystemObserver import FileSystemObserver
+from fileProcessingManager import FileProcessingManager
class ExperimentSessionControllerImpl(DmObjectManager):
""" Experiment session controller implementation class. """
@@ -40,3 +43,17 @@ class ExperimentSessionControllerImpl(DmObjectManager):
experiment['daqEndTime'] = time.time()
FileSystemObserver.getInstance().stopObservingPath(dataDirectory, experiment)
return experiment
+
+ def startUpload(self, name, dataDirectory):
+ experiment = self.dsExperimentApi.getExperimentByName(name)
+ storageDirectory = experiment.get('storageDirectory')
+ if storageDirectory is None:
+ raise InvalidRequest('Experiment %s has not been started.' % name)
+ filePaths = OsUtility.findFiles(dataDirectory)
+ fileProcessingManager = FileProcessingManager.getInstance()
+ for filePath in filePaths:
+ observedFile = ObservedFile(filePath=filePath, daqPath=dataDirectory, experiment=experiment)
+ fileProcessingManager.processObservedFile(observedFile)
+ ExperimentTracker.getInstance().put(name, experiment)
+ return experiment
+
--
GitLab