diff --git a/etc/dm.deploy.conf b/etc/dm.deploy.conf
index 3248a75417e4225466e56dac974175af2e9582fa..444319ce23636ce2958cccaedd1aae2bb83be020 100644
--- a/etc/dm.deploy.conf
+++ b/etc/dm.deploy.conf
@@ -15,5 +15,5 @@ DM_DAQ_WEB_SERVICE_HOST=DM_HOSTNAME
DM_DAQ_WEB_SERVICE_PORT=33336
DM_CAT_WEB_SERVICE_HOST=DM_HOSTNAME
DM_CAT_WEB_SERVICE_PORT=44436
-DM_SOFTWARE_VERSION="0.5 (DM_DATE)"
+DM_SOFTWARE_VERSION="0.6 (DM_DATE)"
diff --git a/src/python/dm/__init__.py b/src/python/dm/__init__.py
index 3c1013ed6b3c8419c2cca228d71503797a14e224..5010cdf402a47e35582a0eaed250b96bc38f99ad 100644
--- a/src/python/dm/__init__.py
+++ b/src/python/dm/__init__.py
@@ -1 +1 @@
-__version__ = "0.3 (2015.09.21)"
+__version__ = "0.6 (2015.11.06)"
diff --git a/src/python/dm/common/processing/plugins/fileTransferPlugin.py b/src/python/dm/common/processing/plugins/fileTransferPlugin.py
index 196377f8ae298b45125dda2697f378b5e71bbf31..0c23098327e96cc080e72cf53657eef0910f0a1e 100755
--- a/src/python/dm/common/processing/plugins/fileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/fileTransferPlugin.py
@@ -27,12 +27,12 @@ class FileTransferPlugin(FileProcessor):
storageHost = experiment.get('storageHost')
storageDirectory = experiment.get('storageDirectory')
destUrl = self.getDestUrl(storageHost, storageDirectory)
- # Use relative path with respect to data directory as a source
- os.chdir(dataDirectory)
srcUrl = self.getSrcUrl(filePath, dataDirectory)
self.start(srcUrl, destUrl)
def getSrcUrl(self, filePath, dataDirectory):
+ # Use relative path with respect to data directory as a source
+ os.chdir(dataDirectory)
srcUrl = os.path.relpath(filePath, dataDirectory)
return srcUrl
diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
index abf5f2cd4184c8f6e92f797ec0bc5434f84174df..6999ccdfd114a7cfe4bb5318491b5af3c9ed5823 100755
--- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
@@ -1,18 +1,19 @@
#!/usr/bin/env python
from fileTransferPlugin import FileTransferPlugin
+
class GridftpFileTransferPlugin(FileTransferPlugin):
COMMAND = 'globus-url-copy -c -cd -r -vb -tcp-bs 512K -p 1 -sync -sync-level 2'
- def __init__(self, src=None, dest=None, command=GridftpFileTransferPlugin.COMMAND):
+ def __init__(self, src=None, dest=None, command=COMMAND):
FileTransferPlugin.__init__(self, command, src, dest)
def getSrcUrl(self, filePath, dataDirectory):
if self.src is None:
srcUrl = 'file://%s' % filePath
else:
- srcUrl = '%s/%s' % filePath
+ srcUrl = '%s/%s' % (self.src,filePath)
return srcUrl
def getDestUrl(self, storageHost, storageDirectory):
diff --git a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
index 5f42e727ec93c53d88a8db904cc4705b82203715..ecc2d56b5fdd82934009bfc4c07f051d8f802723 100755
--- a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
+++ b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
@@ -10,7 +10,6 @@ import time
from dm.common.objects.experiment import Experiment
from dm.common.objects.dmObjectManager import DmObjectManager
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.common.utility.osUtility import OsUtility
from dm.common.objects.observedFile import ObservedFile
from dm.common.processing.fileProcessingManager import FileProcessingManager
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
@@ -20,14 +19,12 @@ from fileSystemObserver import FileSystemObserver
class ExperimentSessionControllerImpl(DmObjectManager):
""" Experiment session controller implementation class. """
- DAQ_PERMISSIONS_MODE = 0777
-
def __init__(self):
DmObjectManager.__init__(self)
self.dsExperimentApi = DsRestApiFactory.getExperimentRestApi()
def startDaq(self, experimentName, dataDirectory, daqInfo):
- OsUtility.createDir(dataDirectory, mode=self.DAQ_PERMISSIONS_MODE)
+ FileSystemObserver.getInstance().createDirectory(dataDirectory)
if daqInfo is None:
daqInfo={}
daqInfo['experimentName'] = experimentName
@@ -69,7 +66,7 @@ class ExperimentSessionControllerImpl(DmObjectManager):
if storageDirectory is None:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
dataDirectory = daqInfo.get('dataDirectory')
- filePaths = OsUtility.findFiles(dataDirectory)
+ filePaths = FileSystemObserver.getInstance().getFiles(dataDirectory)
fileProcessingManager = FileProcessingManager.getInstance()
for filePath in filePaths:
fileInfo = ObservedFile(filePath=filePath, dataDirectory=dataDirectory, experiment=experiment)
diff --git a/src/python/dm/daq_web_service/service/impl/fileSystemObserver.py b/src/python/dm/daq_web_service/service/impl/fileSystemObserver.py
index c9e4bfda185491db1e0e10cdce73c0da6f945cd0..6613f8202e4c09da0fc8992a9565c142d2bef1b4 100755
--- a/src/python/dm/daq_web_service/service/impl/fileSystemObserver.py
+++ b/src/python/dm/daq_web_service/service/impl/fileSystemObserver.py
@@ -63,6 +63,15 @@ class FileSystemObserver(threading.Thread,Singleton):
self.fileSystemObserverAgent = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.fileSystemObserverAgent.setFileSystemObserver(self)
+ @ThreadingUtility.synchronize
+ def createDirectory(self, dataDirectory):
+ self.fileSystemObserverAgent.createDirectory(dataDirectory)
+
+ @ThreadingUtility.synchronize
+ def getFiles(self, dataDirectory):
+ self.logger.debug('Agent is retrieving files for %s' % dataDirectory)
+ return self.fileSystemObserverAgent.getFiles(dataDirectory)
+
@ThreadingUtility.synchronize
def startObservingPath(self, dataDirectory, experiment):
self.logger.debug('Agent is starting observer for %s' % dataDirectory)
diff --git a/src/python/dm/daq_web_service/service/impl/fileSystemObserverAgent.py b/src/python/dm/daq_web_service/service/impl/fileSystemObserverAgent.py
index 3603f19dfad70bc8b9ceb7ddd1f5725e35428aa0..6077a8044546579d92d0f9c3c7bd158f46928b2d 100644
--- a/src/python/dm/daq_web_service/service/impl/fileSystemObserverAgent.py
+++ b/src/python/dm/daq_web_service/service/impl/fileSystemObserverAgent.py
@@ -11,6 +11,12 @@ class FileSystemObserverAgent:
def setFileSystemObserver(self, fileSystemObserver):
self.fileSystemObserver = fileSystemObserver
+ def createDirectory(self, dataDirectory):
+ pass
+
+ def getFiles(self, dataDirectory):
+ pass
+
def startObservingPath(self, dataDirectory, experiment):
pass
diff --git a/src/python/dm/daq_web_service/service/impl/watchdogFileSystemObserverAgent.py b/src/python/dm/daq_web_service/service/impl/watchdogFileSystemObserverAgent.py
index 81def9895419d17a872f07c27e3b035bf7a1c7d0..c709367628be90c25dbafed643d8fcdf9227ee85 100644
--- a/src/python/dm/daq_web_service/service/impl/watchdogFileSystemObserverAgent.py
+++ b/src/python/dm/daq_web_service/service/impl/watchdogFileSystemObserverAgent.py
@@ -1,16 +1,29 @@
#!/usr/bin/env python
+from dm.common.utility.osUtility import OsUtility
+
from watchdog.observers.polling import PollingObserver
from fileSystemObserverAgent import FileSystemObserverAgent
from dmFileSystemEventHandler import DmFileSystemEventHandler
class WatchdogFileSystemObserverAgent(FileSystemObserverAgent):
+ DAQ_PERMISSIONS_MODE = 0777
+
def __init__(self):
FileSystemObserverAgent.__init__(self)
self.observer = PollingObserver()
self.observedWatchDict = {}
+ def createDirectory(self, dataDirectory):
+ try:
+ OsUtility.createDir(dataDirectory, mode=self.DAQ_PERMISSIONS_MODE)
+ except Exception, ex:
+ self.logger.warn('Unable to create directory %s: %s' % (dataDirectory, ex))
+
+ def getFiles(self, dataDirectory):
+ return OsUtility.findFiles(dataDirectory)
+
def startObservingPath(self, dataDirectory, experiment):
self.logger.debug('Starting observer for %s' % dataDirectory)
eventHandler = DmFileSystemEventHandler(self.fileSystemObserver, dataDirectory, experiment)
diff --git a/src/python/dm/ds_web_service/service/impl/experimentSessionControllerImpl.py b/src/python/dm/ds_web_service/service/impl/experimentSessionControllerImpl.py
index 7958faa7493b4c9d37b86520ddd220428bcd6291..05af7916b994e7192afd89fa2551ed05103f73b2 100755
--- a/src/python/dm/ds_web_service/service/impl/experimentSessionControllerImpl.py
+++ b/src/python/dm/ds_web_service/service/impl/experimentSessionControllerImpl.py
@@ -51,9 +51,8 @@ class ExperimentSessionControllerImpl(DmObjectManager):
def updateExperiment(self, name):
experiment = self.experimentDbApi.getExperimentWithUsers(name)
- if experiment.get('startDate') is None:
- raise InvalidRequest('Experiment %s has not been started.' % name)
- ExperimentManager.getInstance().updateExperimentGroupUsers(experiment)
+ if experiment.get('startDate') is not None:
+ ExperimentManager.getInstance().updateExperimentGroupUsers(experiment)
return experiment
def stopExperiment(self, name):