From 1655dcbe21dde20a13047b2f726d5d01a727e3a0 Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Tue, 14 Apr 2015 13:21:03 +0000
Subject: [PATCH] changed file processing interface; added support for relative
file paths (with respect to daq directory)
---
.../common/plugins/fileProcessorInterface.py | 2 +-
.../dm/common/plugins/fileTransferPlugin.py | 33 ++++++++++++++-----
.../common/plugins/rsyncFileTransferPlugin.py | 2 +-
3 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/src/python/dm/common/plugins/fileProcessorInterface.py b/src/python/dm/common/plugins/fileProcessorInterface.py
index 03546ebd..f9aef1ce 100755
--- a/src/python/dm/common/plugins/fileProcessorInterface.py
+++ b/src/python/dm/common/plugins/fileProcessorInterface.py
@@ -4,6 +4,6 @@ import abc
class FileProcessorInterface:
@abc.abstractmethod
- def processFile(self, srcPath):
+ def processFile(self, filePath, daqPath, experiment):
return NotImplemented
diff --git a/src/python/dm/common/plugins/fileTransferPlugin.py b/src/python/dm/common/plugins/fileTransferPlugin.py
index 969fd6d2..1b369025 100755
--- a/src/python/dm/common/plugins/fileTransferPlugin.py
+++ b/src/python/dm/common/plugins/fileTransferPlugin.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+import os
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.dmSubprocess import DmSubprocess
from dm.common.exceptions.invalidArgument import InvalidArgument
@@ -17,11 +18,17 @@ class FileTransferPlugin(FileProcessorInterface):
self.command = command
self.subprocess = None
- def processFile(self, src):
- self.start(src)
+ def processFile(self, filePath, daqPath, experiment):
+ storageHost = experiment.get('storageHost')
+ storageDirectory = experiment.get('storageDirectory')
+ dest = '%s:%s' % (storageHost, storageDirectory)
+ # Use relative path with respect to daq directory as a source
+ os.chdir(daqPath)
+ src = os.path.relpath(filePath, daqPath)
+ self.start(src, dest)
- def getFullCommand(self):
- return '%s %s %s' % (self.command, self.src, self.dest)
+ def getFullCommand(self, src, dest):
+ return '%s %s %s' % (self.command, src, dest)
def setSrc(self, src):
self.src = src
@@ -29,12 +36,20 @@ class FileTransferPlugin(FileProcessorInterface):
def setDest(self, dest):
self.dest = dest
- def start(self, src=None):
- if src is not None:
- self.src = src
- if not self.src or not self.dest:
+ def start(self, src=None, dest=None):
+ # Use preconfigured source if provided source is None
+ fileSrc = src
+ if src is None:
+ fileSrc = self.src
+ # Use provided destination only if preconfigured destination is None
+ # Plugins may have desired destination preconfigured for all files
+ fileDest = self.dest
+ if self.dest is None:
+ fileDest = dest
+
+ if not fileSrc or not fileDest:
raise InvalidRequest('Both source and destination must be non-empty strings.')
- self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand())
+ self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand(fileSrc, fileDest))
return self.subprocess.run()
def wait(self):
diff --git a/src/python/dm/common/plugins/rsyncFileTransferPlugin.py b/src/python/dm/common/plugins/rsyncFileTransferPlugin.py
index 16b10a76..3550d0cf 100755
--- a/src/python/dm/common/plugins/rsyncFileTransferPlugin.py
+++ b/src/python/dm/common/plugins/rsyncFileTransferPlugin.py
@@ -3,7 +3,7 @@
from fileTransferPlugin import FileTransferPlugin
class RsyncFileTransferPlugin(FileTransferPlugin):
- COMMAND = 'rsync -arvlP'
+ COMMAND = 'rsync -arvlPR'
def __init__(self, src=None, dest=None):
FileTransferPlugin.__init__(self, self.COMMAND, src, dest)
--
GitLab