Skip to content
Snippets Groups Projects
Commit 74c9f02e authored by sveseli's avatar sveseli
Browse files

fixes for relative dest directory in case of local destinations

parent a88ca9e5
No related branches found
No related tags found
No related merge requests found
__version__ = "1.1 (2017.03.11)" __version__ = "1.1 (2017.03.13)"
...@@ -94,12 +94,13 @@ class FileProcessingThread(threading.Thread): ...@@ -94,12 +94,13 @@ class FileProcessingThread(threading.Thread):
statusMonitor.fileProcessingError(filePath, processingError, endProcessingTime) statusMonitor.fileProcessingError(filePath, processingError, endProcessingTime)
statusMonitor.updateStatus() statusMonitor.updateStatus()
self.logger.debug('No more %s retries left for file %s, remaining plugins will not process it' % (processorName, filePath)) self.logger.debug('No more %s retries left for file %s, remaining plugins will not process it' % (processorName, filePath))
return
else: else:
if statusMonitor: if statusMonitor:
statusMonitor.fileProcessingSkipped(processorName, filePath, processingError, endProcessingTime) statusMonitor.fileProcessingSkipped(processorName, filePath, processingError, endProcessingTime)
statusMonitor.updateStatus() statusMonitor.updateStatus()
self.logger.debug('No more %s retries left for file %s, skipping it' % (processorName, filePath)) self.logger.debug('No more %s retries left for file %s, skipping it' % (processorName, filePath))
return continue
else: else:
retryWaitPeriod = processor.getRetryWaitPeriodInSeconds() retryWaitPeriod = processor.getRetryWaitPeriodInSeconds()
self.logger.debug('%s may retry processing file %s after at least %s seconds' % (processorName, filePath, retryWaitPeriod)) self.logger.debug('%s may retry processing file %s after at least %s seconds' % (processorName, filePath, retryWaitPeriod))
......
...@@ -61,6 +61,6 @@ class FileProcessor: ...@@ -61,6 +61,6 @@ class FileProcessor:
def setSkipOnFailure(self, skipOnFailure): def setSkipOnFailure(self, skipOnFailure):
self.configDict['skipOnFailure'] = skipOnFailure self.configDict['skipOnFailure'] = skipOnFailure
def getSkipOnFailure(self, skipOnFailure): def getSkipOnFailure(self):
return self.configDict.get('skipOnFailure', False) return self.configDict.get('skipOnFailure', False)
...@@ -27,7 +27,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin): ...@@ -27,7 +27,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
self.deleteOriginal = deleteOriginal self.deleteOriginal = deleteOriginal
self.directoryTransferCommand = directoryTransferCommand self.directoryTransferCommand = directoryTransferCommand
self.pluginMustProcessFiles = pluginMustProcessFiles self.pluginMustProcessFiles = pluginMustProcessFiles
if nRetries: if nRetries is not None:
self.setNumberOfRetries(nRetries) self.setNumberOfRetries(nRetries)
self.setSkipOnFailure(skipOnFailure) self.setSkipOnFailure(skipOnFailure)
......
...@@ -5,6 +5,7 @@ import os ...@@ -5,6 +5,7 @@ import os
from fileTransferPlugin import FileTransferPlugin from fileTransferPlugin import FileTransferPlugin
from dm.common.utility.osUtility import OsUtility from dm.common.utility.osUtility import OsUtility
from dm.common.utility.fileUtility import FileUtility from dm.common.utility.fileUtility import FileUtility
from dm.common.utility.ftpUtility import FtpUtility
from dm.common.exceptions.fileProcessingError import FileProcessingError from dm.common.exceptions.fileProcessingError import FileProcessingError
from dm.common.utility.dmSubprocess import DmSubprocess from dm.common.utility.dmSubprocess import DmSubprocess
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
...@@ -24,7 +25,7 @@ class RsyncFileTransferPlugin(FileTransferPlugin): ...@@ -24,7 +25,7 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
self.remoteMd5Sum = remoteMd5Sum self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal self.deleteOriginal = deleteOriginal
self.pluginMustProcessFiles = pluginMustProcessFiles self.pluginMustProcessFiles = pluginMustProcessFiles
if nRetries: if nRetries is not None:
self.setNumberOfRetries(nRetries) self.setNumberOfRetries(nRetries)
self.setSkipOnFailure(skipOnFailure) self.setSkipOnFailure(skipOnFailure)
...@@ -76,8 +77,9 @@ class RsyncFileTransferPlugin(FileTransferPlugin): ...@@ -76,8 +77,9 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl)) self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl))
command = self.command command = self.command
if destDirectory: if destDirectory:
targetDirectory = '%s/%s' % (storageDirectory,destDirectory) (scheme, host, port, targetDirectory) = FtpUtility.parseUrl(destUrl)
command = self.RSYNC_WITH_MKDIR_COMMAND % targetDirectory command = self.RSYNC_WITH_MKDIR_COMMAND % targetDirectory
command = self.replaceTemplates(command, fileInfo)
self.start(src=srcUrl, dest=destUrl, command=command, templateInfo=fileInfo, cwd=dataDirectory) self.start(src=srcUrl, dest=destUrl, command=command, templateInfo=fileInfo, cwd=dataDirectory)
# Get remote checksum # Get remote checksum
...@@ -116,8 +118,9 @@ class RsyncFileTransferPlugin(FileTransferPlugin): ...@@ -116,8 +118,9 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl)) self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl))
command = self.DIRECTORY_TRANSFER_COMMAND command = self.DIRECTORY_TRANSFER_COMMAND
if destDirectory: if destDirectory:
targetDirectory = '%s/%s' % (storageDirectory,destDirectory) (scheme, host, port, targetDirectory) = FtpUtility.parseUrl(destUrl)
command = self.DIRECTORY_TRANSFER_WITH_MKDIR_COMMAND % targetDirectory command = self.DIRECTORY_TRANSFER_WITH_MKDIR_COMMAND % targetDirectory
command = self.replaceTemplates(command, uploadInfo)
self.start(src=srcUrl, dest=destUrl, command=command, templateInfo=uploadInfo, cwd=dataDirectory) self.start(src=srcUrl, dest=destUrl, command=command, templateInfo=uploadInfo, cwd=dataDirectory)
####################################################################### #######################################################################
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment