From 5b57c7240a07526b9e2c0475ece2b643c77c189d Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Fri, 7 Oct 2016 20:24:43 +0000
Subject: [PATCH] merge from 0.13
---
.../plugins/gridftpFileTransferPlugin.py | 16 ++++++++++---
src/python/dm/common/utility/ftpUtility.py | 5 ++--
src/python/dm/common/utility/sftpUtility.py | 23 +++++++++++++------
.../service/experimentSessionController.py | 2 +-
4 files changed, 33 insertions(+), 13 deletions(-)
diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
index acd76c7b..03c56c2a 100755
--- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
@@ -28,6 +28,16 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
self.directoryTransferCommand = directoryTransferCommand
self.pluginMustProcessFiles = pluginMustProcessFiles
+ def replaceSpecialCharacters(self, url):
+ replacementMap = {
+ '#' : '%23',
+ ' ' : '%20',
+ '~' : '%7E',
+ }
+ for (original, replacement) in replacementMap.items():
+ url = url.replace(original,replacement)
+ return url
+
def getSrcUrl(self, filePath, dataDirectory):
(scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
if scheme:
@@ -36,7 +46,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
srcUrl = 'file://%s' % filePath
else:
srcUrl = '%s/%s' % (self.src,filePath)
- return srcUrl
+ return self.replaceSpecialCharacters(srcUrl)
def getDestUrl(self, filePath, dataDirectory, storageHost, storageDirectory):
(scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
@@ -46,7 +56,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
destUrl = '%s/%s/%s' % (self.dest, dirName, fileName)
else:
destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName)
- return destUrl
+ return self.replaceSpecialCharacters(destUrl)
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
if not self.pluginMustProcessFiles:
@@ -105,7 +115,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
# Calculate checksum
statUtility = self.statUtility
if not statUtility:
- (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
+ (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(srcUrl, defaultPort=self.DEFAULT_PORT)
statUtility = FtpUtility(host, port)
if not fileInfo.get('fileSize'):
statUtility.statFile(filePath, fileInfo)
diff --git a/src/python/dm/common/utility/ftpUtility.py b/src/python/dm/common/utility/ftpUtility.py
index c82f5607..6987a6c0 100755
--- a/src/python/dm/common/utility/ftpUtility.py
+++ b/src/python/dm/common/utility/ftpUtility.py
@@ -167,11 +167,12 @@ class FtpUtility:
if __name__ == '__main__':
print "Round 1: "
- ftpUtility = FtpUtility('s7dserv', 2811)
+ ftpUtility = FtpUtility('s33dserv', 2811)
#files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
files = ftpUtility.getFiles2('/export/dm/test')
print files
files = ftpUtility.getFiles('/export/dm/test')
+ print ftpUtility.parseFtpUrl('/export/dm/test')
print files
#files = ftpUtility.getFiles('/export/7IDSprayimage/Cummins/Data')
#files = ftpUtility.getFiles2('/export/8-id-i/test', replacementDirPath='/data/testing/8-id-i')
@@ -187,7 +188,7 @@ if __name__ == '__main__':
#print "Round 2: "
#ftpUtility = FtpUtility('s7dserv', 2811)
#files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
- #print ftpUtility.getMd5Sum('/export/8-id-i/test/testfile01')
+ print ftpUtility.getMd5Sum('/export/dm/test/testfile01')
#print ftpUtility.statFile('/export/8-id-i/test/testfile01')
#ftpUtility = FtpUtility('xstor-devel', 22)
#files = ftpUtility.getFiles2('/data/testing')
diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py
index 668bba64..3a4829ca 100755
--- a/src/python/dm/common/utility/sftpUtility.py
+++ b/src/python/dm/common/utility/sftpUtility.py
@@ -6,6 +6,7 @@ import stat
import pysftp
from dm.common.utility.timeUtility import TimeUtility
from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.commandFailed import CommandFailed
import urlparse
class SftpUtility:
@@ -90,8 +91,15 @@ class SftpUtility:
if not self.sftpClient:
self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
try:
- md5Sum = self.sftpClient.execute('md5sum "%s"' % filePath)[0].split()[0]
+ #md5Sum = self.sftpClient.execute('md5sum "%s"' % filePath)[0].split()[0]
+ output = self.sftpClient.execute('md5sum "%s"' % filePath)[0].strip()
+ if not output.endswith(filePath):
+ raise CommandFailed(output)
+ md5Sum = output.split()[0]
fileInfo['md5Sum'] = md5Sum
+ except CommandFailed, ex:
+ self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex))
+ raise
except Exception, ex:
self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex))
self.closeConnection()
@@ -137,9 +145,10 @@ class SftpUtility:
if __name__ == '__main__':
#sftpUtility = SftpUtility('s1dserv', username='dmadmin', password='theKey12')
sftpUtility = SftpUtility('s1dserv',privateKey='/home/beams/DMADMIN/.ssh/id_dsa')
- files = sftpUtility.getFiles('/export/dm/test')
- print files
- print sftpUtility.getMd5Sum('/export/dm/test/testfile01')
- print 'Closing connection'
- sftpUtility.closeConnection()
- print sftpUtility.statFile('/export/dm/test/testfile01')
+ #files = sftpUtility.getFiles('/export/dm/test')
+ #print files
+ print sftpUtility.getMd5Sum('/export/dm/test/testfile03(2nd).txt')
+ print sftpUtility.getMd5Sum('/export/dm/test/testfile 04.txt')
+ #print 'Closing connection'
+ #sftpUtility.closeConnection()
+ #print sftpUtility.statFile('/export/dm/test/testfile01')
diff --git a/src/python/dm/daq_web_service/service/experimentSessionController.py b/src/python/dm/daq_web_service/service/experimentSessionController.py
index 27fce7d5..c0697e2a 100755
--- a/src/python/dm/daq_web_service/service/experimentSessionController.py
+++ b/src/python/dm/daq_web_service/service/experimentSessionController.py
@@ -84,7 +84,7 @@ class ExperimentSessionController(DmSessionController):
raise InvalidRequest('Missing data directory.')
dataDirectory = Encoder.decode(dataDirectory)
if not dataDirectory.startswith('/') and not dataDirectory.count('://'):
- raise InvalidRequest('Data directory must be an absolute path.')
+ raise InvalidRequest('Data directory must be an absolute path: %s' % dataDirectory)
daqInfo = {}
encodedDaqInfo = kwargs.get('daqInfo')
--
GitLab