diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py index acd76c7ba5df5ab4cf6911764fc1cd1dade63132..03c56c2ac45b424f1c37896f204e8a940454cc39 100755 --- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py +++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py @@ -28,6 +28,16 @@ class GridftpFileTransferPlugin(FileTransferPlugin): self.directoryTransferCommand = directoryTransferCommand self.pluginMustProcessFiles = pluginMustProcessFiles + def replaceSpecialCharacters(self, url): + replacementMap = { + '#' : '%23', + ' ' : '%20', + '~' : '%7E', + } + for (original, replacement) in replacementMap.items(): + url = url.replace(original,replacement) + return url + def getSrcUrl(self, filePath, dataDirectory): (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT) if scheme: @@ -36,7 +46,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin): srcUrl = 'file://%s' % filePath else: srcUrl = '%s/%s' % (self.src,filePath) - return srcUrl + return self.replaceSpecialCharacters(srcUrl) def getDestUrl(self, filePath, dataDirectory, storageHost, storageDirectory): (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT) @@ -46,7 +56,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin): destUrl = '%s/%s/%s' % (self.dest, dirName, fileName) else: destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName) - return destUrl + return self.replaceSpecialCharacters(destUrl) def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo): if not self.pluginMustProcessFiles: @@ -105,7 +115,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin): # Calculate checksum statUtility = self.statUtility if not statUtility: - (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT) + (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(srcUrl, defaultPort=self.DEFAULT_PORT) statUtility = FtpUtility(host, port) if not fileInfo.get('fileSize'): statUtility.statFile(filePath, fileInfo) diff --git a/src/python/dm/common/utility/ftpUtility.py b/src/python/dm/common/utility/ftpUtility.py index c82f5607c52ad7bc32cfcb14b86c9322013da147..6987a6c0afb6808f1dc5246cbd510b6932c5001d 100755 --- a/src/python/dm/common/utility/ftpUtility.py +++ b/src/python/dm/common/utility/ftpUtility.py @@ -167,11 +167,12 @@ class FtpUtility: if __name__ == '__main__': print "Round 1: " - ftpUtility = FtpUtility('s7dserv', 2811) + ftpUtility = FtpUtility('s33dserv', 2811) #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data') files = ftpUtility.getFiles2('/export/dm/test') print files files = ftpUtility.getFiles('/export/dm/test') + print ftpUtility.parseFtpUrl('/export/dm/test') print files #files = ftpUtility.getFiles('/export/7IDSprayimage/Cummins/Data') #files = ftpUtility.getFiles2('/export/8-id-i/test', replacementDirPath='/data/testing/8-id-i') @@ -187,7 +188,7 @@ if __name__ == '__main__': #print "Round 2: " #ftpUtility = FtpUtility('s7dserv', 2811) #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data') - #print ftpUtility.getMd5Sum('/export/8-id-i/test/testfile01') + print ftpUtility.getMd5Sum('/export/dm/test/testfile01') #print ftpUtility.statFile('/export/8-id-i/test/testfile01') #ftpUtility = FtpUtility('xstor-devel', 22) #files = ftpUtility.getFiles2('/data/testing') diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py index 668bba641a66793414d2b5c86864b5bb64fa8a73..3a4829cad2745ee5a87ee34b5b45687f37dfed64 100755 --- a/src/python/dm/common/utility/sftpUtility.py +++ b/src/python/dm/common/utility/sftpUtility.py @@ -6,6 +6,7 @@ import stat import pysftp from dm.common.utility.timeUtility import TimeUtility from dm.common.utility.loggingManager import LoggingManager +from dm.common.exceptions.commandFailed import CommandFailed import urlparse class SftpUtility: @@ -90,8 +91,15 @@ class SftpUtility: if not self.sftpClient: self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey) try: - md5Sum = self.sftpClient.execute('md5sum "%s"' % filePath)[0].split()[0] + #md5Sum = self.sftpClient.execute('md5sum "%s"' % filePath)[0].split()[0] + output = self.sftpClient.execute('md5sum "%s"' % filePath)[0].strip() + if not output.endswith(filePath): + raise CommandFailed(output) + md5Sum = output.split()[0] fileInfo['md5Sum'] = md5Sum + except CommandFailed, ex: + self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex)) + raise except Exception, ex: self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex)) self.closeConnection() @@ -137,9 +145,10 @@ class SftpUtility: if __name__ == '__main__': #sftpUtility = SftpUtility('s1dserv', username='dmadmin', password='theKey12') sftpUtility = SftpUtility('s1dserv',privateKey='/home/beams/DMADMIN/.ssh/id_dsa') - files = sftpUtility.getFiles('/export/dm/test') - print files - print sftpUtility.getMd5Sum('/export/dm/test/testfile01') - print 'Closing connection' - sftpUtility.closeConnection() - print sftpUtility.statFile('/export/dm/test/testfile01') + #files = sftpUtility.getFiles('/export/dm/test') + #print files + print sftpUtility.getMd5Sum('/export/dm/test/testfile03(2nd).txt') + print sftpUtility.getMd5Sum('/export/dm/test/testfile 04.txt') + #print 'Closing connection' + #sftpUtility.closeConnection() + #print sftpUtility.statFile('/export/dm/test/testfile01') diff --git a/src/python/dm/daq_web_service/service/experimentSessionController.py b/src/python/dm/daq_web_service/service/experimentSessionController.py index 27fce7d5b36c3dfd87cd7bed7c6f052a44abf8ad..c0697e2a0c87896b7ced6a21a30781cc2ce1ad6c 100755 --- a/src/python/dm/daq_web_service/service/experimentSessionController.py +++ b/src/python/dm/daq_web_service/service/experimentSessionController.py @@ -84,7 +84,7 @@ class ExperimentSessionController(DmSessionController): raise InvalidRequest('Missing data directory.') dataDirectory = Encoder.decode(dataDirectory) if not dataDirectory.startswith('/') and not dataDirectory.count('://'): - raise InvalidRequest('Data directory must be an absolute path.') + raise InvalidRequest('Data directory must be an absolute path: %s' % dataDirectory) daqInfo = {} encodedDaqInfo = kwargs.get('daqInfo')