merge from 0.13

src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py

@@ -28,6 +28,16 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
         self.directoryTransferCommand = directoryTransferCommand 
         self.pluginMustProcessFiles = pluginMustProcessFiles 
 
+    def replaceSpecialCharacters(self, url):
+        replacementMap = {
+	    '#' : '%23',
+	    ' ' : '%20',
+	    '~' : '%7E',
+	}
+	for (original, replacement) in replacementMap.items():
+	    url = url.replace(original,replacement)
+	return url
+
     def getSrcUrl(self, filePath, dataDirectory):
         (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
         if scheme:
@@ -36,7 +46,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
             srcUrl = 'file://%s' % filePath
         else:
             srcUrl = '%s/%s' % (self.src,filePath)
-        return srcUrl
+        return self.replaceSpecialCharacters(srcUrl)
     
     def getDestUrl(self, filePath, dataDirectory, storageHost, storageDirectory):
         (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
@@ -46,7 +56,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
             destUrl = '%s/%s/%s' % (self.dest, dirName, fileName)
         else:
             destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName)
-        return destUrl
+        return self.replaceSpecialCharacters(destUrl)
 
     def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
         if not self.pluginMustProcessFiles:
@@ -105,7 +115,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
         # Calculate checksum
         statUtility = self.statUtility
         if not statUtility:
-            (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
+            (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(srcUrl, defaultPort=self.DEFAULT_PORT)
             statUtility = FtpUtility(host, port)
         if not fileInfo.get('fileSize'):
             statUtility.statFile(filePath, fileInfo)

src/python/dm/common/utility/ftpUtility.py

@@ -167,11 +167,12 @@ class FtpUtility:
 
 if __name__ == '__main__':
     print "Round 1: "
-    ftpUtility = FtpUtility('s7dserv', 2811)
+    ftpUtility = FtpUtility('s33dserv', 2811)
     #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
     files = ftpUtility.getFiles2('/export/dm/test')
     print files
     files = ftpUtility.getFiles('/export/dm/test')
+    print ftpUtility.parseFtpUrl('/export/dm/test')
     print files
     #files = ftpUtility.getFiles('/export/7IDSprayimage/Cummins/Data')
     #files = ftpUtility.getFiles2('/export/8-id-i/test', replacementDirPath='/data/testing/8-id-i')
@@ -187,7 +188,7 @@ if __name__ == '__main__':
     #print "Round 2: "
     #ftpUtility = FtpUtility('s7dserv', 2811)
     #files = ftpUtility.getFiles2('/export/7IDSprayimage/Cummins/Data')
-    #print ftpUtility.getMd5Sum('/export/8-id-i/test/testfile01')
+    print ftpUtility.getMd5Sum('/export/dm/test/testfile01')
     #print ftpUtility.statFile('/export/8-id-i/test/testfile01')
     #ftpUtility = FtpUtility('xstor-devel', 22)
     #files = ftpUtility.getFiles2('/data/testing')

src/python/dm/common/utility/sftpUtility.py

@@ -6,6 +6,7 @@ import stat
 import pysftp
 from dm.common.utility.timeUtility import TimeUtility
 from dm.common.utility.loggingManager import LoggingManager
+from dm.common.exceptions.commandFailed import CommandFailed
 import urlparse
 
 class SftpUtility:
@@ -90,8 +91,15 @@ class SftpUtility:
             if not self.sftpClient:
                 self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
             try:
-                md5Sum = self.sftpClient.execute('md5sum "%s"' % filePath)[0].split()[0]
+                #md5Sum = self.sftpClient.execute('md5sum "%s"' % filePath)[0].split()[0]
+                output = self.sftpClient.execute('md5sum "%s"' % filePath)[0].strip()
+                if not output.endswith(filePath):
+                    raise CommandFailed(output)
+                md5Sum = output.split()[0]
                 fileInfo['md5Sum'] = md5Sum
+            except CommandFailed, ex:
+                self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex))
+                raise
             except Exception, ex:
                 self.getLogger().error('Could not get md5sum for file %s: %s' % (filePath,ex))
                 self.closeConnection()
@@ -137,9 +145,10 @@ class SftpUtility:
 if __name__ == '__main__':
     #sftpUtility = SftpUtility('s1dserv', username='dmadmin', password='theKey12')
     sftpUtility = SftpUtility('s1dserv',privateKey='/home/beams/DMADMIN/.ssh/id_dsa')
-    files = sftpUtility.getFiles('/export/dm/test')
-    print files
-    print sftpUtility.getMd5Sum('/export/dm/test/testfile01')
-    print 'Closing connection'
-    sftpUtility.closeConnection()
-    print sftpUtility.statFile('/export/dm/test/testfile01')
+    #files = sftpUtility.getFiles('/export/dm/test')
+    #print files
+    print sftpUtility.getMd5Sum('/export/dm/test/testfile03(2nd).txt')
+    print sftpUtility.getMd5Sum('/export/dm/test/testfile 04.txt')
+    #print 'Closing connection'
+    #sftpUtility.closeConnection()
+    #print sftpUtility.statFile('/export/dm/test/testfile01')

src/python/dm/daq_web_service/service/experimentSessionController.py

@@ -84,7 +84,7 @@ class ExperimentSessionController(DmSessionController):
             raise InvalidRequest('Missing data directory.')
         dataDirectory = Encoder.decode(dataDirectory)
         if not dataDirectory.startswith('/') and not dataDirectory.count('://'):
-            raise InvalidRequest('Data directory must be an absolute path.')
+            raise InvalidRequest('Data directory must be an absolute path: %s' % dataDirectory)
 
         daqInfo = {}
         encodedDaqInfo = kwargs.get('daqInfo')