latest set of fixes for ftp

sbin/dm_install_support.sh

@@ -31,7 +31,7 @@ execute() {
 if [ ! -d $DM_SUPPORT_DIR ]; then
     echo "Creating new DM support directory $DM_SUPPORT_DIR."
     cd `dirname $DM_SUPPORT_DIR`
-    execute svn export $DM_SVN_URL/support support
+    execute svn co $DM_SVN_URL/support 
 fi
 cd $DM_SUPPORT_DIR
 execute svn update

src/python/dm/common/utility/ftpUtility.py

@@ -78,19 +78,19 @@ class FtpUtility:
         name = parts[-1].strip()
         self.mlsdFileStatDict[name] = parseDict
 
-    def getFiles(self, dir, fileDict={}):
+    def getFiles(self, dirPath, fileDict={}):
         if not self.ftpClient:
             self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
         # Need these to be class members for the callback function
         self.mlsdFileDict = {}
         self.mlsdDirList = []
-        self.ftpClient.retrlines('MLSD %s' % dir, self.__parseMlsdOutput)
+        self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput)
         for (fileName,fileInfo) in self.mlsdFileDict.items():
-            filePath = '%s/%s' % (dir, fileName)
+            filePath = '%s/%s' % (dirPath, fileName)
             fileDict[filePath] = fileInfo
         for d in copy.copy(self.mlsdDirList):
-            dirPath = '%s/%s' % (dir,d)
-            self.getFiles(dirPath,fileDict)
+            dirPath2 = '%s/%s' % (dirPath,d)
+            self.getFiles(dirPath2,fileDict)
         return fileDict
 
     def getMd5Sum(self, filePath, fileInfo={}):

src/python/dm/common/utility/sftpUtility.py

+#!/usr/bin/env python
+
+import copy
+import stat
+import pysftp
+from dm.common.utility.loggingManager import LoggingManager
+import urlparse
+
+class SftpUtility:
+
+    DEFAULT_PORT = 22
+
+    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None):
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.sftpClient = None
+
+    @classmethod
+    def parseFtpUrl(cls, url, defaultHost=None, defaultPort=None):
+        host = defaultHost
+        port = defaultPort
+        scheme = None
+        dirPath = url
+        if url.startswith('ftp://'):
+            parseResult = urlparse.urlparse(url)
+            scheme = parseResult.scheme
+            netlocTokens = parseResult.netloc.split(':')
+            host = netlocTokens[0]
+            if len(netlocTokens) > 1:
+                port = int(netlocTokens[1])
+            dirPath = parseResult.path
+        return (scheme, host, port, dirPath)
+
+    @classmethod
+    def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None):
+        sftp = pysftp.Connection(host, username=username, password=password, port=port)
+        return sftp
+
+    @classmethod
+    def getLogger(cls):
+        logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return logger
+
+
+    def __parseKeyValue(cls, keyValue, outputDict={}):
+        key,value = keyValue.split('=')
+        value = value.strip()
+        outputDict[key] = value
+        return outputDict
+        
+    def getFiles(self, dirPath, fileDict={}):
+        if not self.sftpClient:
+            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password)
+        # Need these to be class members for the callback function
+        attrs = self.sftpClient.listdir_attr(dirPath)
+        mode = attrs[0].st_mode
+        for attr in attrs:
+            fileName = attr.filename
+            mode = attr.st_mode
+            fullPath = '%s/%s' % (dirPath, fileName)
+            if stat.S_ISDIR(mode):
+                self.getFiles(fullPath, fileDict)
+            elif stat.S_ISREG(mode):
+                fileInfo = {'filePath' : fullPath, 'fileSize' : attr.st_size, }
+                fileDict[fullPath] = fileInfo
+        return fileDict
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    sftpUtility = SftpUtility('dmstorage', username='dm')
+    files = sftpUtility.getFiles('/opt/DM/data/ESAF/e1')
+    print files