diff --git a/src/python/dm/common/utility/ftpUtility.py b/src/python/dm/common/utility/ftpUtility.py index 8cd2be4b7db63e06ed1c4163b4f18f0d7d2ed9d5..8a5540ea63e41e7aa1abb0dc29b798bf927219c3 100755 --- a/src/python/dm/common/utility/ftpUtility.py +++ b/src/python/dm/common/utility/ftpUtility.py @@ -64,19 +64,19 @@ class FtpUtility: elif type == 'file': self.mlsdFileDict[name] = parseDict - def getFiles(self, dir, fileDict={}): + def getFiles(self, dirPath, fileDict={}): if not self.ftpClient: self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password) # Need these to be class members for the callback function self.mlsdFileDict = {} self.mlsdDirList = [] - self.ftpClient.retrlines('MLSD %s' % dir, self.__parseMlsdOutput) + self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput) for (fileName,fileInfo) in self.mlsdFileDict.items(): - filePath = '%s/%s' % (dir, fileName) + filePath = '%s/%s' % (dirPath, fileName) fileDict[filePath] = fileInfo for d in copy.copy(self.mlsdDirList): - dirPath = '%s/%s' % (dir,d) - self.getFiles(dirPath,fileDict) + dirPath2 = '%s/%s' % (dirPath,d) + self.getFiles(dirPath2,fileDict) return fileDict ####################################################################### diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py new file mode 100755 index 0000000000000000000000000000000000000000..db860c2abc61ed523185a7121dd846397eccb0d5 --- /dev/null +++ b/src/python/dm/common/utility/sftpUtility.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import copy +import stat +import pysftp +from dm.common.utility.loggingManager import LoggingManager +import urlparse + +class SftpUtility: + + DEFAULT_PORT = 22 + + def __init__(self, host, port=DEFAULT_PORT, username=None, password=None): + self.host = host + self.port = port + self.username = username + self.password = password + self.sftpClient = None + + @classmethod + def parseFtpUrl(cls, url, defaultHost=None, defaultPort=None): + host = defaultHost + port = defaultPort + scheme = None + dirPath = url + if url.startswith('ftp://'): + parseResult = urlparse.urlparse(url) + scheme = parseResult.scheme + netlocTokens = parseResult.netloc.split(':') + host = netlocTokens[0] + if len(netlocTokens) > 1: + port = int(netlocTokens[1]) + dirPath = parseResult.path + return (scheme, host, port, dirPath) + + @classmethod + def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None): + sftp = pysftp.Connection(host, username=username, password=password, port=port) + return sftp + + @classmethod + def getLogger(cls): + logger = LoggingManager.getInstance().getLogger(cls.__name__) + return logger + + + def __parseKeyValue(cls, keyValue, outputDict={}): + key,value = keyValue.split('=') + value = value.strip() + outputDict[key] = value + return outputDict + + def getFiles(self, dirPath, fileDict={}): + if not self.sftpClient: + self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password) + # Need these to be class members for the callback function + attrs = self.sftpClient.listdir_attr(dirPath) + mode = attrs[0].st_mode + for attr in attrs: + fileName = attr.filename + mode = attr.st_mode + fullPath = '%s/%s' % (dirPath, fileName) + if stat.S_ISDIR(mode): + self.getFiles(fullPath, fileDict) + elif stat.S_ISREG(mode): + fileInfo = {'filePath' : fullPath, 'fileSize' : attr.st_size, } + fileDict[fullPath] = fileInfo + return fileDict + +####################################################################### +# Testing. + +if __name__ == '__main__': + sftpUtility = SftpUtility('dmstorage', username='dm') + files = sftpUtility.getFiles('/opt/DM/data/ESAF/e1') + print files