diff --git a/sbin/dm_install_support.sh b/sbin/dm_install_support.sh index 0b9c14cf717bf4b79776826b169b3307d133ea3a..1a6730116a2b45c770aafa1cecb7ee2b5c3fe11b 100755 --- a/sbin/dm_install_support.sh +++ b/sbin/dm_install_support.sh @@ -31,7 +31,7 @@ execute() { if [ ! -d $DM_SUPPORT_DIR ]; then echo "Creating new DM support directory $DM_SUPPORT_DIR." cd `dirname $DM_SUPPORT_DIR` - execute svn export $DM_SVN_URL/support support + execute svn co $DM_SVN_URL/support fi cd $DM_SUPPORT_DIR execute svn update diff --git a/src/python/dm/common/utility/ftpUtility.py b/src/python/dm/common/utility/ftpUtility.py index 0b3ad3d4a361fd86c54a2439f2056437d56fd970..a6a1a130b8f3ed7a8a5d0ce0351d4c7845c0e3ae 100755 --- a/src/python/dm/common/utility/ftpUtility.py +++ b/src/python/dm/common/utility/ftpUtility.py @@ -78,19 +78,19 @@ class FtpUtility: name = parts[-1].strip() self.mlsdFileStatDict[name] = parseDict - def getFiles(self, dir, fileDict={}): + def getFiles(self, dirPath, fileDict={}): if not self.ftpClient: self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password) # Need these to be class members for the callback function self.mlsdFileDict = {} self.mlsdDirList = [] - self.ftpClient.retrlines('MLSD %s' % dir, self.__parseMlsdOutput) + self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput) for (fileName,fileInfo) in self.mlsdFileDict.items(): - filePath = '%s/%s' % (dir, fileName) + filePath = '%s/%s' % (dirPath, fileName) fileDict[filePath] = fileInfo for d in copy.copy(self.mlsdDirList): - dirPath = '%s/%s' % (dir,d) - self.getFiles(dirPath,fileDict) + dirPath2 = '%s/%s' % (dirPath,d) + self.getFiles(dirPath2,fileDict) return fileDict def getMd5Sum(self, filePath, fileInfo={}): diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py new file mode 100755 index 0000000000000000000000000000000000000000..db860c2abc61ed523185a7121dd846397eccb0d5 --- /dev/null +++ b/src/python/dm/common/utility/sftpUtility.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import copy +import stat +import pysftp +from dm.common.utility.loggingManager import LoggingManager +import urlparse + +class SftpUtility: + + DEFAULT_PORT = 22 + + def __init__(self, host, port=DEFAULT_PORT, username=None, password=None): + self.host = host + self.port = port + self.username = username + self.password = password + self.sftpClient = None + + @classmethod + def parseFtpUrl(cls, url, defaultHost=None, defaultPort=None): + host = defaultHost + port = defaultPort + scheme = None + dirPath = url + if url.startswith('ftp://'): + parseResult = urlparse.urlparse(url) + scheme = parseResult.scheme + netlocTokens = parseResult.netloc.split(':') + host = netlocTokens[0] + if len(netlocTokens) > 1: + port = int(netlocTokens[1]) + dirPath = parseResult.path + return (scheme, host, port, dirPath) + + @classmethod + def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None): + sftp = pysftp.Connection(host, username=username, password=password, port=port) + return sftp + + @classmethod + def getLogger(cls): + logger = LoggingManager.getInstance().getLogger(cls.__name__) + return logger + + + def __parseKeyValue(cls, keyValue, outputDict={}): + key,value = keyValue.split('=') + value = value.strip() + outputDict[key] = value + return outputDict + + def getFiles(self, dirPath, fileDict={}): + if not self.sftpClient: + self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password) + # Need these to be class members for the callback function + attrs = self.sftpClient.listdir_attr(dirPath) + mode = attrs[0].st_mode + for attr in attrs: + fileName = attr.filename + mode = attr.st_mode + fullPath = '%s/%s' % (dirPath, fileName) + if stat.S_ISDIR(mode): + self.getFiles(fullPath, fileDict) + elif stat.S_ISREG(mode): + fileInfo = {'filePath' : fullPath, 'fileSize' : attr.st_size, } + fileDict[fullPath] = fileInfo + return fileDict + +####################################################################### +# Testing. + +if __name__ == '__main__': + sftpUtility = SftpUtility('dmstorage', username='dm') + files = sftpUtility.getFiles('/opt/DM/data/ESAF/e1') + print files