diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py index f49ab85f5614e406721d421b8e0d56679bbc8dfb..e8066eea63fecfaabce3190e029c9c1b9efec003 100755 --- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py +++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py @@ -48,7 +48,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin): ftpUtility = SftpUtility(storageHost) self.logger.debug("STORAGE DIR: %s", storageDirectory) self.logger.debug("STORAGE HOST: %s", storageHost) - storageFilePathsDict = ftpUtility.getFiles(storageDirectory, {}) + storageFilePathsDict = ftpUtility.getFiles(storageDirectory, {}, dataDIrectory) self.logger.debug("STORAGE PATHS: %s", storageFilePathsDict) self.logger.debug("ORIG PATHS: %s", filePathsDict) pluginFilePathsDict = {} diff --git a/src/python/dm/common/utility/ftpUtility.py b/src/python/dm/common/utility/ftpUtility.py index 8df82f43ed8b51941c7f70afe14159fefcf163ce..42efebd8f4d1dcb1aea4f0e76b4ab3e758e42a88 100755 --- a/src/python/dm/common/utility/ftpUtility.py +++ b/src/python/dm/common/utility/ftpUtility.py @@ -92,21 +92,24 @@ class FtpUtility: del fileStatDict['Modify'] del fileStatDict['Type'] - def getFiles(self, dirPath, fileDict={}): + def getFiles(self, dirPath, fileDict={}, replacementDirPath=None): if not self.ftpClient: self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password) # Need these to be class members for the callback function self.mlsdFileDict = {} self.mlsdDirList = [] self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput) + if not replacementDirPath: + replacementDirPath = dirPath for (fileName,fileInfo) in self.mlsdFileDict.items(): self.__processFileStatDict(fileInfo) del fileInfo['Name'] - filePath = '%s/%s' % (dirPath, fileName) + filePath = '%s/%s' % (replacementDirPath, fileName) fileDict[filePath] = fileInfo for d in copy.copy(self.mlsdDirList): dirPath2 = '%s/%s' % (dirPath,d) - self.getFiles(dirPath2,fileDict) + replacementDirPath2 = '%s/%s' % (replacementDirPath,d) + self.getFiles(dirPath2,fileDict, replacementDirPath2) return fileDict def getMd5Sum(self, filePath, fileInfo={}): @@ -136,5 +139,7 @@ if __name__ == '__main__': ftpUtility = FtpUtility('s8dserv', 2811) files = ftpUtility.getFiles('/export/8-id-i/test') print files + files = ftpUtility.getFiles('/export/8-id-i/test', replacementDirPath='/data/testing/8-id-i') + print files print ftpUtility.getMd5Sum('/export/8-id-i/test/testfile01') print ftpUtility.statFile('/export/8-id-i/test/testfile01') diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py index 46598e57017bac0d1799b86fa851e239b23dd78c..4e69f676f72455ea339f6d79e0ead5a91caa6b0e 100755 --- a/src/python/dm/common/utility/sftpUtility.py +++ b/src/python/dm/common/utility/sftpUtility.py @@ -51,23 +51,26 @@ class SftpUtility: outputDict[key] = value return outputDict - def getFiles(self, dirPath, fileDict={}): + def getFiles(self, dirPath, fileDict={}, replacementDirPath=None): if not self.sftpClient: self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password) - # Need these to be class members for the callback function + if not replacementDirPath: + replacementDirPath = dirPath attrs = self.sftpClient.listdir_attr(dirPath) mode = attrs[0].st_mode for attr in attrs: fileName = attr.filename mode = attr.st_mode - fullPath = '%s/%s' % (dirPath, fileName) if stat.S_ISDIR(mode): - self.getFiles(fullPath, fileDict) + dirPath2 = '%s/%s' % (dirPath, fileName) + replacementDirPath2 = '%s/%s' % (replacementDirPath, fileName) + self.getFiles(dirPath2, fileDict, replacementDirPath2) elif stat.S_ISREG(mode): - fileInfo = {'filePath' : fullPath, 'fileSize' : attr.st_size, + filePath = '%s/%s' % (replacementDirPath, fileName) + fileInfo = {'fileSize' : attr.st_size, 'fileModificationTime' : attr.st_mtime } fileInfo['fileModificationTimeStamp'] = TimeUtility.formatLocalTimeStamp(attr.st_mtime) - fileDict[fullPath] = fileInfo + fileDict[filePath] = fileInfo return fileDict ####################################################################### @@ -77,3 +80,5 @@ if __name__ == '__main__': sftpUtility = SftpUtility('xstor-devel', username='dmadmin') files = sftpUtility.getFiles('/data/testing/test1') print files + files = sftpUtility.getFiles('/data/testing/test1', replacementDirPath='/xyz/ccc') + print files