added replacement of storage directory with data directory for the purpose of...

added replacement of storage directory with data directory for the purpose of comparing original and storage file directory

src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py

@@ -48,7 +48,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
         ftpUtility = SftpUtility(storageHost)
         self.logger.debug("STORAGE DIR: %s", storageDirectory)
         self.logger.debug("STORAGE HOST: %s", storageHost)
-        storageFilePathsDict = ftpUtility.getFiles(storageDirectory, {})
+        storageFilePathsDict = ftpUtility.getFiles(storageDirectory, {}, dataDIrectory)
         self.logger.debug("STORAGE PATHS: %s", storageFilePathsDict)
         self.logger.debug("ORIG PATHS: %s", filePathsDict)
         pluginFilePathsDict = {}

src/python/dm/common/utility/ftpUtility.py

@@ -92,21 +92,24 @@ class FtpUtility:
         del fileStatDict['Modify']
         del fileStatDict['Type']
 
-    def getFiles(self, dirPath, fileDict={}):
+    def getFiles(self, dirPath, fileDict={}, replacementDirPath=None):
         if not self.ftpClient:
             self.ftpClient = self.getFtpClient(self.host, self.port, self.username, self.password)
         # Need these to be class members for the callback function
         self.mlsdFileDict = {}
         self.mlsdDirList = []
         self.ftpClient.retrlines('MLSD %s' % dirPath, self.__parseMlsdOutput)
+        if not replacementDirPath:
+            replacementDirPath = dirPath
         for (fileName,fileInfo) in self.mlsdFileDict.items():
             self.__processFileStatDict(fileInfo)
             del fileInfo['Name']
-            filePath = '%s/%s' % (dirPath, fileName)
+            filePath = '%s/%s' % (replacementDirPath, fileName)
             fileDict[filePath] = fileInfo
         for d in copy.copy(self.mlsdDirList):
             dirPath2 = '%s/%s' % (dirPath,d)
-            self.getFiles(dirPath2,fileDict)
+            replacementDirPath2 = '%s/%s' % (replacementDirPath,d)
+            self.getFiles(dirPath2,fileDict, replacementDirPath2)
         return fileDict
 
     def getMd5Sum(self, filePath, fileInfo={}):
@@ -136,5 +139,7 @@ if __name__ == '__main__':
     ftpUtility = FtpUtility('s8dserv', 2811)
     files = ftpUtility.getFiles('/export/8-id-i/test')
     print files
+    files = ftpUtility.getFiles('/export/8-id-i/test', replacementDirPath='/data/testing/8-id-i')
+    print files
     print ftpUtility.getMd5Sum('/export/8-id-i/test/testfile01')
     print ftpUtility.statFile('/export/8-id-i/test/testfile01')

src/python/dm/common/utility/sftpUtility.py

@@ -51,23 +51,26 @@ class SftpUtility:
         outputDict[key] = value
         return outputDict
         
-    def getFiles(self, dirPath, fileDict={}):
+    def getFiles(self, dirPath, fileDict={}, replacementDirPath=None):
         if not self.sftpClient:
             self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password)
-        # Need these to be class members for the callback function
+        if not replacementDirPath:
+            replacementDirPath = dirPath
         attrs = self.sftpClient.listdir_attr(dirPath)
         mode = attrs[0].st_mode
         for attr in attrs:
             fileName = attr.filename
             mode = attr.st_mode
-            fullPath = '%s/%s' % (dirPath, fileName)
             if stat.S_ISDIR(mode):
-                self.getFiles(fullPath, fileDict)
+                dirPath2 = '%s/%s' % (dirPath, fileName)
+                replacementDirPath2 = '%s/%s' % (replacementDirPath, fileName)
+                self.getFiles(dirPath2, fileDict, replacementDirPath2)
             elif stat.S_ISREG(mode):
-                fileInfo = {'filePath' : fullPath, 'fileSize' : attr.st_size, 
+                filePath = '%s/%s' % (replacementDirPath, fileName)
+                fileInfo = {'fileSize' : attr.st_size, 
                     'fileModificationTime' : attr.st_mtime }
                 fileInfo['fileModificationTimeStamp'] = TimeUtility.formatLocalTimeStamp(attr.st_mtime)
-                fileDict[fullPath] = fileInfo
+                fileDict[filePath] = fileInfo
         return fileDict
 
 #######################################################################
@@ -77,3 +80,5 @@ if __name__ == '__main__':
     sftpUtility = SftpUtility('xstor-devel', username='dmadmin')
     files = sftpUtility.getFiles('/data/testing/test1')
     print files
+    files = sftpUtility.getFiles('/data/testing/test1', replacementDirPath='/xyz/ccc')
+    print files