optimized detection of existing files

src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py

 #!/usr/bin/env python
 
 import os
+import copy
 from fileTransferPlugin import FileTransferPlugin
 from dm.common.utility.fileUtility import FileUtility
 from dm.common.utility.ftpUtility import FtpUtility
@@ -48,27 +49,31 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
         (scheme, host, port, replacementDirPath) = FtpUtility.parseFtpUrl(dataDirectory)
         ftpUtility = SftpUtility(storageHost)
         storageFilePathsDict = ftpUtility.getFiles(storageDirectory, {}, replacementDirPath)
-        pluginFilePathsDict = {}
-        filePaths = filePathsDict.keys()
-        for filePath in filePaths:
+        pluginFilePathsDict = copy.copy(filePathsDict)
+        # Remove file from plugin dict if we do not need to transfer it
+        for (filePath,storageFilePathDict) in storageFilePathsDict.items():
             filePathDict = filePathsDict.get(filePath)
-            storageFilePathDict = storageFilePathsDict.get(filePath)
-
-            if not storageFilePathDict:
-                # remote directory does not have the file
-                pluginFilePathsDict[filePath] = filePathDict
-            else:
-                fSize = filePathDict.get('fileSize') 
-                sfSize = storageFilePathDict.get('fileSize') 
-                # check size
-                if not fSize or not sfSize or fSize != sfSize:
-                    pluginFilePathsDict[filePath] = filePathDict
-                else:
-                    # sizes are the same, check modify time
-                    mTime = filePathDict.get('fileModificationTime') 
-                    smTime = storageFilePathDict.get('fileModificationTime') 
-                    if not mTime or not smTime or mTime > smTime:
-                        pluginFilePathsDict[filePath] = filePathDict
+            if filePathDict is None:
+                # We are not attempting to transfer this file
+                # No need to change plugin file dict
+                continue
+
+            # Check size
+            fSize = filePathDict.get('fileSize') 
+            sfSize = storageFilePathDict.get('fileSize') 
+            if not fSize or not sfSize or fSize != sfSize:
+                # Sizes differ, need to transfer file
+                continue
+
+            # Sizes are the same, check modify time
+            mTime = filePathDict.get('fileModificationTime') 
+            smTime = storageFilePathDict.get('fileModificationTime') 
+            if not mTime or not smTime or mTime > smTime:
+                # Source time is later than storage time, need to transfer file
+                continue
+
+            # No need to transfer file
+            del pluginFilePathsDict[filePath]
 
         self.logger.debug('Number of original files: %s, number of plugin files: %s', len(filePathsDict), len(pluginFilePathsDict))
         return pluginFilePathsDict

src/python/dm/common/utility/ftpUtility.py

@@ -143,3 +143,6 @@ if __name__ == '__main__':
     print files
     print ftpUtility.getMd5Sum('/export/8-id-i/test/testfile01')
     print ftpUtility.statFile('/export/8-id-i/test/testfile01')
+    #ftpUtility = FtpUtility('xstor-devel', 22)
+    #files = ftpUtility.getFiles('/data/testing')
+    #print files