corrected daq issues with sftp

src/python/dm/__init__.py

-__version__ = "0.11 (2016.04.28)"
+__version__ = "0.13 (2016.05.27)"

src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py

@@ -100,12 +100,14 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
         srcUrl = self.getSrcUrl(filePath, dataDirectory)
 
         # Calculate checksum
-        (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
-        ftpUtility = FtpUtility(host, port)
+        statUtility = self.statUtility
+        if not statUtility:
+            (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
+            statUtility = FtpUtility(host, port)
         if not fileInfo.get('fileSize'):
-            ftpUtility.statFile(filePath, fileInfo)
+            statUtility.statFile(filePath, fileInfo)
         if self.localMd5Sum:
-            ftpUtility.getMd5Sum(filePath, fileInfo)
+            statUtility.getMd5Sum(filePath, fileInfo)
 
         # Transfer file
         self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl))

src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py

@@ -20,6 +20,13 @@ class MongoDbFileCatalogPlugin(FileProcessor):
         self.md5Sum = md5Sum
         self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
 
+    def processStat(self, filePath, fileInfo={}):
+        if not self.statUtility:
+            return fileInfo
+        if not fileInfo.has_key('fileSize'):
+            self.statUtility.statFile(filePath)
+        return fileInfo
+
     def processMd5Sum(self, filePath, fileInfo={}):
         if not self.md5Sum or not self.statUtility:
             return fileInfo
@@ -93,6 +100,7 @@ class MongoDbFileCatalogPlugin(FileProcessor):
             if fileInfo2.has_key(key):
                 del fileInfo2[key]
 
+        self.processStat(filePath, fileInfo2)
         self.processHdf5Metadata(filePath, fileInfo2)
         self.processMd5Sum(filePath, fileInfo2)
         self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileInfo2)))

src/python/dm/common/utility/sftpUtility.py

@@ -69,7 +69,6 @@ class SftpUtility:
                 filePath = '%s/%s' % (replacementDirPath, fileName)
                 fileInfo = {'fileSize' : attr.st_size, 
                     'fileModificationTime' : attr.st_mtime }
-                fileInfo['fileModificationTimestamp'] = TimeUtility.formatLocalTimestamp(attr.st_mtime)
                 fileDict[filePath] = fileInfo
         return fileDict
 
@@ -80,6 +79,14 @@ class SftpUtility:
         fileInfo['md5Sum'] = md5Sum
         return md5Sum
 
+    def statFile(self, filePath, fileInfo={}):
+        if not self.sftpClient:
+            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
+        attr = self.sftpClient.stat(filePath)
+        fileInfo['fileSize'] = attr.st_size 
+        fileInfo['fileModificationTime'] = attr.st_mtime
+        return fileInfo
+
 #######################################################################
 # Testing.
 
@@ -89,3 +96,4 @@ if __name__ == '__main__':
     files = sftpUtility.getFiles('/export/dm/test')
     print files
     print sftpUtility.getMd5Sum('/export/dm/test/testfile01')
+    print sftpUtility.statFile('/export/dm/test/testfile01')