From 2f2e472cdbe4cbea48af3e148d63b9f2caab4faf Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Fri, 27 May 2016 20:48:55 +0000
Subject: [PATCH] corrected daq issues with sftp
---
src/python/dm/__init__.py | 2 +-
.../processing/plugins/gridftpFileTransferPlugin.py | 10 ++++++----
.../processing/plugins/mongoDbFileCatalogPlugin.py | 8 ++++++++
src/python/dm/common/utility/sftpUtility.py | 10 +++++++++-
4 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/src/python/dm/__init__.py b/src/python/dm/__init__.py
index 16317eac..f024d30e 100644
--- a/src/python/dm/__init__.py
+++ b/src/python/dm/__init__.py
@@ -1 +1 @@
-__version__ = "0.11 (2016.04.28)"
+__version__ = "0.13 (2016.05.27)"
diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
index 9b5aa797..7d3981be 100755
--- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
@@ -100,12 +100,14 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
srcUrl = self.getSrcUrl(filePath, dataDirectory)
# Calculate checksum
- (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
- ftpUtility = FtpUtility(host, port)
+ statUtility = self.statUtility
+ if not statUtility:
+ (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT)
+ statUtility = FtpUtility(host, port)
if not fileInfo.get('fileSize'):
- ftpUtility.statFile(filePath, fileInfo)
+ statUtility.statFile(filePath, fileInfo)
if self.localMd5Sum:
- ftpUtility.getMd5Sum(filePath, fileInfo)
+ statUtility.getMd5Sum(filePath, fileInfo)
# Transfer file
self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl))
diff --git a/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py
index 92afba0b..f6b61069 100755
--- a/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py
+++ b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py
@@ -20,6 +20,13 @@ class MongoDbFileCatalogPlugin(FileProcessor):
self.md5Sum = md5Sum
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+ def processStat(self, filePath, fileInfo={}):
+ if not self.statUtility:
+ return fileInfo
+ if not fileInfo.has_key('fileSize'):
+ self.statUtility.statFile(filePath)
+ return fileInfo
+
def processMd5Sum(self, filePath, fileInfo={}):
if not self.md5Sum or not self.statUtility:
return fileInfo
@@ -93,6 +100,7 @@ class MongoDbFileCatalogPlugin(FileProcessor):
if fileInfo2.has_key(key):
del fileInfo2[key]
+ self.processStat(filePath, fileInfo2)
self.processHdf5Metadata(filePath, fileInfo2)
self.processMd5Sum(filePath, fileInfo2)
self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileInfo2)))
diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py
index 64d9881c..2b4ef868 100755
--- a/src/python/dm/common/utility/sftpUtility.py
+++ b/src/python/dm/common/utility/sftpUtility.py
@@ -69,7 +69,6 @@ class SftpUtility:
filePath = '%s/%s' % (replacementDirPath, fileName)
fileInfo = {'fileSize' : attr.st_size,
'fileModificationTime' : attr.st_mtime }
- fileInfo['fileModificationTimestamp'] = TimeUtility.formatLocalTimestamp(attr.st_mtime)
fileDict[filePath] = fileInfo
return fileDict
@@ -80,6 +79,14 @@ class SftpUtility:
fileInfo['md5Sum'] = md5Sum
return md5Sum
+ def statFile(self, filePath, fileInfo={}):
+ if not self.sftpClient:
+ self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
+ attr = self.sftpClient.stat(filePath)
+ fileInfo['fileSize'] = attr.st_size
+ fileInfo['fileModificationTime'] = attr.st_mtime
+ return fileInfo
+
#######################################################################
# Testing.
@@ -89,3 +96,4 @@ if __name__ == '__main__':
files = sftpUtility.getFiles('/export/dm/test')
print files
print sftpUtility.getMd5Sum('/export/dm/test/testfile01')
+ print sftpUtility.statFile('/export/dm/test/testfile01')
--
GitLab