diff --git a/src/python/dm/__init__.py b/src/python/dm/__init__.py index 16317eac2df340749b47ecd487d7739795e08c6b..f024d30ee0540f374a9d103799c897379f732b69 100644 --- a/src/python/dm/__init__.py +++ b/src/python/dm/__init__.py @@ -1 +1 @@ -__version__ = "0.11 (2016.04.28)" +__version__ = "0.13 (2016.05.27)" diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py index 9b5aa797430734a4f1d48c6b6db166fb6cd1ab2c..7d3981bed5fdb805129b93e4e2329f883d5f0b7f 100755 --- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py +++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py @@ -100,12 +100,14 @@ class GridftpFileTransferPlugin(FileTransferPlugin): srcUrl = self.getSrcUrl(filePath, dataDirectory) # Calculate checksum - (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT) - ftpUtility = FtpUtility(host, port) + statUtility = self.statUtility + if not statUtility: + (scheme, host, port, dirPath) = FtpUtility.parseFtpUrl(dataDirectory, defaultPort=self.DEFAULT_PORT) + statUtility = FtpUtility(host, port) if not fileInfo.get('fileSize'): - ftpUtility.statFile(filePath, fileInfo) + statUtility.statFile(filePath, fileInfo) if self.localMd5Sum: - ftpUtility.getMd5Sum(filePath, fileInfo) + statUtility.getMd5Sum(filePath, fileInfo) # Transfer file self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl)) diff --git a/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py index 92afba0ba3a025504cfe84ce6d739ecbfeadbddd..f6b6106973d6dbd0820c3e6c6306993f24b69fdf 100755 --- a/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py +++ b/src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py @@ -20,6 +20,13 @@ class MongoDbFileCatalogPlugin(FileProcessor): self.md5Sum = md5Sum self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__) + def processStat(self, filePath, fileInfo={}): + if not self.statUtility: + return fileInfo + if not fileInfo.has_key('fileSize'): + self.statUtility.statFile(filePath) + return fileInfo + def processMd5Sum(self, filePath, fileInfo={}): if not self.md5Sum or not self.statUtility: return fileInfo @@ -93,6 +100,7 @@ class MongoDbFileCatalogPlugin(FileProcessor): if fileInfo2.has_key(key): del fileInfo2[key] + self.processStat(filePath, fileInfo2) self.processHdf5Metadata(filePath, fileInfo2) self.processMd5Sum(filePath, fileInfo2) self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileInfo2))) diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py index 64d9881cba55e874241e74e9cd41a40cf96d8678..2b4ef86816126c0aec3bc27c180dd981ae6599ea 100755 --- a/src/python/dm/common/utility/sftpUtility.py +++ b/src/python/dm/common/utility/sftpUtility.py @@ -69,7 +69,6 @@ class SftpUtility: filePath = '%s/%s' % (replacementDirPath, fileName) fileInfo = {'fileSize' : attr.st_size, 'fileModificationTime' : attr.st_mtime } - fileInfo['fileModificationTimestamp'] = TimeUtility.formatLocalTimestamp(attr.st_mtime) fileDict[filePath] = fileInfo return fileDict @@ -80,6 +79,14 @@ class SftpUtility: fileInfo['md5Sum'] = md5Sum return md5Sum + def statFile(self, filePath, fileInfo={}): + if not self.sftpClient: + self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey) + attr = self.sftpClient.stat(filePath) + fileInfo['fileSize'] = attr.st_size + fileInfo['fileModificationTime'] = attr.st_mtime + return fileInfo + ####################################################################### # Testing. @@ -89,3 +96,4 @@ if __name__ == '__main__': files = sftpUtility.getFiles('/export/dm/test') print files print sftpUtility.getMd5Sum('/export/dm/test/testfile01') + print sftpUtility.statFile('/export/dm/test/testfile01')