changes to sftp utility to allow calculating md5 sum, as well as to be able to...

changes to sftp utility to allow calculating md5 sum, as well as to be able to use ssh key for authorization

src/python/dm/common/utility/sftpUtility.py

@@ -11,11 +11,12 @@ class SftpUtility:
 
     DEFAULT_PORT = 22
 
-    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None):
+    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None):
         self.host = host
         self.port = port
         self.username = username
         self.password = password
+        self.privateKey = privateKey
         self.sftpClient = None
 
     @classmethod
@@ -35,8 +36,8 @@ class SftpUtility:
         return (scheme, host, port, dirPath)
 
     @classmethod
-    def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None):
-        sftp = pysftp.Connection(host, username=username, password=password, port=port)
+    def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None):
+        sftp = pysftp.Connection(host, username=username, password=password, port=port, private_key=privateKey)
         return sftp
 
     @classmethod
@@ -53,7 +54,7 @@ class SftpUtility:
         
     def getFiles(self, dirPath, fileDict={}, replacementDirPath=None):
         if not self.sftpClient:
-            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password)
+            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
         if not replacementDirPath:
             replacementDirPath = dirPath
         attrs = self.sftpClient.listdir_attr(dirPath)
@@ -72,12 +73,19 @@ class SftpUtility:
                 fileDict[filePath] = fileInfo
         return fileDict
 
+    def getMd5Sum(self, filePath, fileInfo={}):
+        if not self.sftpClient:
+            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
+        md5Sum = self.sftpClient.execute('md5sum %s' % filePath)[0].split()[0]
+        fileInfo['md5Sum'] = md5Sum
+        return md5Sum
+
 #######################################################################
 # Testing.
 
 if __name__ == '__main__':
-    sftpUtility = SftpUtility('xstor-devel', username='dmadmin')
-    files = sftpUtility.getFiles('/data/testing/test1')
-    print files
-    files = sftpUtility.getFiles('/data/testing/test1', replacementDirPath='/xyz/ccc')
+    #sftpUtility = SftpUtility('s1dserv', username='dmadmin', password='theKey12')
+    sftpUtility = SftpUtility('s1dserv',privateKey='/home/beams/DMADMIN/.ssh/id_dsa')
+    files = sftpUtility.getFiles('/export/dm/test')
     print files
+    print sftpUtility.getMd5Sum('/export/dm/test/testfile01')

src/python/dm/daq_web_service/service/impl/sftpFileSystemObserverAgent.py

+#!/usr/bin/env python
+
+from threading import Timer
+from pollingFileSystemObserverAgent import PollingFileSystemObserverAgent
+from dm.common.utility.sftpUtility import SftpUtility
+
+class SftpFileSystemObserverAgent(PollingFileSystemObserverAgent):
+
+    DEFAULT_POLLING_PERIOD_IN_SECONDS = 5
+    DEFAULT_PORT = 22
+
+    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None, pollingPeriod=DEFAULT_POLLING_PERIOD_IN_SECONDS):
+        PollingFileSystemObserverAgent.__init__(self, pollingPeriod)
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.privateKey = privateKey
+
+    def getFiles(self, dataDirectory):
+        (scheme, host, port, dirPath) = SftpUtility.parseFtpUrl(dataDirectory, defaultHost=self.host, defaultPort=self.port)
+        self.logger.debug('Retrieving files from SFTP host: %s, port: %s, directory path: %s' % (host, port, dirPath))
+        sftpUtility = SftpUtility(host, port, self.username, self.password, self.privateKey)
+        return sftpUtility.getFiles(dirPath, {})
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    import time
+    dirPath='/export/beams12/S1IDUSER/mnt/orthros/park_apr16_rec_reduced'
+    agent = SftpFileSystemObserverAgent('s1dserv', privateKey='/home/beams/DMADMIN/.ssh/id_dsa')
+    print 'TIME1: ', time.time()
+    print 'ORIGINAL FILES: ', len(agent.getFiles(dirPath))
+    print 'TIME2: ', time.time()
+    #agent.startObservingPath('/export/dm/test', 'e1')
+    #time.sleep(100)
+    #agent.stopObservingPath('/export/dm/test', 'e1')
+