From 71f6658a4f9d988338c76500fd0e190a9699e186 Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Fri, 27 May 2016 17:39:11 +0000
Subject: [PATCH] changes to sftp utility to allow calculating md5 sum, as well
 as to be able to use ssh key for authorization

---
 src/python/dm/common/utility/sftpUtility.py   | 24 ++++++++----
 .../impl/sftpFileSystemObserverAgent.py       | 39 +++++++++++++++++++
 2 files changed, 55 insertions(+), 8 deletions(-)
 create mode 100755 src/python/dm/daq_web_service/service/impl/sftpFileSystemObserverAgent.py

diff --git a/src/python/dm/common/utility/sftpUtility.py b/src/python/dm/common/utility/sftpUtility.py
index b79a2c7c..64d9881c 100755
--- a/src/python/dm/common/utility/sftpUtility.py
+++ b/src/python/dm/common/utility/sftpUtility.py
@@ -11,11 +11,12 @@ class SftpUtility:
 
     DEFAULT_PORT = 22
 
-    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None):
+    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None):
         self.host = host
         self.port = port
         self.username = username
         self.password = password
+        self.privateKey = privateKey
         self.sftpClient = None
 
     @classmethod
@@ -35,8 +36,8 @@ class SftpUtility:
         return (scheme, host, port, dirPath)
 
     @classmethod
-    def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None):
-        sftp = pysftp.Connection(host, username=username, password=password, port=port)
+    def getSftpClient(cls, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None):
+        sftp = pysftp.Connection(host, username=username, password=password, port=port, private_key=privateKey)
         return sftp
 
     @classmethod
@@ -53,7 +54,7 @@ class SftpUtility:
         
     def getFiles(self, dirPath, fileDict={}, replacementDirPath=None):
         if not self.sftpClient:
-            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password)
+            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
         if not replacementDirPath:
             replacementDirPath = dirPath
         attrs = self.sftpClient.listdir_attr(dirPath)
@@ -72,12 +73,19 @@ class SftpUtility:
                 fileDict[filePath] = fileInfo
         return fileDict
 
+    def getMd5Sum(self, filePath, fileInfo={}):
+        if not self.sftpClient:
+            self.sftpClient = self.getSftpClient(self.host, self.port, self.username, self.password, self.privateKey)
+        md5Sum = self.sftpClient.execute('md5sum %s' % filePath)[0].split()[0]
+        fileInfo['md5Sum'] = md5Sum
+        return md5Sum
+
 #######################################################################
 # Testing.
 
 if __name__ == '__main__':
-    sftpUtility = SftpUtility('xstor-devel', username='dmadmin')
-    files = sftpUtility.getFiles('/data/testing/test1')
-    print files
-    files = sftpUtility.getFiles('/data/testing/test1', replacementDirPath='/xyz/ccc')
+    #sftpUtility = SftpUtility('s1dserv', username='dmadmin', password='theKey12')
+    sftpUtility = SftpUtility('s1dserv',privateKey='/home/beams/DMADMIN/.ssh/id_dsa')
+    files = sftpUtility.getFiles('/export/dm/test')
     print files
+    print sftpUtility.getMd5Sum('/export/dm/test/testfile01')
diff --git a/src/python/dm/daq_web_service/service/impl/sftpFileSystemObserverAgent.py b/src/python/dm/daq_web_service/service/impl/sftpFileSystemObserverAgent.py
new file mode 100755
index 00000000..33b406d1
--- /dev/null
+++ b/src/python/dm/daq_web_service/service/impl/sftpFileSystemObserverAgent.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+from threading import Timer
+from pollingFileSystemObserverAgent import PollingFileSystemObserverAgent
+from dm.common.utility.sftpUtility import SftpUtility
+
+class SftpFileSystemObserverAgent(PollingFileSystemObserverAgent):
+
+    DEFAULT_POLLING_PERIOD_IN_SECONDS = 5
+    DEFAULT_PORT = 22
+
+    def __init__(self, host, port=DEFAULT_PORT, username=None, password=None, privateKey=None, pollingPeriod=DEFAULT_POLLING_PERIOD_IN_SECONDS):
+        PollingFileSystemObserverAgent.__init__(self, pollingPeriod)
+        self.host = host
+        self.port = port
+        self.username = username
+        self.password = password
+        self.privateKey = privateKey
+
+    def getFiles(self, dataDirectory):
+        (scheme, host, port, dirPath) = SftpUtility.parseFtpUrl(dataDirectory, defaultHost=self.host, defaultPort=self.port)
+        self.logger.debug('Retrieving files from SFTP host: %s, port: %s, directory path: %s' % (host, port, dirPath))
+        sftpUtility = SftpUtility(host, port, self.username, self.password, self.privateKey)
+        return sftpUtility.getFiles(dirPath, {})
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    import time
+    dirPath='/export/beams12/S1IDUSER/mnt/orthros/park_apr16_rec_reduced'
+    agent = SftpFileSystemObserverAgent('s1dserv', privateKey='/home/beams/DMADMIN/.ssh/id_dsa')
+    print 'TIME1: ', time.time()
+    print 'ORIGINAL FILES: ', len(agent.getFiles(dirPath))
+    print 'TIME2: ', time.time()
+    #agent.startObservingPath('/export/dm/test', 'e1')
+    #time.sleep(100)
+    #agent.stopObservingPath('/export/dm/test', 'e1')
+
-- 
GitLab