merge from 0.7 branch

src/python/dm/common/utility/ftpUtility.py

@@ -82,6 +82,6 @@ class FtpUtility:
 # Testing.
 
 if __name__ == '__main__':
-    ftpUtility = FtpUtility('zagreb', 2811)
-    files=ftpUtility.getFiles('/tmp/test')
+    ftpUtility = FtpUtility('s8dserv', 2811)
+    files=ftpUtility.getFiles('/export/8-id-i/test')
     print files

src/python/dm/common/utility/ldapLinuxPlatformUtility.py

 #!/usr/bin/env python
 
+import grp
 import ldap
 import ldap.modlist
 import copy
@@ -165,22 +166,57 @@ class LdapLinuxPlatformUtility:
             logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
             raise InternalError(exception=ex)
 
+    def deleteUserFromGroup(self, username, groupName):
+        """ Remove user from group. """
+        logger = self.getLogger()
+        ldapClient = self.getLdapClient()
+        username = str(username)
+        groupName = str(groupName)
+        try:
+            groupDn = self.groupDnFormat % groupName
+            resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+            groupTuple = resultList[0]
+            groupAttrs = groupTuple[1]
+            memberUidList = groupAttrs.get('memberUid', [])
+            if username not in memberUidList:
+                logger.debug('Group %s does not contain user %s' % (groupName, username))
+                return
+        except Exception, ex:
+            raise InternalError(exception=ex)
+        logger.debug('Removing user %s from group %s' % (username, groupName))
+        memberUidList2 = copy.copy(memberUidList)
+        memberUidList2.remove(username)
+        groupAttrs2 = copy.copy(groupAttrs)
+        groupAttrs2['memberUid'] = memberUidList2
+        try:
+            groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+            ldapClient.modify_s(groupDn, groupLdif)
+        except Exception, ex:
+            logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
+            raise InternalError(exception=ex)
 
     @classmethod
-    def addLocalUserToGroup(cls, username, groupName):
-        """ Add local user to group. """
+    def createLocalGroup(cls, name):
+        """ Create local group if it does not exist. """
         logger = cls.getLogger()
-        logger.debug('Adding local user %s to group %s' % (username, groupName))
-        cmd = '%s -a %s %s' % (cls.GPASSWD_CMD, username, groupName)
+        try:
+            group = grp.getgrnam(name)
+            logger.debug('Group %s already exists' % name)
+            return
+        except KeyError, ex:
+            # ok, we need to create group
+            pass
+        logger.debug('Creating group %s' % name)
+        cmd = '%s %s' % (cls.GROUPADD_CMD, name)
         cls.executeSudoCommand(cmd)
 
-    @classmethod
-    def deleteLocalUserFromGroup(cls, username, groupName):
+    def addLocalUserToGroup(self, username, groupName):
+        """ Add local user to group. """
+        self.addUserToGroup(username, groupName)
+
+    def deleteLocalUserFromGroup(self, username, groupName):
         """ Remove local user from group. """
-        logger = cls.getLogger()
-        logger.debug('Removing local user %s from group %s' % (username, groupName))
-        cmd = '%s -d %s %s' % (cls.GPASSWD_CMD, username, groupName)
-        cls.executeSudoCommand(cmd)
+        self.deleteUserFromGroup(username, groupName)
 
     def getGroupInfo(self, groupName):
         """ Get given group info. """
@@ -241,5 +277,9 @@ class LdapLinuxPlatformUtility:
 
 if __name__ == '__main__':
     utility = LdapLinuxPlatformUtility('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=dmadmin,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', '/tmp/ldapPassword', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
-    print utility.getGroupInfo(u'satija201510')
+    print utility.getGroupInfo(u'id8i-test02')
+    utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
+    print utility.getGroupInfo(u'id8i-test02')
+    #utility.deleteLocalUserFromGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
 

src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py

@@ -86,7 +86,12 @@ class ExperimentSessionControllerImpl(DmObjectManager):
             fileInfo['daqInfo'] = daqInfo
             fileInfo['uploadId'] = uploadId
             fileUploadInfo = { 'processed' : False }
-            FileUtility.statFile(filePath, fileUploadInfo)
+	    # Stat should be done by agent, not by observer.
+	    try:
+                FileUtility.statFile(filePath, fileUploadInfo)
+            except:
+	        # Ok, may be remote file
+		pass
             fileDict[filePath] = fileUploadInfo
             fileProcessingManager.processFile(fileInfo)
         uploadInfo['fileDict'] = fileDict

src/python/dm/ds_web_service/api/fileRestApi.py

@@ -64,8 +64,10 @@ class FileRestApi(DsRestApi):
             if experimentFilePath:
                 src = '%s/%s' % (src, experimentFilePath)
             dest = destDirectory
-             
-            fileTransfer = RsyncFileTransfer(src=src, dest=dest)
+            
+            rsyncPath = '/tmp/rsync.%s.%s' % (username, experimentName)
+            flags = '-arvlP --rsync-path="%s"' % rsyncPath
+            fileTransfer = RsyncFileTransfer(src=src, dest=dest, flags=flags)
             self.logger.info('Executing file download on behalf of %s (experiment: %s)' % (username, experimentName))
             fileTransfer.execute()
         finally:

src/python/dm/ds_web_service/service/impl/experimentManager.py

@@ -24,6 +24,7 @@ class ExperimentManager(Singleton):
     MANAGE_STORAGE_PERMISSIONS_KEY = 'managestoragepermissions'
     PLATFORM_UTILITY_KEY = 'platformutility'
 
+    RSYNC_SCRIPT_PERMISSIONS_MODE = 0755
     FILE_PERMISSIONS_MODE = 0640
     DIR_PERMISSIONS_MODE = 0750
 
@@ -91,22 +92,39 @@ class ExperimentManager(Singleton):
             experimentUsers = experiment.get('experimentUsernameList', [])
             self.platformUtility.setGroupUsers(experimentName, experimentUsers)
 
+    def createRsyncScript(self, username, experimentName):
+        fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
+        self.logger.debug('Creating rsync script %s' % (fileName))
+        f = open(fileName, 'w')
+        f.write('#!/bin/sh\n')
+        f.write('exec sg %s "rsync $*"\n' % experimentName)
+        f.close()
+        OsUtility.chmodPath(fileName, fileMode=self.RSYNC_SCRIPT_PERMISSIONS_MODE)
+    def deleteRsyncScript(self, username, experimentName):
+        fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
+        self.logger.debug('Removing rsync script %s' % (fileName))
+        OsUtility.removeFile(fileName)
+
     def authorizeDownload(self, username, experimentName):
+        self.logger.debug('Authorizing download for %s from experiment %s' % (username, experimentName))
         experiment = self.experimentDbApi.getExperimentByName(experimentName)
         storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
         if os.path.exists(storageDirectory):
             self.platformUtility.addLocalUserToGroup(username, experimentName)
         else:
             raise InvalidRequest('Experiment %s has not been started.' % experimentName)
+        self.createRsyncScript(username, experimentName)
         return experiment
 
     def deauthorizeDownload(self, username, experimentName):
         experiment = self.experimentDbApi.getExperimentByName(experimentName)
+        self.logger.debug('De-authorizing download for %s from experiment %s' % (username, experimentName))
         storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
         if os.path.exists(storageDirectory):
             self.platformUtility.deleteLocalUserFromGroup(username, experimentName)
         else:
             raise InvalidRequest('Experiment %s has not been started.' % experimentName)
+        self.deleteRsyncScript(username, experimentName)
         return experiment
 
     def createExperimentGroup(self, experiment):