diff --git a/src/python/dm/common/utility/ftpUtility.py b/src/python/dm/common/utility/ftpUtility.py
index ab6f03146e13f9b4b90361973627e8001af13a83..8a75aed46f25d0ed5805ca8d33ca6486bc6b9ba0 100755
--- a/src/python/dm/common/utility/ftpUtility.py
+++ b/src/python/dm/common/utility/ftpUtility.py
@@ -82,6 +82,6 @@ class FtpUtility:
# Testing.
if __name__ == '__main__':
- ftpUtility = FtpUtility('zagreb', 2811)
- files=ftpUtility.getFiles('/tmp/test')
+ ftpUtility = FtpUtility('s8dserv', 2811)
+ files=ftpUtility.getFiles('/export/8-id-i/test')
print files
diff --git a/src/python/dm/common/utility/ldapLinuxPlatformUtility.py b/src/python/dm/common/utility/ldapLinuxPlatformUtility.py
index 4db4845577f4b25d4d44ae666ed51701a4b01f98..3b4e1a1aff2c0dba6ef9b356d9db3023a8c29417 100755
--- a/src/python/dm/common/utility/ldapLinuxPlatformUtility.py
+++ b/src/python/dm/common/utility/ldapLinuxPlatformUtility.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
+import grp
import ldap
import ldap.modlist
import copy
@@ -165,22 +166,57 @@ class LdapLinuxPlatformUtility:
logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
raise InternalError(exception=ex)
+ def deleteUserFromGroup(self, username, groupName):
+ """ Remove user from group. """
+ logger = self.getLogger()
+ ldapClient = self.getLdapClient()
+ username = str(username)
+ groupName = str(groupName)
+ try:
+ groupDn = self.groupDnFormat % groupName
+ resultList = ldapClient.search_s(groupDn, ldap.SCOPE_BASE)
+ groupTuple = resultList[0]
+ groupAttrs = groupTuple[1]
+ memberUidList = groupAttrs.get('memberUid', [])
+ if username not in memberUidList:
+ logger.debug('Group %s does not contain user %s' % (groupName, username))
+ return
+ except Exception, ex:
+ raise InternalError(exception=ex)
+ logger.debug('Removing user %s from group %s' % (username, groupName))
+ memberUidList2 = copy.copy(memberUidList)
+ memberUidList2.remove(username)
+ groupAttrs2 = copy.copy(groupAttrs)
+ groupAttrs2['memberUid'] = memberUidList2
+ try:
+ groupLdif = ldap.modlist.modifyModlist(groupAttrs, groupAttrs2)
+ ldapClient.modify_s(groupDn, groupLdif)
+ except Exception, ex:
+ logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
+ raise InternalError(exception=ex)
@classmethod
- def addLocalUserToGroup(cls, username, groupName):
- """ Add local user to group. """
+ def createLocalGroup(cls, name):
+ """ Create local group if it does not exist. """
logger = cls.getLogger()
- logger.debug('Adding local user %s to group %s' % (username, groupName))
- cmd = '%s -a %s %s' % (cls.GPASSWD_CMD, username, groupName)
+ try:
+ group = grp.getgrnam(name)
+ logger.debug('Group %s already exists' % name)
+ return
+ except KeyError, ex:
+ # ok, we need to create group
+ pass
+ logger.debug('Creating group %s' % name)
+ cmd = '%s %s' % (cls.GROUPADD_CMD, name)
cls.executeSudoCommand(cmd)
- @classmethod
- def deleteLocalUserFromGroup(cls, username, groupName):
+ def addLocalUserToGroup(self, username, groupName):
+ """ Add local user to group. """
+ self.addUserToGroup(username, groupName)
+
+ def deleteLocalUserFromGroup(self, username, groupName):
""" Remove local user from group. """
- logger = cls.getLogger()
- logger.debug('Removing local user %s from group %s' % (username, groupName))
- cmd = '%s -d %s %s' % (cls.GPASSWD_CMD, username, groupName)
- cls.executeSudoCommand(cmd)
+ self.deleteUserFromGroup(username, groupName)
def getGroupInfo(self, groupName):
""" Get given group info. """
@@ -241,5 +277,9 @@ class LdapLinuxPlatformUtility:
if __name__ == '__main__':
utility = LdapLinuxPlatformUtility('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=dmadmin,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', '/tmp/ldapPassword', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
- print utility.getGroupInfo(u'satija201510')
+ print utility.getGroupInfo(u'id8i-test02')
+ utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
+ print utility.getGroupInfo(u'id8i-test02')
+ #utility.deleteLocalUserFromGroup(u'sveseli', u'id8i-test02')
+ #print utility.getGroupInfo(u'id8i-test02')
diff --git a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
index c047191b20cf0088144e83df7bf61b85dcb348c6..07085c6d180fb1717b6bd48f664dd47dd0d3662a 100755
--- a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
+++ b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
@@ -86,7 +86,12 @@ class ExperimentSessionControllerImpl(DmObjectManager):
fileInfo['daqInfo'] = daqInfo
fileInfo['uploadId'] = uploadId
fileUploadInfo = { 'processed' : False }
- FileUtility.statFile(filePath, fileUploadInfo)
+ # Stat should be done by agent, not by observer.
+ try:
+ FileUtility.statFile(filePath, fileUploadInfo)
+ except:
+ # Ok, may be remote file
+ pass
fileDict[filePath] = fileUploadInfo
fileProcessingManager.processFile(fileInfo)
uploadInfo['fileDict'] = fileDict
diff --git a/src/python/dm/ds_web_service/api/fileRestApi.py b/src/python/dm/ds_web_service/api/fileRestApi.py
index 6d0c5d96683d53ea32e434ea8f61bb4f2815597e..52cf596eef2f7d795662d86159cc81539b69d7ba 100755
--- a/src/python/dm/ds_web_service/api/fileRestApi.py
+++ b/src/python/dm/ds_web_service/api/fileRestApi.py
@@ -64,8 +64,10 @@ class FileRestApi(DsRestApi):
if experimentFilePath:
src = '%s/%s' % (src, experimentFilePath)
dest = destDirectory
-
- fileTransfer = RsyncFileTransfer(src=src, dest=dest)
+
+ rsyncPath = '/tmp/rsync.%s.%s' % (username, experimentName)
+ flags = '-arvlP --rsync-path="%s"' % rsyncPath
+ fileTransfer = RsyncFileTransfer(src=src, dest=dest, flags=flags)
self.logger.info('Executing file download on behalf of %s (experiment: %s)' % (username, experimentName))
fileTransfer.execute()
finally:
diff --git a/src/python/dm/ds_web_service/service/impl/experimentManager.py b/src/python/dm/ds_web_service/service/impl/experimentManager.py
index 108b62ff5d01959dafffc3962d75504dbd4ed626..de809aeb4357f43bfd568911358d774d9c8b18f1 100755
--- a/src/python/dm/ds_web_service/service/impl/experimentManager.py
+++ b/src/python/dm/ds_web_service/service/impl/experimentManager.py
@@ -24,6 +24,7 @@ class ExperimentManager(Singleton):
MANAGE_STORAGE_PERMISSIONS_KEY = 'managestoragepermissions'
PLATFORM_UTILITY_KEY = 'platformutility'
+ RSYNC_SCRIPT_PERMISSIONS_MODE = 0755
FILE_PERMISSIONS_MODE = 0640
DIR_PERMISSIONS_MODE = 0750
@@ -91,22 +92,39 @@ class ExperimentManager(Singleton):
experimentUsers = experiment.get('experimentUsernameList', [])
self.platformUtility.setGroupUsers(experimentName, experimentUsers)
+ def createRsyncScript(self, username, experimentName):
+ fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
+ self.logger.debug('Creating rsync script %s' % (fileName))
+ f = open(fileName, 'w')
+ f.write('#!/bin/sh\n')
+ f.write('exec sg %s "rsync $*"\n' % experimentName)
+ f.close()
+ OsUtility.chmodPath(fileName, fileMode=self.RSYNC_SCRIPT_PERMISSIONS_MODE)
+ def deleteRsyncScript(self, username, experimentName):
+ fileName = '/tmp/rsync.%s.%s' % (username, experimentName)
+ self.logger.debug('Removing rsync script %s' % (fileName))
+ OsUtility.removeFile(fileName)
+
def authorizeDownload(self, username, experimentName):
+ self.logger.debug('Authorizing download for %s from experiment %s' % (username, experimentName))
experiment = self.experimentDbApi.getExperimentByName(experimentName)
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.addLocalUserToGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
+ self.createRsyncScript(username, experimentName)
return experiment
def deauthorizeDownload(self, username, experimentName):
experiment = self.experimentDbApi.getExperimentByName(experimentName)
+ self.logger.debug('De-authorizing download for %s from experiment %s' % (username, experimentName))
storageDirectory = self.updateExperimentWithStorageDataDirectory(experiment)
if os.path.exists(storageDirectory):
self.platformUtility.deleteLocalUserFromGroup(username, experimentName)
else:
raise InvalidRequest('Experiment %s has not been started.' % experimentName)
+ self.deleteRsyncScript(username, experimentName)
return experiment
def createExperimentGroup(self, experiment):