From 9a004c5230c1b086c5eea1d8de18dc684422cd5b Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Thu, 25 Feb 2016 17:52:00 +0000
Subject: [PATCH] introduce timer for directory processing; fix for updating
file permissions when directory processing is used
---
.../utility/ldapLinuxPlatformUtility.py | 8 ++
src/python/dm/common/utility/linuxUtility.py | 13 ++
.../service/impl/experimentManager.py | 112 +++++++++++-------
3 files changed, 88 insertions(+), 45 deletions(-)
diff --git a/src/python/dm/common/utility/ldapLinuxPlatformUtility.py b/src/python/dm/common/utility/ldapLinuxPlatformUtility.py
index 76097df6..ca074a4f 100755
--- a/src/python/dm/common/utility/ldapLinuxPlatformUtility.py
+++ b/src/python/dm/common/utility/ldapLinuxPlatformUtility.py
@@ -23,6 +23,7 @@ class LdapLinuxPlatformUtility:
CHOWN_CMD = '/bin/chown'
GPASSWD_CMD = '/usr/bin/gpasswd'
NSCD_CMD = '/usr/sbin/nscd'
+ FIND_CMD = '/bin/find'
CONFIG_SECTION_NAME = 'LdapLinuxPlatformUtility'
REFRESH_AUTH_FILES_COMMAND_KEY = 'refreshauthfilescommand'
@@ -337,6 +338,13 @@ class LdapLinuxPlatformUtility:
except Exception, ex:
logger.warn('Failed to refresh auth files: %s' % (str(ex)))
+ @classmethod
+ def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+ logger = cls.getLogger()
+ logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+ cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+ cls.executeCommand(cmd)
+
#######################################################################
# Testing.
diff --git a/src/python/dm/common/utility/linuxUtility.py b/src/python/dm/common/utility/linuxUtility.py
index 0fd3ccdf..1e62092b 100755
--- a/src/python/dm/common/utility/linuxUtility.py
+++ b/src/python/dm/common/utility/linuxUtility.py
@@ -12,6 +12,7 @@ class LinuxUtility:
SETFACL_CMD = '/usr/bin/setfacl'
CHOWN_CMD = '/bin/chown'
GPASSWD_CMD = '/usr/bin/gpasswd'
+ FIND_CMD = '/bin/find'
@classmethod
def getLogger(cls):
@@ -23,6 +24,11 @@ class LinuxUtility:
p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
p.run()
+ @classmethod
+ def executeCommand(cls, cmd):
+ p = DmSubprocess('%s' % (cmd))
+ p.run()
+
@classmethod
def createGroup(cls, name):
""" Create group if it does not exist. """
@@ -92,6 +98,13 @@ class LinuxUtility:
cmd = '%s -R \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
cls.executeSudoCommand(cmd)
+ @classmethod
+ def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+ logger = cls.getLogger()
+ logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+ cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+ cls.executeCommand(cmd)
+
#######################################################################
# Testing.
diff --git a/src/python/dm/ds_web_service/service/impl/experimentManager.py b/src/python/dm/ds_web_service/service/impl/experimentManager.py
index 5b657251..eb6a697a 100755
--- a/src/python/dm/ds_web_service/service/impl/experimentManager.py
+++ b/src/python/dm/ds_web_service/service/impl/experimentManager.py
@@ -28,8 +28,11 @@ class ExperimentManager(Singleton):
RSYNC_SCRIPT_PERMISSIONS_MODE = 0755
FILE_PERMISSIONS_MODE = 0640
+ FILE_PERMISSIONS_MODE_STRING = '0640'
DIR_PERMISSIONS_MODE = 0750
+ DIRECTORY_PROCESSING_DELAY_IN_SECONDS = 1
+
# Singleton.
__instanceLock = threading.RLock()
__instance = None
@@ -173,7 +176,6 @@ class ExperimentManager(Singleton):
if self.manageStoragePermissions:
self.createExperimentGroup(experiment)
- @ThreadingUtility.synchronize
def processExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
@@ -181,29 +183,30 @@ class ExperimentManager(Singleton):
filePath = os.path.join(storageDirectory, experimentFilePath)
fileInfo['filePath'] = filePath
fileInfo['experiment'] = experiment
- if os.path.exists(filePath):
- self.logger.debug('Processing file path %s (fileInfo: %s)' % (filePath, fileInfo))
- if self.manageStoragePermissions:
- self.logger.debug('Modifying permissions for %s' % filePath)
- OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
- self.logger.debug('Changing group owner for %s to %s' % (filePath, experimentName))
- self.platformUtility.changePathGroupOwner(filePath, experimentName)
- # Recursively modify subdirectory permissions
- dirPath = os.path.dirname(filePath)
- while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
- if self.pathTracker.get(dirPath) is None:
- self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
- self.platformUtility.changePathGroupOwner(dirPath, experimentName)
- ownerUpdateTime = time.time()
- self.pathTracker.put(dirPath, ownerUpdateTime)
- else:
- self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
- dirPath = os.path.dirname(dirPath)
-
- self.logger.debug('Processing file %s' % filePath)
- self.fileProcessingManager.processFile(fileInfo)
- else:
- self.logger.debug('File path %s does not exist' % filePath)
+ if not os.path.exists(filePath):
+ self.logger.error('File path %s does not exist' % filePath)
+ return
+
+ if self.manageStoragePermissions:
+ self.logger.debug('Setting permissions for %s to %s' % (filePath,self.FILE_PERMISSIONS_MODE_STRING))
+ OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
+ self.logger.debug('Changing group owner for %s to %s' % (filePath, experimentName))
+ self.platformUtility.changePathGroupOwner(filePath, experimentName)
+
+ # Recursively modify subdirectory permissions
+ dirPath = os.path.dirname(filePath)
+ while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
+ if self.pathTracker.get(dirPath) is None:
+ self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
+ self.platformUtility.changePathGroupOwner(dirPath, experimentName)
+ ownerUpdateTime = time.time()
+ self.pathTracker.put(dirPath, ownerUpdateTime)
+ else:
+ self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
+ dirPath = os.path.dirname(dirPath)
+
+ self.logger.debug('Processing file path %s (fileInfo: %s)' % (filePath, fileInfo))
+ self.fileProcessingManager.processFile(fileInfo)
def statExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
experimentName = experiment.get('name')
@@ -220,7 +223,6 @@ class ExperimentManager(Singleton):
self.logger.debug('File path %s does not exist' % filePath)
raise ObjectNotFound('File %s does not exist' % filePath)
- @ThreadingUtility.synchronize
def processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
experimentName = experiment.get('name')
self.updateExperimentWithStorageDataDirectory(experiment)
@@ -228,26 +230,46 @@ class ExperimentManager(Singleton):
directoryPath = os.path.join(storageDirectory, experimentDirectoryPath)
directoryInfo['directoryPath'] = directoryPath
directoryInfo['experiment'] = experiment
- if os.path.exists(directoryPath):
- self.logger.debug('Processing directory path %s (directoryInfo: %s)' % (directoryPath, directoryInfo))
- if self.manageStoragePermissions:
- self.logger.debug('Modifying permissions for directory %s to %s' % (directoryPath, self.DIR_PERMISSIONS_MODE))
- OsUtility.chmodPath(directoryPath, dirMode=self.DIR_PERMISSIONS_MODE)
- self.logger.debug('Changing group owner for %s to %s' % (directoryPath, experimentName))
- self.platformUtility.recursivelyChangePathGroupOwner(directoryPath, experimentName)
- # Recursively modify subdirectory permissions
- dirPath = os.path.dirname(directoryPath)
- while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
- if self.pathTracker.get(dirPath) is None:
- self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
- self.platformUtility.changePathGroupOwner(dirPath, experimentName)
- ownerUpdateTime = time.time()
- self.pathTracker.put(dirPath, ownerUpdateTime)
- else:
- self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
- dirPath = os.path.dirname(dirPath)
- else:
- self.logger.debug('Directory path %s does not exist' % directoryPath)
+
+ if not self.manageStoragePermissions:
+ self.logger.error('Skipping permission management for directory path %s' % directoryPath)
+ return
+
+ self.logger.debug('Processing directory path %s in background' % (directoryPath))
+ timer = threading.Timer(self.DIRECTORY_PROCESSING_DELAY_IN_SECONDS, self.__processExperimentDirectory, args=[experimentDirectoryPath, experiment, directoryInfo])
+ timer.start()
+
+ def __processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
+ experimentName = experiment.get('name')
+ storageDirectory = experiment.get('storageDirectory')
+ directoryPath = directoryInfo.get('directoryPath')
+
+ if not os.path.exists(directoryPath):
+ self.logger.error('Directory path %s does not exist' % directoryPath)
+ return
+
+ # Modify ownership
+ self.logger.debug('Processing directory path %s (directoryInfo: %s)' % (directoryPath, directoryInfo))
+ self.logger.debug('Modifying permissions for directory %s to %s' % (directoryPath, self.DIR_PERMISSIONS_MODE))
+ OsUtility.chmodPath(directoryPath, dirMode=self.DIR_PERMISSIONS_MODE)
+ self.logger.debug('Changing group owner for %s to %s' % (directoryPath, experimentName))
+ self.platformUtility.recursivelyChangePathGroupOwner(directoryPath, experimentName)
+
+ # Recursively modify subdirectory permissions
+ dirPath = os.path.dirname(directoryPath)
+ while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
+ if self.pathTracker.get(dirPath) is None:
+ self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
+ self.platformUtility.changePathGroupOwner(dirPath, experimentName)
+ ownerUpdateTime = time.time()
+ self.pathTracker.put(dirPath, ownerUpdateTime)
+ else:
+ self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
+ dirPath = os.path.dirname(dirPath)
+
+ # Update file permissions
+ self.logger.debug('Changing permissions for all files in %s for experiment %s' % (directoryPath, experimentName))
+ self.platformUtility.chmodPathForFilesInDirectory(directoryPath, self.FILE_PERMISSIONS_MODE_STRING)
@ThreadingUtility.synchronize
def start(self):
--
GitLab