introduce timer for directory processing; fix for updating file permissions...

introduce timer for directory processing; fix for updating file permissions when directory processing is used

src/python/dm/common/utility/ldapLinuxPlatformUtility.py

@@ -23,6 +23,7 @@ class LdapLinuxPlatformUtility:
     CHOWN_CMD = '/bin/chown'
     GPASSWD_CMD = '/usr/bin/gpasswd'
     NSCD_CMD = '/usr/sbin/nscd'
+    FIND_CMD = '/bin/find'
 
     CONFIG_SECTION_NAME = 'LdapLinuxPlatformUtility'
     REFRESH_AUTH_FILES_COMMAND_KEY = 'refreshauthfilescommand'
@@ -337,6 +338,13 @@ class LdapLinuxPlatformUtility:
         except Exception, ex:
             logger.warn('Failed to refresh auth files: %s' % (str(ex)))
 
+    @classmethod
+    def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+        logger = cls.getLogger()
+        logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+        cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+        cls.executeCommand(cmd)
+
 #######################################################################
 # Testing.
 

src/python/dm/common/utility/linuxUtility.py

@@ -12,6 +12,7 @@ class LinuxUtility:
     SETFACL_CMD = '/usr/bin/setfacl'
     CHOWN_CMD = '/bin/chown'
     GPASSWD_CMD = '/usr/bin/gpasswd'
+    FIND_CMD = '/bin/find'
 
     @classmethod
     def getLogger(cls):
@@ -23,6 +24,11 @@ class LinuxUtility:
         p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
         p.run()
 
+    @classmethod
+    def executeCommand(cls, cmd):
+        p = DmSubprocess('%s' % (cmd))
+        p.run()
+
     @classmethod
     def createGroup(cls, name):
         """ Create group if it does not exist. """
@@ -92,6 +98,13 @@ class LinuxUtility:
         cmd = '%s -R \:%s "%s"' % (cls.CHOWN_CMD, groupName, path)
         cls.executeSudoCommand(cmd)
 
+    @classmethod
+    def chmodPathForFilesInDirectory(cls, directoryPath, fileMode):
+        logger = cls.getLogger()
+        logger.debug('Modifying permissions for all files in directory %s to %s' % (directoryPath, fileMode))
+        cmd = '%s %s -type f -exec chmod %s {} \;' % (cls.FIND_CMD, directoryPath, fileMode)
+        cls.executeCommand(cmd)
+
 #######################################################################
 # Testing.
 

src/python/dm/ds_web_service/service/impl/experimentManager.py

@@ -28,8 +28,11 @@ class ExperimentManager(Singleton):
 
     RSYNC_SCRIPT_PERMISSIONS_MODE = 0755
     FILE_PERMISSIONS_MODE = 0640
+    FILE_PERMISSIONS_MODE_STRING = '0640'
     DIR_PERMISSIONS_MODE = 0750
 
+    DIRECTORY_PROCESSING_DELAY_IN_SECONDS = 1
+
     # Singleton.
     __instanceLock = threading.RLock()
     __instance = None
@@ -173,7 +176,6 @@ class ExperimentManager(Singleton):
         if self.manageStoragePermissions:
             self.createExperimentGroup(experiment)
       
-    @ThreadingUtility.synchronize
     def processExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
         experimentName = experiment.get('name')
         self.updateExperimentWithStorageDataDirectory(experiment)
@@ -181,29 +183,30 @@ class ExperimentManager(Singleton):
         filePath = os.path.join(storageDirectory, experimentFilePath)
         fileInfo['filePath'] = filePath
         fileInfo['experiment'] = experiment
-        if os.path.exists(filePath):
-            self.logger.debug('Processing file path %s (fileInfo: %s)' % (filePath, fileInfo))
-            if self.manageStoragePermissions:
-                self.logger.debug('Modifying permissions for %s' % filePath)
-                OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
-                self.logger.debug('Changing group owner for %s to %s' % (filePath, experimentName))
-                self.platformUtility.changePathGroupOwner(filePath, experimentName)
-                # Recursively modify subdirectory permissions
-                dirPath = os.path.dirname(filePath)
-                while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
-                    if self.pathTracker.get(dirPath) is None:
-                        self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
-                        self.platformUtility.changePathGroupOwner(dirPath, experimentName)
-                        ownerUpdateTime = time.time()
-                        self.pathTracker.put(dirPath, ownerUpdateTime)
-                    else: 
-                        self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
-                    dirPath = os.path.dirname(dirPath)
-
-                self.logger.debug('Processing file %s' % filePath)
-                self.fileProcessingManager.processFile(fileInfo)
-        else:
-            self.logger.debug('File path %s does not exist' % filePath)
+        if not os.path.exists(filePath):
+            self.logger.error('File path %s does not exist' % filePath)
+            return
+
+        if self.manageStoragePermissions:
+            self.logger.debug('Setting permissions for %s to %s' % (filePath,self.FILE_PERMISSIONS_MODE_STRING))
+            OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
+            self.logger.debug('Changing group owner for %s to %s' % (filePath, experimentName))
+            self.platformUtility.changePathGroupOwner(filePath, experimentName)
+
+            # Recursively modify subdirectory permissions
+            dirPath = os.path.dirname(filePath)
+            while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
+                if self.pathTracker.get(dirPath) is None:
+                    self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
+                    self.platformUtility.changePathGroupOwner(dirPath, experimentName)
+                    ownerUpdateTime = time.time()
+                    self.pathTracker.put(dirPath, ownerUpdateTime)
+                else: 
+                    self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
+                dirPath = os.path.dirname(dirPath)
+
+        self.logger.debug('Processing file path %s (fileInfo: %s)' % (filePath, fileInfo))
+        self.fileProcessingManager.processFile(fileInfo)
 
     def statExperimentFile(self, experimentFilePath, experiment, fileInfo={}):
         experimentName = experiment.get('name')
@@ -220,7 +223,6 @@ class ExperimentManager(Singleton):
             self.logger.debug('File path %s does not exist' % filePath)
             raise ObjectNotFound('File %s does not exist' % filePath)
 
-    @ThreadingUtility.synchronize
     def processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
         experimentName = experiment.get('name')
         self.updateExperimentWithStorageDataDirectory(experiment)
@@ -228,26 +230,46 @@ class ExperimentManager(Singleton):
         directoryPath = os.path.join(storageDirectory, experimentDirectoryPath)
         directoryInfo['directoryPath'] = directoryPath
         directoryInfo['experiment'] = experiment
-        if os.path.exists(directoryPath):
-            self.logger.debug('Processing directory path %s (directoryInfo: %s)' % (directoryPath, directoryInfo))
-            if self.manageStoragePermissions:
-                self.logger.debug('Modifying permissions for directory %s to %s' % (directoryPath, self.DIR_PERMISSIONS_MODE))
-                OsUtility.chmodPath(directoryPath, dirMode=self.DIR_PERMISSIONS_MODE)
-                self.logger.debug('Changing group owner for %s to %s' % (directoryPath, experimentName))
-                self.platformUtility.recursivelyChangePathGroupOwner(directoryPath, experimentName)
-                # Recursively modify subdirectory permissions
-                dirPath = os.path.dirname(directoryPath)
-                while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
-                    if self.pathTracker.get(dirPath) is None:
-                        self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
-                        self.platformUtility.changePathGroupOwner(dirPath, experimentName)
-                        ownerUpdateTime = time.time()
-                        self.pathTracker.put(dirPath, ownerUpdateTime)
-                    else: 
-                        self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
-                    dirPath = os.path.dirname(dirPath)
-        else:
-            self.logger.debug('Directory path %s does not exist' % directoryPath)
+
+        if not self.manageStoragePermissions:
+            self.logger.error('Skipping permission management for directory path %s' % directoryPath)
+            return
+
+        self.logger.debug('Processing directory path %s in background' % (directoryPath))
+        timer = threading.Timer(self.DIRECTORY_PROCESSING_DELAY_IN_SECONDS, self.__processExperimentDirectory, args=[experimentDirectoryPath, experiment, directoryInfo])
+        timer.start()
+
+    def __processExperimentDirectory(self, experimentDirectoryPath, experiment, directoryInfo={}):
+        experimentName = experiment.get('name')
+        storageDirectory = experiment.get('storageDirectory')
+        directoryPath = directoryInfo.get('directoryPath')
+
+        if not os.path.exists(directoryPath):
+            self.logger.error('Directory path %s does not exist' % directoryPath)
+            return
+
+        # Modify ownership
+        self.logger.debug('Processing directory path %s (directoryInfo: %s)' % (directoryPath, directoryInfo))
+        self.logger.debug('Modifying permissions for directory %s to %s' % (directoryPath, self.DIR_PERMISSIONS_MODE))
+        OsUtility.chmodPath(directoryPath, dirMode=self.DIR_PERMISSIONS_MODE)
+        self.logger.debug('Changing group owner for %s to %s' % (directoryPath, experimentName))
+        self.platformUtility.recursivelyChangePathGroupOwner(directoryPath, experimentName)
+
+        # Recursively modify subdirectory permissions
+        dirPath = os.path.dirname(directoryPath)
+        while (os.path.abspath(dirPath) != os.path.abspath(storageDirectory)):
+            if self.pathTracker.get(dirPath) is None:
+                self.logger.debug('Changing group owner for experiment subdirectory %s to %s' % (dirPath, experimentName))
+                self.platformUtility.changePathGroupOwner(dirPath, experimentName)
+                ownerUpdateTime = time.time()
+                self.pathTracker.put(dirPath, ownerUpdateTime)
+            else: 
+                self.logger.debug('Group owner for experiment subdirectory %s is already set to %s' % (dirPath, experimentName))
+            dirPath = os.path.dirname(dirPath)
+
+        # Update file permissions
+        self.logger.debug('Changing permissions for all files in %s for experiment %s' % (directoryPath, experimentName))
+        self.platformUtility.chmodPathForFilesInDirectory(directoryPath, self.FILE_PERMISSIONS_MODE_STRING)
 
     @ThreadingUtility.synchronize
     def start(self):