fix for auth files refreshing on a remote machine

doc/RELEASE_NOTES.txt

@@ -3,7 +3,7 @@ Release 0.8 (01/26/2016)
 
 - Enhanced upload/daq performance and functionality (hidden files are not
   processed; for uploads system can detect files that had been processed 
-  already; added handling and reporting for processing errors)
+  already; improved handling and reporting of processing errors)
 - Source file checksum is calculated for rsync/gridftp plugins by default
 
 Release 0.7 (12/08/2015)

etc/ds-web-service.conf.template

@@ -18,6 +18,9 @@ manageStoragePermissions=True
 #platformUtility=dm.common.utility.linuxUtility.LinuxUtility()
 platformUtility=dm.common.utility.ldapLinuxPlatformUtility.LdapLinuxPlatformUtility('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=DM_SYSTEM_UNIX_ACCOUNT,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', 'DM_INSTALL_DIR/etc/DM_SYSTEM_UNIX_ACCOUNT.ldap.passwd', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
 
+[LdapLinuxPlatformUtility]
+refreshAuthFilesCommand=
+
 [DbManager]
 dbSchema=DM_DB_NAME
 dbUser=DM_DB_NAME

src/python/dm/common/utility/ldapLinuxPlatformUtility.py

@@ -4,8 +4,10 @@ import grp
 import ldap
 import ldap.modlist
 import copy
+import threading
 
 from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
 from dm.common.exceptions.configurationError import ConfigurationError
 from dm.common.exceptions.internalError import InternalError
 from dm.common.exceptions.authenticationError import AuthenticationError
@@ -22,11 +24,17 @@ class LdapLinuxPlatformUtility:
     GPASSWD_CMD = '/usr/bin/gpasswd'
     NSCD_CMD = '/usr/sbin/nscd'
 
+    CONFIG_SECTION_NAME = 'LdapLinuxPlatformUtility'
+    REFRESH_AUTH_FILES_COMMAND_KEY = 'refreshauthfilescommand'
+    TIMER_DELAY_IN_SECONDS = 10
+
     def __init__(self, serverUrl, adminDn, adminPasswordFile, groupDnFormat, minGidNumber=None):
         self.serverUrl = serverUrl
         self.adminDn = adminDn
         self.groupDnFormat = groupDnFormat
         self.minGidNumber = minGidNumber
+        self.refreshAuthFilesCommand = None
+        self.refreshAuthFilesTimer = None
         self.getLogger().debug('Min GID number: %s' % minGidNumber)
         # Remove first entry from the dn format to get tree base
         self.groupBaseDn = ','.join(groupDnFormat.split(',')[1:])
@@ -37,6 +45,13 @@ class LdapLinuxPlatformUtility:
             raise ConfigurationError('LDAP password could not be found in %s file' % adminPasswordFile) 
         self.ldapClient = None
 
+    def __configure(self):
+        cm = ConfigurationManager.getInstance()
+        configItems = cm.getConfigItems(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME)
+        self.logger.debug('Got config items: %s' % configItems)
+        self.refreshAuthFilesCommand = cm.getConfigOption(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME, LdapLinuxPlatformUtility.REFRESH_AUTH_FILES_COMMAND_KEY)
+        self.logger.debug('Refresh auth files command: %s' % self.refreshAuthFilesCommand)
+
     @classmethod
     def getLogger(cls):
         logger = LoggingManager.getInstance().getLogger(cls.__name__)
@@ -47,6 +62,11 @@ class LdapLinuxPlatformUtility:
         p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
         p.run()
 
+    @classmethod
+    def executeCommand(cls, cmd):
+        p = DmSubprocess('%s' % (cmd))
+        p.run()
+
     def getLdapClient(self):
         if self.ldapClient is not None:
             try:
@@ -167,8 +187,8 @@ class LdapLinuxPlatformUtility:
             logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
             raise InternalError(exception=ex)
 
-        # Refresh NSCD cache
-        self.refreshNscdGroupCache()
+        # Refresh auth files
+        self.refreshAuthFiles()
 
     def deleteUserFromGroup(self, username, groupName):
         """ Remove user from group. """
@@ -199,9 +219,8 @@ class LdapLinuxPlatformUtility:
             logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
             raise InternalError(exception=ex)
 
-        # Refresh NSCD cache
-        self.refreshNscdGroupCache()
-
+        # Refresh auth files
+        self.refreshAuthFiles()
 
     @classmethod
     def createLocalGroup(cls, name):
@@ -265,8 +284,8 @@ class LdapLinuxPlatformUtility:
             logger.error('Could not set users %s for group %s: %s' % (usernameList, groupName, ex))
             raise InternalError(exception=ex)
 
-        # Refresh NSCD cache
-        self.refreshNscdGroupCache()
+        # Refresh auth files 
+        self.refreshAuthFiles()
 
     @classmethod
     def setPathReadExecutePermissionsForGroup(cls, path, groupName):
@@ -293,6 +312,22 @@ class LdapLinuxPlatformUtility:
         except Exception, ex:
             logger.warn('Failed to refresh NCSD group cache: %s' % (str(ex)))
 
+    # Refresh auth files in a timer, to avoid running command too often
+    def refreshAuthFiles(self):
+        if not self.refreshAuthFilesCommand:
+            return
+        if self.refreshAuthFilesTimer and self.refreshAuthFilesTimer.is_alive():
+            return
+        self.refreshAuthFilesTimer = threading.Timer(self.TIMER_DELAY_IN_SECONDS, self.__refreshAuthFiles)
+        self.refreshAuthFilesTimer.start()
+
+    def __refreshAuthFiles(self):
+        logger = self.getLogger()
+        try:
+            logger.debug('Refreshing auth files')
+            self.executeCommand(self.refreshAuthFilesCommand:)
+        except Exception, ex:
+            logger.warn('Failed to refresh auth files: %s' % (str(ex)))
 
 #######################################################################
 # Testing.