resolved nscd cache issue

doc/RELEASE_NOTES.txt

@@ -3,8 +3,9 @@ Release 0.8 (01/26/2016)
 
 - Enhanced upload/daq performance and functionality (hidden files are not
   processed; for uploads system can detect files that had been processed 
-  already; added handling and reporting for processing errors)
+  already; improved handling and reporting of processing errors)
 - Source file checksum is calculated for rsync/gridftp plugins by default
+- Resolved globus online user authorization delay issue 
 
 Release 0.7 (12/08/2015)
 =============================

src/python/dm/__init__.py

-__version__ = "0.8 (2016.01.26)"
+__version__ = "0.8 (2016.02.05)"

src/python/dm/common/service/dmRestWebServiceBase.py

@@ -21,7 +21,7 @@ from dm.common.exceptions.configurationError import ConfigurationError
 
 class DmRestWebServiceBase:
 
-    DEFAULT_N_SERVER_REQUEST_THREADS = 10
+    DEFAULT_N_SERVER_REQUEST_THREADS = 50
     DEFAULT_SERVER_SOCKET_TIMEOUT = 30
     CONFIG_SECTION_NAME = 'WebService'
     CONFIG_OPTION_NAME_LIST = [ 'serviceHost', 'servicePort', 

src/python/dm/common/utility/ldapLinuxPlatformUtility.py

@@ -4,8 +4,10 @@ import grp
 import ldap
 import ldap.modlist
 import copy
+import threading
 
 from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
 from dm.common.exceptions.configurationError import ConfigurationError
 from dm.common.exceptions.internalError import InternalError
 from dm.common.exceptions.authenticationError import AuthenticationError
@@ -22,11 +24,17 @@ class LdapLinuxPlatformUtility:
     GPASSWD_CMD = '/usr/bin/gpasswd'
     NSCD_CMD = '/usr/sbin/nscd'
 
+    CONFIG_SECTION_NAME = 'LdapLinuxPlatformUtility'
+    REFRESH_AUTH_FILES_COMMAND_KEY = 'refreshauthfilescommand'
+    TIMER_DELAY_IN_SECONDS = 10
+
     def __init__(self, serverUrl, adminDn, adminPasswordFile, groupDnFormat, minGidNumber=None):
         self.serverUrl = serverUrl
         self.adminDn = adminDn
         self.groupDnFormat = groupDnFormat
         self.minGidNumber = minGidNumber
+        self.refreshAuthFilesCommand = None
+        self.refreshAuthFilesTimer = None
         self.getLogger().debug('Min GID number: %s' % minGidNumber)
         # Remove first entry from the dn format to get tree base
         self.groupBaseDn = ','.join(groupDnFormat.split(',')[1:])
@@ -36,6 +44,14 @@ class LdapLinuxPlatformUtility:
         if not self.adminPassword:
             raise ConfigurationError('LDAP password could not be found in %s file' % adminPasswordFile) 
         self.ldapClient = None
+        self.__configure()
+
+    def __configure(self):
+        cm = ConfigurationManager.getInstance()
+        configItems = cm.getConfigItems(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME)
+        self.getLogger().debug('Got config items: %s' % configItems)
+        self.refreshAuthFilesCommand = cm.getConfigOption(LdapLinuxPlatformUtility.CONFIG_SECTION_NAME, LdapLinuxPlatformUtility.REFRESH_AUTH_FILES_COMMAND_KEY)
+        self.getLogger().debug('Refresh auth files command: %s' % self.refreshAuthFilesCommand)
 
     @classmethod
     def getLogger(cls):
@@ -47,6 +63,11 @@ class LdapLinuxPlatformUtility:
         p = DmSubprocess('%s %s' % (cls.SUDO_CMD, cmd))
         p.run()
 
+    @classmethod
+    def executeCommand(cls, cmd):
+        p = DmSubprocess('%s' % (cmd))
+        p.run()
+
     def getLdapClient(self):
         if self.ldapClient is not None:
             try:
@@ -167,8 +188,8 @@ class LdapLinuxPlatformUtility:
             logger.error('Could not add user %s to group %s: %s' % (username, groupName, ex))
             raise InternalError(exception=ex)
 
-        # Refresh NSCD cache
-        self.refreshNscdGroupCache()
+        # Refresh auth files
+        self.refreshAuthFiles()
 
     def deleteUserFromGroup(self, username, groupName):
         """ Remove user from group. """
@@ -199,9 +220,8 @@ class LdapLinuxPlatformUtility:
             logger.error('Could not remove user %s from group %s: %s' % (username, groupName, ex))
             raise InternalError(exception=ex)
 
-        # Refresh NSCD cache
-        self.refreshNscdGroupCache()
-
+        # Refresh auth files
+        self.refreshAuthFiles()
 
     @classmethod
     def createLocalGroup(cls, name):
@@ -265,8 +285,8 @@ class LdapLinuxPlatformUtility:
             logger.error('Could not set users %s for group %s: %s' % (usernameList, groupName, ex))
             raise InternalError(exception=ex)
 
-        # Refresh NSCD cache
-        self.refreshNscdGroupCache()
+        # Refresh auth files 
+        self.refreshAuthFiles()
 
     @classmethod
     def setPathReadExecutePermissionsForGroup(cls, path, groupName):
@@ -293,15 +313,31 @@ class LdapLinuxPlatformUtility:
         except Exception, ex:
             logger.warn('Failed to refresh NCSD group cache: %s' % (str(ex)))
 
+    # Refresh auth files in a timer, to avoid running command too often
+    def refreshAuthFiles(self):
+        if not self.refreshAuthFilesCommand:
+            return
+        if self.refreshAuthFilesTimer and self.refreshAuthFilesTimer.is_alive():
+            return
+        self.refreshAuthFilesTimer = threading.Timer(self.TIMER_DELAY_IN_SECONDS, self.__refreshAuthFiles)
+        self.refreshAuthFilesTimer.start()
+
+    def __refreshAuthFiles(self):
+        logger = self.getLogger()
+        try:
+            logger.debug('Refreshing auth files')
+            self.executeCommand(self.refreshAuthFilesCommand)
+        except Exception, ex:
+            logger.warn('Failed to refresh auth files: %s' % (str(ex)))
 
 #######################################################################
 # Testing.
 
 if __name__ == '__main__':
     utility = LdapLinuxPlatformUtility('ldaps://dmid-vm.xray.aps.anl.gov:636', 'uid=dmadmin,ou=People,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', '/tmp/ldapPassword', groupDnFormat='cn=%s,ou=DM,ou=Group,o=aps.anl.gov,dc=aps,dc=anl,dc=gov', minGidNumber=66000)
-    print utility.getGroupInfo(u'id8i-test02')
-    utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
-    print utility.getGroupInfo(u'id8i-test02')
+    print utility.getGroupInfo(u's1id-test03')
+    #utility.addLocalUserToGroup(u'sveseli', u'id8i-test02')
+    #print utility.getGroupInfo(u'id8i-test02')
     #utility.deleteLocalUserFromGroup(u'sveseli', u'id8i-test02')
     #print utility.getGroupInfo(u'id8i-test02')