updateUsersFromApsDbCli.py

#!/usr/bin/env python

import datetime
import time
import os
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.exceptions.invalidArgument import InvalidArgument
from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
from dm.common.cli.dmCli import DmCli
from dm.common.db.api.userDbApi import UserDbApi 
from dm.common.utility.configurationManager import ConfigurationManager
from dm.common.utility.loggingManager import LoggingManager
from dm.common.utility.ldapUserManager import LdapUserManager
from dm.aps_user_db.api.apsUserDbApi import ApsUserDbApi
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory

class UpdateUsersFromApsDbCli(DmCli):
    def __init__(self):
        DmCli.__init__(self)
        self.addOption('', '--config-file', dest='configFile', help='Utility configuration file.')
        self.addOption('', '--without-dm-db', dest='withoutDmDb', action='store_true', default=False, help='Do not use DM DB for synchronizing user information.')
        self.addOption('', '--use-dm-rest-api', dest='useDmRestApi', action='store_true', default=False, help='Use DM REST API for accessing user information. This option does not have any effect if --without-dm-db is set.')
        self.addOption('', '--quiet', dest='quiet', action='store_true', default=False, help='Do not print any output to console.')

    def checkArgs(self):
        configFile = self.options.configFile
        if configFile is None:
            raise InvalidRequest('Config file must be provided.')
        if not os.path.exists(configFile):
            raise InvalidArgument('Invalid config file: %s.' % configFile)

    def runCommand(self):
        self.parseArgs(usage="""
    dm-update-users-from-aps-db --config-file=CONFIGFILE
        [--without-dm-db]
        [--use-dm-rest-api]
        [--quiet]

Description:
    Updates all users in DM LDAP and database using information from APS DB. This command by default uses DM DB to determine whether or not user needs to be created or updated. If the --without-dm-db flag is set, DM DB will not be used nor it will be updated.
        """)
        startTime = time.time()
        ConfigurationManager.getInstance().setConfigFile(self.options.configFile)
        LoggingManager.getInstance().configure()
        self.checkArgs()

        self.logger.debug('Starting sync process')
        apsUserDbApi = ApsUserDbApi()
        ldapUserManager = LdapUserManager.createInstance()

        # Get DM user API
        dmUserApi = None
        if not self.options.withoutDmDb:
            try:
                if self.options.useDmRestApi:
                    self.logger.debug('Using DM REST API')
                    dmUserApi = DsRestApiFactory.getUserRestApi()
                else:
                    self.logger.debug('Using DM DB API')
                    dmUserApi = UserDbApi()
            except Exception, ex:
                self.logger.error('Cannot use DM DB: %s' % ex)

        self.logger.debug('Retrieving APS users')
        apsUsers = apsUserDbApi.getApsUsers()
        self.logger.debug('Number of APS Users: %s' % len(apsUsers))

        # Retrieve DM users from DB if we got DM user API
        dmUserMap = {}
        if dmUserApi:
            self.logger.debug('Retrieving DM users from DB')
            try:
                nDmUsersWithBadge = 0
                dmUsers = dmUserApi.getUsers()
                self.logger.debug('Number of DM Users: %s' % len(dmUsers))
                for dmUser in dmUsers:
                    username = dmUser.get('username')
                    if dmUser.get('badge'):
                        nDmUsersWithBadge += 1
                    dmUserMap[username] = dmUser
                self.logger.debug('Number of DM Users with badge: %s' % nDmUsersWithBadge)
            except Exception, ex:
                self.logger.error('Error retrieving users from DM DB: %s' % ex)
                dmUserApi = None

        # We cannot use DM user API, retrieve users from LDAP
        if not dmUserApi:
            self.logger.debug('Retrieving DM users using LDAP client')
            dmUserMap = ldapUserManager.getUserInfoMapByUid()
            self.logger.debug('Number of DM LDAP Users: %s' % len(dmUserMap))

        nCreatedUsers = 0
        nUpdatedUsers = 0
        nErrors = 0
        for apsUser in apsUsers:
            apsLastChangeDate = apsUser.get('lastChangeDate')

            # DM DB attributes
            badge = apsUser['badgeNumber']
            username = 'd%s' % badge

            firstName = apsUser.get('firstName')
            middleName = apsUser.get('middleName')
            lastName = apsUser.get('lastName')
            email = apsUser.get('email')
            isLocalUser = 0
            lastUpdate = str(apsLastChangeDate)
            globusUsername = None
            description = None
            password = None

            # LDAP attributes
            ldapModAttrDict = {}
            ldapModAttrDict['gecos'] = apsUser['name']
            ldapModAttrDict['userPassword'] = apsUser['passwordHashValue']

            dmUser = dmUserMap.get(username)
            if not dmUser:
                # We must create user in LDAP/DM DB
                try:
                    try:
                        self.logger.debug('Creating LDAP user with badge %s: %s' % (badge, apsUser['name']))
                        ldapUser = ldapUserManager.createUserInfo(username, apsUser)
                    except ObjectAlreadyExists, ex:
                        # LDAP user already exists, simply update it.
                        self.logger.debug('LDAP user with badge %s already exists, attempting to modify it' % (badge))
                        ldapUserManager.modifyUserInfo(username, ldapModAttrDict)
                except Exception, ex:
                    self.logger.error('LDAP problem caught while creating user with badge %s: %s' % (badge, str(ex)))
                    nErrors += 1
                    continue

                if dmUserApi:
                    try:
                        self.logger.debug('Creating DM DB user with badge %s: %s' % (badge, apsUser['name']))
                        dmUserApi.addUser(username, firstName, lastName, middleName, email, badge, globusUsername, description, password, isLocalUser, lastUpdate)
                        nCreatedUsers += 1
                    except Exception, ex:
                        self.logger.error('DM DB problem caught while creating user with badge %s: %s' % (badge, str(ex)))
                        nErrors += 1
                        continue
            else:
                # User already exists, simply update it.

                if dmUserApi:
                    # We have DM DB API
                    dmLastUpdate = str(dmUser.get('lastUpdate'))
                    if dmLastUpdate:
                        # need to convert DM last update time to datetime object
                        # datetime cannot handle time zone at the moment
                        timeZonePos = dmLastUpdate.rfind('-')
                        dmLastUpdate = dmLastUpdate[0:timeZonePos]
                        # remove microseconds from the timestamp
                        secondPos = dmLastUpdate.rfind('.')
                        if secondPos > 0:
                            dmLastUpdate = dmLastUpdate[0:secondPos]
                        dmLastUpdate = datetime.datetime.strptime(dmLastUpdate, '%Y-%m-%d %H:%M:%S')
                    if not dmLastUpdate or dmLastUpdate < apsLastChangeDate:
                        # User needs update
                        try:
                            self.logger.debug('Modifying LDAP user with badge %s, username %s' % (badge, username))
                            ldapUserManager.modifyUserInfo(username, ldapModAttrDict)
                        except Exception, ex:
                            self.logger.error('LDAP problem caught while modifying user with badge %s: %s' % (badge, str(ex)))
                            nErrors += 1
                            continue

                        try:
                            id = dmUser.get('id')
                            self.logger.debug('Modifying DM DB user with badge %s: %s (DM id: %s)' % (badge, apsUser['name'], id))
                            dmUserApi.updateUser(id, username, firstName, lastName, middleName, email, badge, globusUsername, description, password, isLocalUser, lastUpdate)
                            nUpdatedUsers += 1
                        except Exception, ex:
                            self.logger.error('DM DB problem caught while modifying user with badge %s: %s' % (badge, str(ex)))
                            nErrors += 1
                            continue
                    else:
                        # User is up to date
                        continue
                else:
                    # No DM DB API
                    apsPasswordHash = apsUser['passwordHashValue']
                    dmPasswordHash = ldapUserManager.decodePasswordHash(dmUser['userAttrs']['userPassword'][0])
                    if apsPasswordHash != dmPasswordHash:
                        # User needs update
                        try:
                            self.logger.debug('Modifying LDAP user with badge %s, username %s' % (badge, username))
                            #self.logger.debug('APS password hash: %s, DM password hash: %s' % (apsPasswordHash, dmPasswordHash))
                            #self.logger.debug('Encoded DM password hash: %s' % (dmUser['userAttrs']['userPassword'][0]))
                            ldapUserManager.modifyUserInfo(username, ldapModAttrDict)
                            nUpdatedUsers += 1
                        except Exception, ex:
                            self.logger.error('LDAP problem caught while modifying user with badge %s: %s' % (badge, str(ex)))
                            nErrors += 1
                            continue
                    else:       
                        # User is up to date
                        continue

        endTime = time.time()
        runTime = endTime - startTime

        self.logger.debug('Number of new DM users: %s' % (nCreatedUsers))
        self.logger.debug('Number of updated DM users: %s' % (nUpdatedUsers))
        self.logger.debug('Number of update errors: %s' % (nErrors))
        self.logger.debug('Completed sync process in %.3f seconds' % runTime)

        if not self.options.quiet:
            print('Number of new DM users: %s' % (nCreatedUsers))
            print('Number of updated DM users: %s' % (nUpdatedUsers))
            print('Number of update errors: %s' % (nErrors))
            print('Sync process runtime: %.3f seconds' % (runTime))

#######################################################################
# Run command.
if __name__ == '__main__':
    cli = UpdateUsersFromApsDbCli()
    cli.run()