src/python/dm/aps_user_db/cli/updateUserFromApsDbCli.py

+#!/usr/bin/env python
+
+import os
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.cli.dmCli import DmCli
+from dm.common.utility.ldapUserManager import LdapUserManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.utility.loggingManager import LoggingManager
+from dm.aps_user_db.api.apsUserDbApi import ApsUserDbApi
+
+class UpdateUserFromApsDbCli(DmCli):
+    def __init__(self):
+        DmCli.__init__(self)
+        self.addOption('', '--badge', dest='badge', help='User badge number.')
+        self.addOption('', '--config-file', dest='configFile', help='Utility configuration file.')
+
+    def checkArgs(self):
+        if self.options.badge is None:
+            raise InvalidRequest('Badge number must be provided.')
+        configFile = self.options.configFile
+        if configFile is None:
+            raise InvalidRequest('Config file must be provided.')
+        if not os.path.exists(configFile):
+            raise InvalidArgument('Invalid config file: %s.' % configFile)
+
+    def getBadge(self):
+        return self.options.badge
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-update-user-from-aps-db --badge=BADGE --config-file=CONFIGFILE
+
+Description:
+    Updates single user in DM LDAP using information from APS DB. This command will not update DM DB.
+        """)
+        ConfigurationManager.getInstance().setConfigFile(self.options.configFile)
+        LoggingManager.getInstance().configure()
+        self.checkArgs()
+        apsUserDbApi = ApsUserDbApi()
+        badge = self.getBadge()
+        username = 'd%s' % badge
+        self.logger.debug('Retrieving APS user with badge %s' % badge)
+        apsUser = apsUserDbApi.getApsUserByBadgeNumber(badge)
+        #self.logger.info('APS User Info: %s' % apsUser)
+        print('APS User Info: %s\n' % apsUser)
+        ldapUserManager = LdapUserManager.createInstance()
+        try:
+            self.logger.debug('Modifying LDAP user with badge %s: %s' % (badge, apsUser['name']))
+            ldapUser = ldapUserManager.getUserInfo(username)
+            #self.logger.info('LDAP User Info (before update): %s' % ldapUser)
+            #self.logger.info('LDAP User Password Hash (before update): %s\n' % ldapUserManager.decodePasswordHash(ldapUser.getUserPassword()))
+            print('LDAP User Info (before update): %s' % ldapUser)
+            print('LDAP User Password Hash (before update): %s\n' % ldapUserManager.decodePasswordHash(ldapUser.getUserPassword()))
+
+            ldapUserManager.modifyUserInfo(username, apsUser)
+            ldapUser = ldapUserManager.getUserInfo(username)
+        except ObjectNotFound, ex:
+            self.logger.debug('User with badge %s does not exist, creating it' % (badge))
+            ldapUser = ldapUserManager.createUserInfo(username, apsUser)
+        #self.logger.info('LDAP User Info (after update): %s' % ldapUser)
+        #self.logger.info('LDAP User Password Hash (after update): %s\n' % ldapUserManager.decodePasswordHash(ldapUser.getUserPassword()))
+        print('LDAP User Info (after update): %s' % ldapUser)
+        print('LDAP User Password Hash (after update): %s\n' % ldapUserManager.decodePasswordHash(ldapUser.getUserPassword()))
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = UpdateUserFromApsDbCli()
+    cli.run()

src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py

+#!/usr/bin/env python
+
+import datetime
+import time
+import os
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.exceptions.invalidArgument import InvalidArgument
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.cli.dmCli import DmCli
+from dm.common.db.api.userDbApi import UserDbApi 
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.ldapUserManager import LdapUserManager
+from dm.aps_user_db.api.apsUserDbApi import ApsUserDbApi
+from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
+
+class UpdateUsersFromApsDbCli(DmCli):
+    def __init__(self):
+        DmCli.__init__(self)
+        self.addOption('', '--config-file', dest='configFile', help='Utility configuration file.')
+        self.addOption('', '--without-dm-db', dest='withoutDmDb', action='store_true', default=False, help='Do not use DM DB for synchronizing user information.')
+        self.addOption('', '--use-dm-rest-api', dest='useDmRestApi', action='store_true', default=False, help='Use DM REST API for accessing user information. This option does not have any effect if --without-dm-db is set.')
+        self.addOption('', '--quiet', dest='quiet', action='store_true', default=False, help='Do not print any output to console.')
+
+    def checkArgs(self):
+        configFile = self.options.configFile
+        if configFile is None:
+            raise InvalidRequest('Config file must be provided.')
+        if not os.path.exists(configFile):
+            raise InvalidArgument('Invalid config file: %s.' % configFile)
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-update-users-from-aps-db --config-file=CONFIGFILE
+        [--without-dm-db]
+        [--use-dm-rest-api]
+        [--quiet]
+
+Description:
+    Updates all users in DM LDAP and database using information from APS DB. This command by default uses DM DB to determine whether or not user needs to be created or updated. If the --without-dm-db flag is set, DM DB will not be used nor it will be updated.
+        """)
+        startTime = time.time()
+        ConfigurationManager.getInstance().setConfigFile(self.options.configFile)
+        LoggingManager.getInstance().configure()
+        self.checkArgs()
+
+        self.logger.debug('Starting sync process')
+        apsUserDbApi = ApsUserDbApi()
+        ldapUserManager = LdapUserManager.createInstance()
+
+        # Get DM user API
+        dmUserApi = None
+        if not self.options.withoutDmDb:
+            try:
+                if self.options.useDmRestApi:
+                    self.logger.debug('Using DM REST API')
+                    dmUserApi = DsRestApiFactory.getUserDsApi()
+                else:
+                    self.logger.debug('Using DM DB API')
+                    dmUserApi = UserDbApi()
+            except Exception, ex:
+                self.logger.error('Cannot use DM DB: %s' % ex)
+
+        self.logger.debug('Retrieving APS users')
+        apsUsers = apsUserDbApi.getApsUsers()
+        self.logger.debug('Number of APS Users: %s' % len(apsUsers))
+
+        # Retrieve DM users from DB if we got DM user API
+        dmUserMap = {}
+        if dmUserApi:
+            self.logger.debug('Retrieving DM users from DB')
+            try:
+                nDmUsersWithBadge = 0
+                dmUsers = dmUserApi.getUsers()
+                self.logger.debug('Number of DM Users: %s' % len(dmUsers))
+                for dmUser in dmUsers:
+                    username = dmUser.get('username')
+                    if dmUser.get('badge'):
+                        nDmUsersWithBadge += 1
+                    dmUserMap[username] = dmUser
+                self.logger.debug('Number of DM Users with badge: %s' % nDmUsersWithBadge)
+            except Exception, ex:
+                self.logger.error('Error retrieving users from DM DB: %s' % ex)
+                dmUserApi = None
+
+        # We cannot use DM user API, retrieve users from LDAP
+        if not dmUserApi:
+            self.logger.debug('Retrieving DM users using LDAP client')
+            dmUserMap = ldapUserManager.getUserInfoMapByUid()
+            self.logger.debug('Number of DM LDAP Users: %s' % len(dmUserMap))
+
+        nCreatedUsers = 0
+        nUpdatedUsers = 0
+        nErrors = 0
+        for apsUser in apsUsers:
+            apsLastChangeDate = apsUser.get('lastChangeDate')
+
+            # DM DB attributes
+            badge = apsUser['badgeNumber']
+            username = 'd%s' % badge
+
+            firstName = apsUser.get('firstName')
+            middleName = apsUser.get('middleName')
+            lastName = apsUser.get('lastName')
+            email = apsUser.get('email')
+            isLocalUser = 0
+            lastUpdate = str(apsLastChangeDate)
+            globusUsername = None
+            description = None
+            password = None
+
+            # LDAP attributes
+            ldapModAttrDict = {}
+            ldapModAttrDict['gecos'] = apsUser['name']
+            ldapModAttrDict['userPassword'] = apsUser['passwordHashValue']
+
+            dmUser = dmUserMap.get(username)
+            if not dmUser:
+                # We must create user in LDAP/DM DB
+                try:
+                    try:
+                        self.logger.debug('Creating LDAP user with badge %s: %s' % (badge, apsUser['name']))
+                        ldapUser = ldapUserManager.createUserInfo(username, apsUser)
+                    except ObjectAlreadyExists, ex:
+                        # LDAP user already exists, simply update it.
+                        self.logger.debug('LDAP user with badge %s already exists, attempting to modify it' % (badge))
+                        ldapUserManager.modifyUserInfo(username, ldapModAttrDict)
+                except Exception, ex:
+                    self.logger.error('LDAP problem caught while creating user with badge %s: %s' % (badge, str(ex)))
+                    nErrors += 1
+                    continue
+
+                if dmUserApi:
+                    try:
+                        self.logger.debug('Creating DM DB user with badge %s: %s' % (badge, apsUser['name']))
+                        dmUserApi.addUser(username, firstName, lastName, middleName, email, badge, globusUsername, description, password, isLocalUser, lastUpdate)
+                        nCreatedUsers += 1
+                    except Exception, ex:
+                        self.logger.error('DM DB problem caught while creating user with badge %s: %s' % (badge, str(ex)))
+                        nErrors += 1
+                        continue
+            else:
+                # User already exists, simply update it.
+
+                if dmUserApi:
+                    # We have DM DB API
+                    dmLastUpdate = str(dmUser.get('lastUpdate'))
+                    if dmLastUpdate:
+                        # need to convert DM last update time to datetime object
+                        # datetime cannot handle time zone at the moment
+                        timeZonePos = dmLastUpdate.rfind('-')
+                        dmLastUpdate = dmLastUpdate[0:timeZonePos]
+                        # remove microseconds from the timestamp
+                        secondPos = dmLastUpdate.rfind('.')
+                        if secondPos > 0:
+                            dmLastUpdate = dmLastUpdate[0:secondPos]
+                        dmLastUpdate = datetime.datetime.strptime(dmLastUpdate, '%Y-%m-%d %H:%M:%S')
+                    if not dmLastUpdate or dmLastUpdate < apsLastChangeDate:
+                        # User needs update
+                        try:
+                            self.logger.debug('Modifying LDAP user with badge %s, username %s' % (badge, username))
+                            ldapUserManager.modifyUserInfo(username, ldapModAttrDict)
+                        except Exception, ex:
+                            self.logger.error('LDAP problem caught while modifying user with badge %s: %s' % (badge, str(ex)))
+                            nErrors += 1
+                            continue
+
+                        try:
+                            id = dmUser.get('id')
+                            self.logger.debug('Modifying DM DB user with badge %s: %s (DM id: %s)' % (badge, apsUser['name'], id))
+                            dmUserApi.updateUser(id, username, firstName, lastName, middleName, email, badge, globusUsername, description, password, isLocalUser, lastUpdate)
+                            nUpdatedUsers += 1
+                        except Exception, ex:
+                            self.logger.error('DM DB problem caught while modifying user with badge %s: %s' % (badge, str(ex)))
+                            nErrors += 1
+                            continue
+                    else:
+                        # User is up to date
+                        continue
+                else:
+                    # No DM DB API
+                    apsPasswordHash = apsUser['passwordHashValue']
+                    dmPasswordHash = ldapUserManager.decodePasswordHash(dmUser['userAttrs']['userPassword'][0])
+                    if apsPasswordHash != dmPasswordHash:
+                        # User needs update
+                        try:
+                            self.logger.debug('Modifying LDAP user with badge %s, username %s' % (badge, username))
+                            #self.logger.debug('APS password hash: %s, DM password hash: %s' % (apsPasswordHash, dmPasswordHash))
+                            #self.logger.debug('Encoded DM password hash: %s' % (dmUser['userAttrs']['userPassword'][0]))
+                            ldapUserManager.modifyUserInfo(username, ldapModAttrDict)
+                            nUpdatedUsers += 1
+                        except Exception, ex:
+                            self.logger.error('LDAP problem caught while modifying user with badge %s: %s' % (badge, str(ex)))
+                            nErrors += 1
+                            continue
+                    else:       
+                        # User is up to date
+                        continue
+
+        endTime = time.time()
+        runTime = endTime - startTime
+
+        self.logger.debug('Number of new DM users: %s' % (nCreatedUsers))
+        self.logger.debug('Number of updated DM users: %s' % (nUpdatedUsers))
+        self.logger.debug('Number of update errors: %s' % (nErrors))
+        self.logger.debug('Completed sync process in %.3f seconds' % runTime)
+
+        if not self.options.quiet:
+            print('Number of new DM users: %s' % (nCreatedUsers))
+            print('Number of updated DM users: %s' % (nUpdatedUsers))
+            print('Number of update errors: %s' % (nErrors))
+            print('Sync process runtime: %.3f seconds' % (runTime))
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = UpdateUsersFromApsDbCli()
+    cli.run()

src/python/dm/aps_user_db/entities/apsUserDbEntityMap.py

+#!/usr/bin/env python
+
+from dm.aps_user_db.entities.apsUserInfo import ApsUserInfo
+
+# Map db table/db entity class
+# Use generic 'self' key as name for primary key mapping (needed by sqlalchemy)
+APS_USER_DB_ENTITY_MAP = {
+    'FL$03_BL_APV_VIEW_V2' : (ApsUserInfo, {'self' : {'primary_key' : 'badge_no'}}), 
+}
+

src/python/dm/aps_user_db/entities/apsUserInfo.py

+#!/usr/bin/env python
+
+from dm.common.db.entities.dmDbEntity import DmDbEntity
+from dm.common.objects import apsUserInfo
+
+class ApsUserInfo(DmDbEntity):
+
+    mappedColumnDict = { 
+        'badge_no' : 'badgeNumber',
+        'first_name' : 'firstName',
+        'middle_name' : 'middleName',
+        'last_name' : 'lastName',
+        'last_change_date' : 'lastChangeDate',
+        'pwd_hash_value' : 'passwordHashValue',
+        'is_usr_not_anl_emp' : 'isUserNotAnlEmployee',
+    }
+    dmObjectClass = apsUserInfo.ApsUserInfo
+
+    def __init__(self, **kwargs):
+        DmDbEntity.__init__(self, **kwargs)
+
+
+

src/python/dm/aps_user_db/impl/apsUserDbManager.py

+#!/usr/bin/env python
+
+import threading
+import os.path
+import sqlalchemy 
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import mapper
+from sqlalchemy.orm import relationship
+
+from dm.common.exceptions.commandFailed import CommandFailed
+from dm.common.exceptions.configurationError import ConfigurationError
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+from dm.aps_user_db.entities import apsUserDbEntityMap
+
+
+class ApsUserDbManager:
+    """ Singleton class for db management. """
+
+    DB_CONNECTION_POOL_SIZE = 10
+    DB_CONNECTION_POOL_MAX_OVERFLOW = 2
+    DB_CONNECTION_POOL_RECYCYLE_TIME = 600
+    DB_CONNECTION_POOL_TIMEOUT = 60
+    DB_CONNECTION_LOGGING_FLAG = False
+
+    CONFIG_SECTION_NAME = 'ApsUserDbManager'
+    DB_KEY = 'db'
+    DB_USER_KEY = 'dbuser'
+    DB_HOST_KEY = 'dbhost'
+    DB_PORT_KEY = 'dbport'
+    DB_NAME_KEY = 'dbname'
+    DB_SCHEMA_KEY = 'dbschema'
+    DB_PASSWORD_FILE_KEY = 'dbpasswordfile'
+
+    # Singleton.
+    __lock = threading.RLock()
+    __instance = None
+
+    @classmethod
+    def getInstance(cls):
+        from dm.aps_user_db.impl.apsUserDbManager import ApsUserDbManager
+        try:
+            mgr = ApsUserDbManager()
+        except ApsUserDbManager, ex:
+            mgr = ex
+        return mgr 
+
+    def __configure(self):
+        cm = ConfigurationManager.getInstance()
+        self.logger.debug('Configuring APS User DB Manager')
+        self.db = cm.getConfigOption(ApsUserDbManager.CONFIG_SECTION_NAME, ApsUserDbManager.DB_KEY)
+        #self.logger.debug('Using APS DB: %s' % self.db)
+        self.dbUser = cm.getConfigOption(ApsUserDbManager.CONFIG_SECTION_NAME, ApsUserDbManager.DB_USER_KEY)
+        #self.logger.debug('Using APS DB user: %s' % self.dbUser)
+        self.dbHost = cm.getConfigOption(ApsUserDbManager.CONFIG_SECTION_NAME, ApsUserDbManager.DB_HOST_KEY)
+        #self.logger.debug('Using APS DB host: %s' % self.dbHost)
+        self.dbPort = cm.getConfigOption(ApsUserDbManager.CONFIG_SECTION_NAME, ApsUserDbManager.DB_PORT_KEY)
+        #self.logger.debug('Using APS DB port: %s' % self.dbPort)
+        self.dbName = cm.getConfigOption(ApsUserDbManager.CONFIG_SECTION_NAME, ApsUserDbManager.DB_NAME_KEY)
+        #self.logger.debug('Using APS DB name: %s' % self.dbName)
+        self.dbSchema = cm.getConfigOption(ApsUserDbManager.CONFIG_SECTION_NAME, ApsUserDbManager.DB_SCHEMA_KEY)
+        #self.logger.debug('Using APS DB schema: %s' % self.dbSchema)
+        self.dbPasswordFile = cm.getConfigOption(ApsUserDbManager.CONFIG_SECTION_NAME, ApsUserDbManager.DB_PASSWORD_FILE_KEY)
+        #self.logger.debug('Using APS DB password file: %s' % self.dbPasswordFile)
+        self.dbPassword = open(self.dbPasswordFile, 'r').readline().strip()
+
+    def __init__(self):
+        ApsUserDbManager.__lock.acquire()
+        try:
+            if ApsUserDbManager.__instance is not None:
+                raise ApsUserDbManager.__instance
+            ApsUserDbManager.__instance = self
+            self.lock = threading.RLock()
+            self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
+            self.__configure()
+
+            engineUrl = '%s://%s:%s@%s:%s/%s' % (self.db, self.dbUser, self.dbPassword, self.dbHost, self.dbPort, self.dbName)
+            #self.logger.debug('Using engine URL: %s' % engineUrl)
+            self.engine = sqlalchemy.create_engine(engineUrl, 
+                pool_size=ApsUserDbManager.DB_CONNECTION_POOL_SIZE, 
+                max_overflow=ApsUserDbManager.DB_CONNECTION_POOL_MAX_OVERFLOW, 
+                pool_recycle=ApsUserDbManager.DB_CONNECTION_POOL_RECYCYLE_TIME, 
+                echo=ApsUserDbManager.DB_CONNECTION_LOGGING_FLAG, 
+                pool_timeout=ApsUserDbManager.DB_CONNECTION_POOL_TIMEOUT)
+            self.metadata = sqlalchemy.MetaData(engineUrl, schema=self.dbSchema)
+
+            self.logger.debug('Mapping DB tables')
+            for (dbTableName, (dbEntityClass, dbRelationDict)) in apsUserDbEntityMap.APS_USER_DB_ENTITY_MAP.items():
+                self.mapTable(dbEntityClass, dbTableName, dbRelationDict)
+            self.logger.debug('Initialized SQLalchemy APS User DB engine')
+
+        finally:
+            ApsUserDbManager.__lock.release()
+
+    def getLogger(self):
+        return self.logger
+
+    def inspectTables(self):
+        from sqlalchemy import inspect
+        inspector = inspect(self.engine)
+        self.logger.debug('Inspecting tables')
+        for tableName in inspector.get_table_names():
+            self.logger.debug('Table: %s' % tableName)
+            for column in inspector.get_columns(tableName):
+                self.logger.debug('Column: %s' % column['name'])
+
+    def inspectTables2(self):
+        from sqlalchemy import MetaData
+        m = MetaData()
+        m.reflect(self.engine)
+        self.logger.debug('Inspecting tables via metadata')
+        for table in m.tables.values():
+            self.logger.debug('Table: %s' % table.name)
+            for column in table.c:
+                self.logger.debug('Column: %s' % column.name)
+
+    def initTable(self, tableClass, tableName):
+        """ Initialize DB table. """
+        self.lock.acquire()
+        try:
+            tbl = sqlalchemy.Table(tableName, self.metadata, autoload=True)
+            tableClass.columns = tbl.columns
+            return tbl
+        finally:
+            self.lock.release()
+
+    def mapTable(self, tableClass, tableName, relationDict):
+        """ Map DB table to a given class. """
+        self.lock.acquire()
+        try:
+            table = sqlalchemy.Table(tableName, self.metadata, autoload=True)
+            tableClass.columns = table.columns
+
+            # Build relations from specified foreign key columns and other properties.
+            tableRelations = {}
+            primaryKey = None
+            for (name, propertyDict) in relationDict.items():
+                if name == 'self':
+                    primaryKey = propertyDict.get('primary_key')
+                    continue
+                lazy = propertyDict.get('lazy')
+                parentEntity = propertyDict.get('parentEntity')
+                foreignKeyColumns = propertyDict.get('foreignKeyColumns', [])
+                if len(foreignKeyColumns):
+                    fkList = []
+                    for fk in foreignKeyColumns:
+                        fkList.append(table.columns.get(fk))
+                    tableRelations[name] = relationship(parentEntity, foreign_keys=fkList, lazy=lazy)
+                else:
+                    tableRelations[name] = relationship(parentEntity, lazy=lazy)
+
+            if primaryKey:
+                mapper(tableClass, table, tableRelations, primary_key=table.columns.get(primaryKey))
+            else:
+                mapper(tableClass, table, tableRelations)
+            return table
+        finally:
+            self.lock.release()
+
+    def getMetadataTable(self, table):
+        return self.metadata.tables[table]
+
+    def openSession(self):
+        """ Open db session. """
+        self.lock.acquire()
+        try:
+            Session = sessionmaker(bind=self.engine)
+            return Session()
+        finally:
+            self.lock.release()
+
+    def closeSession(self, session):
+        """ Close db session. """
+        self.lock.acquire()
+        try:
+            session.close()
+        finally:
+            self.lock.release()
+
+    def acquireConnection(self):
+        """ Get db connection. """
+        self.lock.acquire()
+        try:
+            return self.engine.connect()
+        finally:
+            self.lock.release()
+                                                                        
+    def releaseConnection(self, connection):
+        """ Release db connection. """
+        self.lock.acquire()
+        try:
+            if connection:
+                connection.close()
+        finally:
+            self.lock.release()
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    ConfigurationManager.getInstance().setConsoleLogLevel('debug')
+    mgr = ApsUserDbManager.getInstance()
+    mgr.acquireConnection()
+    mgr.inspectTables()
+    mgr.inspectTables2()
+    print 'Got connection'
+
+

src/python/dm/aps_user_db/impl/apsUserInfoHandler.py

+#!/usr/bin/env python
+
+from sqlalchemy import and_
+from sqlalchemy.orm.exc import NoResultFound
+
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.exceptions.objectNotFound import ObjectNotFound
+from dm.common.exceptions.objectAlreadyExists import ObjectAlreadyExists
+from dm.common.db.impl.dmDbEntityHandler import DmDbEntityHandler
+from dm.aps_user_db.entities.apsUserInfo import ApsUserInfo
+
+class ApsUserInfoHandler(DmDbEntityHandler):
+
+    def __init__(self):
+        DmDbEntityHandler.__init__(self)
+
+    def getApsUsers(self, session):
+        dbUserInfos = session.query(ApsUserInfo).all()
+        return dbUserInfos
+
+    def getApsUserByBadgeNumber(self, session, badgeNumber):
+        try:
+            dbUserInfo = session.query(ApsUserInfo).filter(ApsUserInfo.badge_no==badgeNumber).one()
+            return dbUserInfo
+        except NoResultFound, ex:
+            raise ObjectNotFound('APS user with badge number %s does not exist.' % (badgeNumber))
+

src/python/dm/cat_web_service/__init__.py

+__version__ = "Development Snapshot"

src/python/dm/cat_web_service/api/__init__.py

+__version__ = "Development Snapshot"

src/python/dm/cat_web_service/api/catRestApi.py

+#!/usr/bin/env python
+
+from dm.common.api.dmRestApi import DmRestApi
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class CatRestApi(DmRestApi):
+    """ Base CAT DM REST api class. """
+
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        if host == None:
+            host = ConfigurationManager.getInstance().getCatWebServiceHost()
+        if port == None:
+            port = ConfigurationManager.getInstance().getCatWebServicePort()
+        DmRestApi.__init__(self, username, password, host, port, protocol)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    pass
+
+

src/python/dm/cat_web_service/api/catRestApiFactory.py

+#!/usr/bin/env python
+
+from dm.common.utility.loggingManager import LoggingManager
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class CatRestApiFactory:
+
+    CONFIG_SECTION_NAME = 'CatRestApiFactory'
+    USERNAME_KEY = 'username'
+    PASSWORD_FILE_KEY = 'passwordfile'
+    HOST_KEY = 'host'
+    PORT_KEY = 'port'
+    PROTOCOL_KEY = 'protocol'
+
+    __logger = None
+    __username = None
+    __password = None
+    __host = None
+    __port = None
+    __protocol = None
+
+    @classmethod
+    def getLogger(cls):
+        if cls.__logger is None:
+            cls.__logger = LoggingManager.getInstance().getLogger(cls.__name__)
+        return cls.__logger
+
+    @classmethod
+    def __getConfiguration(cls):
+        if cls.__username is None:
+            cls.__username = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.USERNAME_KEY)
+            cls.__password = open(ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.PASSWORD_FILE_KEY)).read().strip()
+            cls.__host = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.HOST_KEY)
+            cls.__port = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.PORT_KEY)
+            cls.__protocol = ConfigurationManager.getInstance().getConfigOption(cls.CONFIG_SECTION_NAME, cls.PROTOCOL_KEY)
+        return (cls.__username, cls.__password, cls.__host, cls.__port, cls.__protocol)
+
+    @classmethod
+    def getFileCatApi(cls):
+        from userCatApi import FileCatApi
+        (username, password, host, port, protocol) = cls.__getConfiguration()
+        api = FileCatApi(username, password, host, port, protocol)
+        return api
+
+####################################################################
+# Testing
+
+if __name__ == '__main__':
+    pass
+

src/python/dm/cat_web_service/api/datasetCatApi.py

+#!/usr/bin/env python
+
+import os
+import urllib
+import json
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.datasetMetadata import DatasetMetadata
+from dm.common.objects.fileMetadata import FileMetadata
+from catRestApi import CatRestApi
+
+class DatasetCatApi(CatRestApi):
+    
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        CatRestApi.__init__(self, username, password, host, port, protocol)
+
+    @CatRestApi.execute
+    def addExperimentDataset(self, datasetInfo):
+        experimentName = datasetInfo.get('experimentName')
+        if not experimentName:
+            raise InvalidRequest('Dataset metadata must contain experimentName key.')
+        datasetName = datasetInfo.get('datasetName')
+        if not datasetName:
+            raise InvalidRequest('Dataset metadata must contain datasetName key.')
+        url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+        responseData = self.sendSessionRequest(url=url, method='POST')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def updateExperimentDataset(self, datasetInfo):
+        experimentName = datasetInfo.get('experimentName')
+        if not experimentName:
+            raise InvalidRequest('Dataset metadata must contain experimentName key.')
+        datasetName = datasetInfo.get('datasetName')
+        if not datasetName:
+            raise InvalidRequest('Dataset metadata must contain datasetName key.')
+        url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def updateDatasetById(self, datasetInfo):
+        id = datasetInfo.get('id')
+        if not id:
+            raise InvalidRequest('Dataset metadata must contain id key.')
+        url = '%s/datasets/%s' % (self.getContextRoot(), id)
+        url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def getDatasets(self, queryDict={}):
+        url = '%s/datasets' % (self.getContextRoot())
+        url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, DatasetMetadata)
+
+    @CatRestApi.execute
+    def getExperimentDatasets(self, experimentName, queryDict={}):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        url = '%s/datasetsByExperiment/%s' % (self.getContextRoot(), experimentName)
+        url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, DatasetMetadata)
+
+    @CatRestApi.execute
+    def getDatasetById(self, id):
+        if not id:
+            raise InvalidRequest('Invalid dataset id provided.')
+        url = '%s/datasets/%s' % (self.getContextRoot(), id)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def getExperimentDataset(self, experimentName, datasetName):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def getExperimentDatasetFiles(self, experimentName, datasetName):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        url = '%s/filesByExperimentDataset/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, FileMetadata)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = DatasetCatApi()
+    print api.getDatasets()
+    print api.getDatasetById('556de0059e058b0ef4c4413b')
+    print api.getDatasetByName('xyz-001')
+
+    import time
+    t = long(time.time())
+    datasetInfo = {
+        'datasetName' : 'sv-%s' % t,
+        'experimentName' : 'exp1',
+        'power' : 12,
+        'powerUnits' : 'kW',
+        'force' : 15,
+        'forceUnits' : 'N',
+        'cKey' : {'a' : 1, 'b' : 'B', 'c' : 2.2},
+    }
+    datasetMetadata = api.addDatasetByName(datasetInfo)
+
+    print '\nADDED DATASET:\n', datasetMetadata
+ 
+    datasetInfo['updateKey'] = 'here is desc'
+    datasetMetadata = api.updateDatasetByName(datasetInfo)
+    print '\nUPDATED DATASET:\n', datasetMetadata
+
+    datasetInfo['updateKey2'] = 'new desc'
+    datasetInfo['id'] = datasetMetadata.get('id')
+    datasetMetadata = api.updateDatasetById(datasetInfo)
+    print '\nUPDATED DATASET:\n', datasetMetadata
+
+    queryDict = { 'experimentName' : 'exp2' }
+    print '\nQUERY DATASETS:\n', api.getDatasets(queryDict=queryDict)
+

src/python/dm/cat_web_service/api/fileCatApi.py

+#!/usr/bin/env python
+
+import os
+import urllib
+import json
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.fileMetadata import FileMetadata
+from catRestApi import CatRestApi
+
+class FileCatApi(CatRestApi):
+    
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        CatRestApi.__init__(self, username, password, host, port, protocol)
+
+    @CatRestApi.execute
+    def addExperimentFile(self, fileInfo):
+        experimentName = fileInfo.get('experimentName')
+        if not experimentName:
+            raise InvalidRequest('File metadata must contain experimentName key.')
+        experimentFilePath = fileInfo.get('experimentFilePath')
+        if not experimentFilePath:
+            raise InvalidRequest('File metadata must contain experimentFilePath key.')
+        url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseData = self.sendSessionRequest(url=url, method='POST')
+        return FileMetadata(responseData)
+
+    @CatRestApi.execute
+    def updateExperimentFile(self, fileInfo):
+        experimentName = fileInfo.get('experimentName')
+        if not experimentName:
+            raise InvalidRequest('File metadata must contain experimentName key.')
+        experimentFilePath = fileInfo.get('experimentFilePath')
+        if not experimentFilePath:
+            raise InvalidRequest('File metadata must contain experimentFilePath key.')
+        url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return FileMetadata(responseData)
+
+    @CatRestApi.execute
+    def updateExperimentFileById(self, fileInfo):
+        experimentName = fileInfo.get('experimentName')
+        if not experimentName:
+            raise InvalidRequest('File metadata must contain experimentName key.')
+        id = fileInfo.get('id')
+        if not id:
+            raise InvalidRequest('File metadata must contain id key.')
+        url = '%s/filesByExperimentAndId/%s/%s' % (self.getContextRoot(), experimentName, id)
+        url += '?fileInfo=%s' % (Encoder.encode(json.dumps(fileInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return FileMetadata(responseData)
+
+    @CatRestApi.execute
+    def getExperimentFiles(self, experimentName, queryDict={}):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        url = '%s/filesByExperiment/%s' % (self.getContextRoot(), experimentName)
+        url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, FileMetadata)
+
+    @CatRestApi.execute
+    def getExperimentFileById(self, experimentName, id):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not id:
+            raise InvalidRequest('Invalid file id provided.')
+        url = '%s/filesByExperimentAndId/%s/%s' % (self.getContextRoot(), experimentName, id)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return FileMetadata(responseData)
+
+    @CatRestApi.execute
+    def getExperimentFile(self, experimentName, experimentFilePath):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not experimentFilePath:
+            raise InvalidRequest('Invalid experiment file path provided.')
+        url = '%s/filesByExperimentAndPath/%s/%s' % (self.getContextRoot(), experimentName, Encoder.encode(experimentFilePath))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return FileMetadata(responseData)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = FileCatApi()
+
+    import time
+    t = long(time.time())
+    fileInfo = {
+        'experimentFilePath' : 'sv-%s' % t,
+        'experimentName' : 'exp1',
+        'power' : 12,
+        'powerUnits' : 'kW',
+        'force' : 15,
+        'forceUnits' : 'N',
+        'cKey' : {'a' : 1, 'b' : 'B', 'c' : 2.2},
+    }
+    fileMetadata = api.addFileByName(fileInfo)
+
+    print '\nADDED FILE:\n', fileMetadata
+ 
+    fileInfo['updateKey'] = 'here is desc'
+    fileMetadata = api.updateFileByName(fileInfo)
+    print '\nUPDATED FILE:\n', fileMetadata
+
+    fileInfo['updateKey2'] = 'new desc'
+    fileInfo['id'] = fileMetadata.get('id')
+    fileMetadata = api.updateFileById(fileInfo)
+    print '\nUPDATED FILE:\n', fileMetadata
+
+    queryDict = { 'experimentName' : 'exp2' }
+    print '\nQUERY FILES:\n', api.getFiles(queryDict=queryDict)
+

src/python/dm/cat_web_service/cli/__init__.py

+__version__ = "Development Snapshot"

src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py

+#!/usr/bin/env python
+
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class AddExperimentDatasetCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--dataset', dest='datasetName', help='Dataset name.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.options.datasetName is None:
+            raise InvalidRequest('Dataset name must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getDatasetName(self):
+        return self.options.datasetName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-add-experiment-dataset --dataset=DATASETNAME --experiment=EXPERIMENTNAME 
+        [key1:value1, key2:value2, ...]
+
+Description:
+    Adds experiment dataset to the metadata catalog. All provided key/value 
+    pairs are interpreted as dataset metadata.
+        """)
+        self.checkArgs()
+        api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        datasetInfo = self.splitArgsIntoDict()
+        datasetInfo['experimentName'] = self.getExperimentName()
+        datasetInfo['datasetName'] = self.getDatasetName()
+        datasetMetadata = api.addExperimentDataset(datasetInfo)
+        print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = AddExperimentDatasetCli()
+    cli.run()
+

src/python/dm/cat_web_service/cli/addExperimentFileCli.py

+#!/usr/bin/env python
+
+from dm.cat_web_service.api.fileCatApi import FileCatApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class AddExperimentFileCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--file', dest='experimentFilePath', help='Experiment file path.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.options.experimentFilePath is None:
+            raise InvalidRequest('Experiment file path must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getExperimentFilePath(self):
+        return self.options.experimentFilePath
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-add-experiment-file --file=EXPERIMENTFILEPATH --experiment=EXPERIMENTNAME 
+        [key1:value1, key2:value2, ...]
+
+Description:
+    Adds experiment file to the metadata catalog. All provided key/value pairs 
+    are interpreted as file metadata.
+        """)
+        self.checkArgs()
+        api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        fileInfo = self.splitArgsIntoDict()
+        fileInfo['experimentName'] = self.getExperimentName()
+        fileInfo['experimentFilePath'] = self.getExperimentFilePath()
+        fileMetadata = api.addExperimentFile(fileInfo)
+        print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = AddExperimentFileCli()
+    cli.run()
+

src/python/dm/cat_web_service/cli/catWebServiceCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmRestCli import DmRestCli
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class CatWebServiceCli(DmRestCli):
+    """ DM CAT web service cli class. """
+
+    def __init__(self, validArgCount=0):
+        DmRestCli.__init__(self, validArgCount)
+
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getCatWebServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getCatWebServicePort()
+                        

src/python/dm/cat_web_service/cli/catWebServiceSessionCli.py

+#!/usr/bin/env python
+
+from dm.common.cli.dmRestSessionCli import DmRestSessionCli
+from dm.common.utility.osUtility import OsUtility
+from dm.common.utility.configurationManager import ConfigurationManager
+
+class CatWebServiceSessionCli(DmRestSessionCli):
+    """ DM CAT web service session cli class. """
+
+    DEFAULT_SESSION_CACHE_FILE = OsUtility.getUserHomeDir() + '/.dm/.cat.session.cache'
+
+    def __init__(self, validArgCount=0):
+        DmRestSessionCli.__init__(self, validArgCount)
+        ConfigurationManager.getInstance().setSessionCacheFile(CatWebServiceSessionCli.DEFAULT_SESSION_CACHE_FILE)
+
+    def getDefaultServiceHost(self):
+        return ConfigurationManager.getInstance().getCatWebServiceHost()
+
+    def getDefaultServicePort(self):
+        return ConfigurationManager.getInstance().getCatWebServicePort()
+