added dataset CLI and API classes for mongodb catalogging prototype

bin/dm-add-experiment-dataset

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-get-datasets

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getDatasetsCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-get-experiment-dataset

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-get-experiment-dataset-files

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-get-experiment-datasets

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-update-experiment-dataset

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+

src/python/dm/cat_web_service/api/datasetRestApi.py

+#!/usr/bin/env python
+
+import os
+import urllib
+import json
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.datasetMetadata import DatasetMetadata
+from dm.common.objects.fileMetadata import FileMetadata
+from catRestApi import CatRestApi
+
+class DatasetRestApi(CatRestApi):
+    
+    def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+        CatRestApi.__init__(self, username, password, host, port, protocol)
+
+    @CatRestApi.execute
+    def addExperimentDataset(self, datasetInfo):
+        experimentName = datasetInfo.get('experimentName')
+        if not experimentName:
+            raise InvalidRequest('Dataset metadata must contain experimentName key.')
+        datasetName = datasetInfo.get('datasetName')
+        if not datasetName:
+            raise InvalidRequest('Dataset metadata must contain datasetName key.')
+        url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+        responseData = self.sendSessionRequest(url=url, method='POST')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def updateExperimentDataset(self, datasetInfo):
+        experimentName = datasetInfo.get('experimentName')
+        if not experimentName:
+            raise InvalidRequest('Dataset metadata must contain experimentName key.')
+        datasetName = datasetInfo.get('datasetName')
+        if not datasetName:
+            raise InvalidRequest('Dataset metadata must contain datasetName key.')
+        url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def updateDatasetById(self, datasetInfo):
+        id = datasetInfo.get('id')
+        if not id:
+            raise InvalidRequest('Dataset metadata must contain id key.')
+        url = '%s/datasets/%s' % (self.getContextRoot(), id)
+        url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+        responseData = self.sendSessionRequest(url=url, method='PUT')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def getDatasets(self, queryDict={}):
+        url = '%s/datasets' % (self.getContextRoot())
+        url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, DatasetMetadata)
+
+    @CatRestApi.execute
+    def getExperimentDatasets(self, experimentName, queryDict={}):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        url = '%s/datasetsByExperiment/%s' % (self.getContextRoot(), experimentName)
+        url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, DatasetMetadata)
+
+    @CatRestApi.execute
+    def getDatasetById(self, id):
+        if not id:
+            raise InvalidRequest('Invalid dataset id provided.')
+        url = '%s/datasets/%s' % (self.getContextRoot(), id)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def getExperimentDataset(self, experimentName, datasetName):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return DatasetMetadata(responseData)
+
+    @CatRestApi.execute
+    def getExperimentDatasetFiles(self, experimentName, datasetName):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        url = '%s/filesByExperimentDataset/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+        responseData = self.sendSessionRequest(url=url, method='GET')
+        return self.toDmObjectList(responseData, FileMetadata)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+    api = DatasetRestApi()
+    print api.getDatasets()
+    print api.getDatasetById('556de0059e058b0ef4c4413b')
+    print api.getDatasetByName('xyz-001')
+
+    import time
+    t = long(time.time())
+    datasetInfo = {
+        'datasetName' : 'sv-%s' % t,
+        'experimentName' : 'exp1',
+        'power' : 12,
+        'powerUnits' : 'kW',
+        'force' : 15,
+        'forceUnits' : 'N',
+        'cKey' : {'a' : 1, 'b' : 'B', 'c' : 2.2},
+    }
+    datasetMetadata = api.addDatasetByName(datasetInfo)
+
+    print '\nADDED DATASET:\n', datasetMetadata
+ 
+    datasetInfo['updateKey'] = 'here is desc'
+    datasetMetadata = api.updateDatasetByName(datasetInfo)
+    print '\nUPDATED DATASET:\n', datasetMetadata
+
+    datasetInfo['updateKey2'] = 'new desc'
+    datasetInfo['id'] = datasetMetadata.get('id')
+    datasetMetadata = api.updateDatasetById(datasetInfo)
+    print '\nUPDATED DATASET:\n', datasetMetadata
+
+    queryDict = { 'experimentName' : 'exp2' }
+    print '\nQUERY DATASETS:\n', api.getDatasets(queryDict=queryDict)
+

src/python/dm/cat_web_service/service/datasetRouteDescriptor.py

+#!/usr/bin/env python
+
+#
+# Dataset route descriptor.
+#
+
+from dm.common.utility.configurationManager import ConfigurationManager
+from datasetSessionController import DatasetSessionController
+
+class DatasetRouteDescriptor:
+
+    @classmethod
+    def getRoutes(cls):
+        contextRoot = ConfigurationManager.getInstance().getContextRoot()
+
+        # Static instances shared between different routes
+        datasetSessionController = DatasetSessionController()
+
+        # Define routes.
+        routes = [
+
+            # Add experiment dataset
+            {
+                'name' : 'addExperimentDataset',
+                'path' : '%s/datasetsByExperiment/:(experimentName)/:(datasetName)' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'addExperimentDataset',
+                'method' : ['POST']
+            },
+
+            # Update dataset by id
+            {
+                'name' : 'updateDatasetById',
+                'path' : '%s/datasets/:(id)' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'updateDatasetById',
+                'method' : ['PUT']
+            },
+
+            # Update experiment dataset by name
+            {
+                'name' : 'updateExperimentDataset',
+                'path' : '%s/datasetsByExperiment/:(experimentName)/:(datasetName)' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'updateExperimentDataset',
+                'method' : ['PUT']
+            },
+
+            # Get dataset info list
+            {
+                'name' : 'getDatasets',
+                'path' : '%s/datasets' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'getDatasets',
+                'method' : ['GET']
+            },
+
+            # Get experiment dataset info list 
+            {
+                'name' : 'getExperimentDatasets',
+                'path' : '%s/datasetsByExperiment/:(experimentName)' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'getExperimentDatasets',
+                'method' : ['GET']
+            },
+
+            # Get dataset by id
+            {
+                'name' : 'getDatasetById',
+                'path' : '%s/datasets/:(id)' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'getDatasetById',
+                'method' : ['GET']
+            },
+
+            # Get experiment dataset by name
+            {
+                'name' : 'getExperimentDataset',
+                'path' : '%s/datasetsByExperiment/:(experimentName)/:(datasetName)' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'getExperimentDataset',
+                'method' : ['GET']
+            },
+
+            # Get experiment dataset files
+            {
+                'name' : 'getExperimentDatasetFiles',
+                'path' : '%s/filesByExperimentDataset/:(experimentName)/:(datasetName)' % contextRoot,
+                'controller' : datasetSessionController,
+                'action' : 'getExperimentDatasetFiles',
+                'method' : ['GET']
+            },
+
+
+        ]
+       
+        return routes
+
+

src/python/dm/cat_web_service/service/datasetSessionController.py

+#!/usr/bin/env python
+
+import cherrypy
+import json
+from dm.common.utility.encoder import Encoder
+from dm.common.service.dmSessionController import DmSessionController
+from dm.cat_web_service.service.impl.datasetSessionControllerImpl import DatasetSessionControllerImpl
+
+class DatasetSessionController(DmSessionController):
+
+    def __init__(self):
+        DmSessionController.__init__(self)
+        self.datasetSessionControllerImpl = DatasetSessionControllerImpl()
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def addExperimentDataset(self, experimentName, datasetName, **kwargs):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        encodedDatasetInfo = kwargs.get('datasetInfo')
+        if not encodedDatasetInfo:
+            raise InvalidRequest('Invalid dataset info provided.')
+        datasetInfo = json.loads(Encoder.decode(encodedDatasetInfo))
+        datasetInfo['datasetName'] = datasetName
+        datasetInfo['experimentName'] = experimentName
+        response = self.datasetSessionControllerImpl.addExperimentDataset(datasetInfo).getFullJsonRep()
+        self.logger.debug('Added dataset %s: %s' % (datasetName,response))
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def updateExperimentDataset(self, experimentName, datasetName, **kwargs):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        encodedDatasetInfo = kwargs.get('datasetInfo')
+        if not encodedDatasetInfo:
+            raise InvalidRequest('Invalid dataset info provided.')
+        datasetInfo = json.loads(Encoder.decode(encodedDatasetInfo))
+        datasetInfo['datasetName'] = datasetName
+        datasetInfo['experimentName'] = experimentName
+        response = self.datasetSessionControllerImpl.updateExperimentDataset(datasetInfo).getFullJsonRep()
+        self.logger.debug('Updated dataset %s: %s' % (datasetName,response))
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def updateDatasetById(self, id, **kwargs):
+        if not id:
+            raise InvalidRequest('Invalid dataset id provided.')
+        encodedDatasetInfo = kwargs.get('datasetInfo')
+        if not encodedDatasetInfo:
+            raise InvalidRequest('Invalid dataset info provided.')
+        datasetInfo = json.loads(Encoder.decode(encodedDatasetInfo))
+        response = self.datasetSessionControllerImpl.updateDatasetById(datasetInfo).getFullJsonRep()
+        self.logger.debug('Updated dataset id %s: %s' % (id,response))
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def getDatasets(self, **kwargs):
+        encodedQueryDict = kwargs.get('queryDict')
+        queryDict = {}
+        if encodedQueryDict:
+            queryDict = json.loads(Encoder.decode(encodedQueryDict))
+        return self.listToJson(self.datasetSessionControllerImpl.getDatasets(queryDict=queryDict))
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def getExperimentDatasets(self, experimentName, **kwargs):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        encodedQueryDict = kwargs.get('queryDict')
+        queryDict = {}
+        if encodedQueryDict:
+            queryDict = json.loads(Encoder.decode(encodedQueryDict))
+        return self.listToJson(self.datasetSessionControllerImpl.getExperimentDatasets(experimentName, queryDict=queryDict))
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def getDatasetById(self, id, **kwargs):
+        if not id:
+            raise InvalidRequest('Invalid id provided.')
+        response = self.datasetSessionControllerImpl.getDatasetById(id).getFullJsonRep()
+        self.logger.debug('Returning dataset id %s: %s' % (id,response))
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def getExperimentDataset(self, experimentName, datasetName, **kwargs):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        response = self.datasetSessionControllerImpl.getExperimentDataset(experimentName, datasetName).getFullJsonRep()
+        self.logger.debug('Returning dataset %s: %s' % (datasetName,response))
+        return response
+
+    @cherrypy.expose
+    @DmSessionController.require(DmSessionController.isAdministrator())
+    @DmSessionController.execute
+    def getExperimentDatasetFiles(self, experimentName, datasetName, **kwargs):
+        if not experimentName:
+            raise InvalidRequest('Invalid experiment name provided.')
+        if not datasetName:
+            raise InvalidRequest('Invalid dataset name provided.')
+        response = self.listToJson(self.datasetSessionControllerImpl.getExperimentDatasetFiles(experimentName, datasetName))
+        return response
+