From 32901a78af9bfa7fdf8840fef04b3752de716b7f Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Tue, 7 Jul 2015 15:34:39 +0000
Subject: [PATCH] added dataset CLI and API classes for mongodb catalogging
prototype
---
bin/dm-add-experiment-dataset | 18 +++
bin/dm-get-datasets | 18 +++
bin/dm-get-experiment-dataset | 18 +++
bin/dm-get-experiment-dataset-files | 18 +++
bin/dm-get-experiment-datasets | 18 +++
bin/dm-update-experiment-dataset | 18 +++
.../dm/cat_web_service/api/datasetRestApi.py | 134 ++++++++++++++++++
.../service/datasetRouteDescriptor.py | 99 +++++++++++++
.../service/datasetSessionController.py | 119 ++++++++++++++++
9 files changed, 460 insertions(+)
create mode 100755 bin/dm-add-experiment-dataset
create mode 100755 bin/dm-get-datasets
create mode 100755 bin/dm-get-experiment-dataset
create mode 100755 bin/dm-get-experiment-dataset-files
create mode 100755 bin/dm-get-experiment-datasets
create mode 100755 bin/dm-update-experiment-dataset
create mode 100755 src/python/dm/cat_web_service/api/datasetRestApi.py
create mode 100755 src/python/dm/cat_web_service/service/datasetRouteDescriptor.py
create mode 100755 src/python/dm/cat_web_service/service/datasetSessionController.py
diff --git a/bin/dm-add-experiment-dataset b/bin/dm-add-experiment-dataset
new file mode 100755
index 00000000..8b8dc897
--- /dev/null
+++ b/bin/dm-add-experiment-dataset
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+ cd `dirname $0` && myDir=`pwd`
+ setupFile=$myDir/../setup.sh
+ if [ ! -f $setupFile ]; then
+ echo "Cannot find setup file: $setupFile"
+ exit 1
+ fi
+ source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+
diff --git a/bin/dm-get-datasets b/bin/dm-get-datasets
new file mode 100755
index 00000000..9e245c96
--- /dev/null
+++ b/bin/dm-get-datasets
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+ cd `dirname $0` && myDir=`pwd`
+ setupFile=$myDir/../setup.sh
+ if [ ! -f $setupFile ]; then
+ echo "Cannot find setup file: $setupFile"
+ exit 1
+ fi
+ source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getDatasetsCli.py $DM_COMMAND_ARGS"
+
+
diff --git a/bin/dm-get-experiment-dataset b/bin/dm-get-experiment-dataset
new file mode 100755
index 00000000..93a8d2f9
--- /dev/null
+++ b/bin/dm-get-experiment-dataset
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+ cd `dirname $0` && myDir=`pwd`
+ setupFile=$myDir/../setup.sh
+ if [ ! -f $setupFile ]; then
+ echo "Cannot find setup file: $setupFile"
+ exit 1
+ fi
+ source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+
diff --git a/bin/dm-get-experiment-dataset-files b/bin/dm-get-experiment-dataset-files
new file mode 100755
index 00000000..99e29e33
--- /dev/null
+++ b/bin/dm-get-experiment-dataset-files
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+ cd `dirname $0` && myDir=`pwd`
+ setupFile=$myDir/../setup.sh
+ if [ ! -f $setupFile ]; then
+ echo "Cannot find setup file: $setupFile"
+ exit 1
+ fi
+ source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py $DM_COMMAND_ARGS"
+
+
diff --git a/bin/dm-get-experiment-datasets b/bin/dm-get-experiment-datasets
new file mode 100755
index 00000000..b5b4cd07
--- /dev/null
+++ b/bin/dm-get-experiment-datasets
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+ cd `dirname $0` && myDir=`pwd`
+ setupFile=$myDir/../setup.sh
+ if [ ! -f $setupFile ]; then
+ echo "Cannot find setup file: $setupFile"
+ exit 1
+ fi
+ source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py $DM_COMMAND_ARGS"
+
+
diff --git a/bin/dm-update-experiment-dataset b/bin/dm-update-experiment-dataset
new file mode 100755
index 00000000..25397c9c
--- /dev/null
+++ b/bin/dm-update-experiment-dataset
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+ cd `dirname $0` && myDir=`pwd`
+ setupFile=$myDir/../setup.sh
+ if [ ! -f $setupFile ]; then
+ echo "Cannot find setup file: $setupFile"
+ exit 1
+ fi
+ source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+
diff --git a/src/python/dm/cat_web_service/api/datasetRestApi.py b/src/python/dm/cat_web_service/api/datasetRestApi.py
new file mode 100755
index 00000000..f75254f0
--- /dev/null
+++ b/src/python/dm/cat_web_service/api/datasetRestApi.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+
+import os
+import urllib
+import json
+
+from dm.common.utility.encoder import Encoder
+from dm.common.exceptions.dmException import DmException
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.datasetMetadata import DatasetMetadata
+from dm.common.objects.fileMetadata import FileMetadata
+from catRestApi import CatRestApi
+
+class DatasetRestApi(CatRestApi):
+
+ def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
+ CatRestApi.__init__(self, username, password, host, port, protocol)
+
+ @CatRestApi.execute
+ def addExperimentDataset(self, datasetInfo):
+ experimentName = datasetInfo.get('experimentName')
+ if not experimentName:
+ raise InvalidRequest('Dataset metadata must contain experimentName key.')
+ datasetName = datasetInfo.get('datasetName')
+ if not datasetName:
+ raise InvalidRequest('Dataset metadata must contain datasetName key.')
+ url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+ url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+ responseData = self.sendSessionRequest(url=url, method='POST')
+ return DatasetMetadata(responseData)
+
+ @CatRestApi.execute
+ def updateExperimentDataset(self, datasetInfo):
+ experimentName = datasetInfo.get('experimentName')
+ if not experimentName:
+ raise InvalidRequest('Dataset metadata must contain experimentName key.')
+ datasetName = datasetInfo.get('datasetName')
+ if not datasetName:
+ raise InvalidRequest('Dataset metadata must contain datasetName key.')
+ url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+ url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+ responseData = self.sendSessionRequest(url=url, method='PUT')
+ return DatasetMetadata(responseData)
+
+ @CatRestApi.execute
+ def updateDatasetById(self, datasetInfo):
+ id = datasetInfo.get('id')
+ if not id:
+ raise InvalidRequest('Dataset metadata must contain id key.')
+ url = '%s/datasets/%s' % (self.getContextRoot(), id)
+ url += '?datasetInfo=%s' % (Encoder.encode(json.dumps(datasetInfo)))
+ responseData = self.sendSessionRequest(url=url, method='PUT')
+ return DatasetMetadata(responseData)
+
+ @CatRestApi.execute
+ def getDatasets(self, queryDict={}):
+ url = '%s/datasets' % (self.getContextRoot())
+ url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+ responseData = self.sendSessionRequest(url=url, method='GET')
+ return self.toDmObjectList(responseData, DatasetMetadata)
+
+ @CatRestApi.execute
+ def getExperimentDatasets(self, experimentName, queryDict={}):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ url = '%s/datasetsByExperiment/%s' % (self.getContextRoot(), experimentName)
+ url += '?queryDict=%s' % (Encoder.encode(json.dumps(queryDict)))
+ responseData = self.sendSessionRequest(url=url, method='GET')
+ return self.toDmObjectList(responseData, DatasetMetadata)
+
+ @CatRestApi.execute
+ def getDatasetById(self, id):
+ if not id:
+ raise InvalidRequest('Invalid dataset id provided.')
+ url = '%s/datasets/%s' % (self.getContextRoot(), id)
+ responseData = self.sendSessionRequest(url=url, method='GET')
+ return DatasetMetadata(responseData)
+
+ @CatRestApi.execute
+ def getExperimentDataset(self, experimentName, datasetName):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ if not datasetName:
+ raise InvalidRequest('Invalid dataset name provided.')
+ url = '%s/datasetsByExperiment/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+ responseData = self.sendSessionRequest(url=url, method='GET')
+ return DatasetMetadata(responseData)
+
+ @CatRestApi.execute
+ def getExperimentDatasetFiles(self, experimentName, datasetName):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ if not datasetName:
+ raise InvalidRequest('Invalid dataset name provided.')
+ url = '%s/filesByExperimentDataset/%s/%s' % (self.getContextRoot(), experimentName, datasetName)
+ responseData = self.sendSessionRequest(url=url, method='GET')
+ return self.toDmObjectList(responseData, FileMetadata)
+
+#######################################################################
+# Testing.
+
+if __name__ == '__main__':
+ api = DatasetRestApi()
+ print api.getDatasets()
+ print api.getDatasetById('556de0059e058b0ef4c4413b')
+ print api.getDatasetByName('xyz-001')
+
+ import time
+ t = long(time.time())
+ datasetInfo = {
+ 'datasetName' : 'sv-%s' % t,
+ 'experimentName' : 'exp1',
+ 'power' : 12,
+ 'powerUnits' : 'kW',
+ 'force' : 15,
+ 'forceUnits' : 'N',
+ 'cKey' : {'a' : 1, 'b' : 'B', 'c' : 2.2},
+ }
+ datasetMetadata = api.addDatasetByName(datasetInfo)
+
+ print '\nADDED DATASET:\n', datasetMetadata
+
+ datasetInfo['updateKey'] = 'here is desc'
+ datasetMetadata = api.updateDatasetByName(datasetInfo)
+ print '\nUPDATED DATASET:\n', datasetMetadata
+
+ datasetInfo['updateKey2'] = 'new desc'
+ datasetInfo['id'] = datasetMetadata.get('id')
+ datasetMetadata = api.updateDatasetById(datasetInfo)
+ print '\nUPDATED DATASET:\n', datasetMetadata
+
+ queryDict = { 'experimentName' : 'exp2' }
+ print '\nQUERY DATASETS:\n', api.getDatasets(queryDict=queryDict)
+
diff --git a/src/python/dm/cat_web_service/service/datasetRouteDescriptor.py b/src/python/dm/cat_web_service/service/datasetRouteDescriptor.py
new file mode 100755
index 00000000..b5f5d6c4
--- /dev/null
+++ b/src/python/dm/cat_web_service/service/datasetRouteDescriptor.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+#
+# Dataset route descriptor.
+#
+
+from dm.common.utility.configurationManager import ConfigurationManager
+from datasetSessionController import DatasetSessionController
+
+class DatasetRouteDescriptor:
+
+ @classmethod
+ def getRoutes(cls):
+ contextRoot = ConfigurationManager.getInstance().getContextRoot()
+
+ # Static instances shared between different routes
+ datasetSessionController = DatasetSessionController()
+
+ # Define routes.
+ routes = [
+
+ # Add experiment dataset
+ {
+ 'name' : 'addExperimentDataset',
+ 'path' : '%s/datasetsByExperiment/:(experimentName)/:(datasetName)' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'addExperimentDataset',
+ 'method' : ['POST']
+ },
+
+ # Update dataset by id
+ {
+ 'name' : 'updateDatasetById',
+ 'path' : '%s/datasets/:(id)' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'updateDatasetById',
+ 'method' : ['PUT']
+ },
+
+ # Update experiment dataset by name
+ {
+ 'name' : 'updateExperimentDataset',
+ 'path' : '%s/datasetsByExperiment/:(experimentName)/:(datasetName)' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'updateExperimentDataset',
+ 'method' : ['PUT']
+ },
+
+ # Get dataset info list
+ {
+ 'name' : 'getDatasets',
+ 'path' : '%s/datasets' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'getDatasets',
+ 'method' : ['GET']
+ },
+
+ # Get experiment dataset info list
+ {
+ 'name' : 'getExperimentDatasets',
+ 'path' : '%s/datasetsByExperiment/:(experimentName)' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'getExperimentDatasets',
+ 'method' : ['GET']
+ },
+
+ # Get dataset by id
+ {
+ 'name' : 'getDatasetById',
+ 'path' : '%s/datasets/:(id)' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'getDatasetById',
+ 'method' : ['GET']
+ },
+
+ # Get experiment dataset by name
+ {
+ 'name' : 'getExperimentDataset',
+ 'path' : '%s/datasetsByExperiment/:(experimentName)/:(datasetName)' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'getExperimentDataset',
+ 'method' : ['GET']
+ },
+
+ # Get experiment dataset files
+ {
+ 'name' : 'getExperimentDatasetFiles',
+ 'path' : '%s/filesByExperimentDataset/:(experimentName)/:(datasetName)' % contextRoot,
+ 'controller' : datasetSessionController,
+ 'action' : 'getExperimentDatasetFiles',
+ 'method' : ['GET']
+ },
+
+
+ ]
+
+ return routes
+
+
diff --git a/src/python/dm/cat_web_service/service/datasetSessionController.py b/src/python/dm/cat_web_service/service/datasetSessionController.py
new file mode 100755
index 00000000..867b0737
--- /dev/null
+++ b/src/python/dm/cat_web_service/service/datasetSessionController.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+
+import cherrypy
+import json
+from dm.common.utility.encoder import Encoder
+from dm.common.service.dmSessionController import DmSessionController
+from dm.cat_web_service.service.impl.datasetSessionControllerImpl import DatasetSessionControllerImpl
+
+class DatasetSessionController(DmSessionController):
+
+ def __init__(self):
+ DmSessionController.__init__(self)
+ self.datasetSessionControllerImpl = DatasetSessionControllerImpl()
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def addExperimentDataset(self, experimentName, datasetName, **kwargs):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ if not datasetName:
+ raise InvalidRequest('Invalid dataset name provided.')
+ encodedDatasetInfo = kwargs.get('datasetInfo')
+ if not encodedDatasetInfo:
+ raise InvalidRequest('Invalid dataset info provided.')
+ datasetInfo = json.loads(Encoder.decode(encodedDatasetInfo))
+ datasetInfo['datasetName'] = datasetName
+ datasetInfo['experimentName'] = experimentName
+ response = self.datasetSessionControllerImpl.addExperimentDataset(datasetInfo).getFullJsonRep()
+ self.logger.debug('Added dataset %s: %s' % (datasetName,response))
+ return response
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def updateExperimentDataset(self, experimentName, datasetName, **kwargs):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ if not datasetName:
+ raise InvalidRequest('Invalid dataset name provided.')
+ encodedDatasetInfo = kwargs.get('datasetInfo')
+ if not encodedDatasetInfo:
+ raise InvalidRequest('Invalid dataset info provided.')
+ datasetInfo = json.loads(Encoder.decode(encodedDatasetInfo))
+ datasetInfo['datasetName'] = datasetName
+ datasetInfo['experimentName'] = experimentName
+ response = self.datasetSessionControllerImpl.updateExperimentDataset(datasetInfo).getFullJsonRep()
+ self.logger.debug('Updated dataset %s: %s' % (datasetName,response))
+ return response
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def updateDatasetById(self, id, **kwargs):
+ if not id:
+ raise InvalidRequest('Invalid dataset id provided.')
+ encodedDatasetInfo = kwargs.get('datasetInfo')
+ if not encodedDatasetInfo:
+ raise InvalidRequest('Invalid dataset info provided.')
+ datasetInfo = json.loads(Encoder.decode(encodedDatasetInfo))
+ response = self.datasetSessionControllerImpl.updateDatasetById(datasetInfo).getFullJsonRep()
+ self.logger.debug('Updated dataset id %s: %s' % (id,response))
+ return response
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def getDatasets(self, **kwargs):
+ encodedQueryDict = kwargs.get('queryDict')
+ queryDict = {}
+ if encodedQueryDict:
+ queryDict = json.loads(Encoder.decode(encodedQueryDict))
+ return self.listToJson(self.datasetSessionControllerImpl.getDatasets(queryDict=queryDict))
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def getExperimentDatasets(self, experimentName, **kwargs):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ encodedQueryDict = kwargs.get('queryDict')
+ queryDict = {}
+ if encodedQueryDict:
+ queryDict = json.loads(Encoder.decode(encodedQueryDict))
+ return self.listToJson(self.datasetSessionControllerImpl.getExperimentDatasets(experimentName, queryDict=queryDict))
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def getDatasetById(self, id, **kwargs):
+ if not id:
+ raise InvalidRequest('Invalid id provided.')
+ response = self.datasetSessionControllerImpl.getDatasetById(id).getFullJsonRep()
+ self.logger.debug('Returning dataset id %s: %s' % (id,response))
+ return response
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def getExperimentDataset(self, experimentName, datasetName, **kwargs):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ if not datasetName:
+ raise InvalidRequest('Invalid dataset name provided.')
+ response = self.datasetSessionControllerImpl.getExperimentDataset(experimentName, datasetName).getFullJsonRep()
+ self.logger.debug('Returning dataset %s: %s' % (datasetName,response))
+ return response
+
+ @cherrypy.expose
+ @DmSessionController.require(DmSessionController.isAdministrator())
+ @DmSessionController.execute
+ def getExperimentDatasetFiles(self, experimentName, datasetName, **kwargs):
+ if not experimentName:
+ raise InvalidRequest('Invalid experiment name provided.')
+ if not datasetName:
+ raise InvalidRequest('Invalid dataset name provided.')
+ response = self.listToJson(self.datasetSessionControllerImpl.getExperimentDatasetFiles(experimentName, datasetName))
+ return response
+
--
GitLab