From f3b6d5ab5cb4a518520b651f3159148523eb1f52 Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Tue, 7 Jul 2015 15:33:22 +0000
Subject: [PATCH] added dataset CLI and API classes for  mongodb catalogging
 prototype

---
 .../cli/addExperimentDatasetCli.py            | 47 +++++++++++++++++
 .../dm/cat_web_service/cli/getDatasetsCli.py  | 37 ++++++++++++++
 .../cli/getExperimentDatasetCli.py            | 42 ++++++++++++++++
 .../cli/getExperimentDatasetFilesCli.py       | 44 ++++++++++++++++
 .../cli/getExperimentDatasetsCli.py           | 42 ++++++++++++++++
 .../cli/updateExperimentDatasetCli.py         | 50 +++++++++++++++++++
 6 files changed, 262 insertions(+)
 create mode 100755 src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
 create mode 100755 src/python/dm/cat_web_service/cli/getDatasetsCli.py
 create mode 100755 src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
 create mode 100755 src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
 create mode 100755 src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
 create mode 100755 src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py

diff --git a/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
new file mode 100755
index 00000000..ba84de0a
--- /dev/null
+++ b/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class AddExperimentDatasetCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--dataset', dest='datasetName', help='Dataset name.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.options.datasetName is None:
+            raise InvalidRequest('Dataset name must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getDatasetName(self):
+        return self.options.datasetName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-add-experiment-dataset --dataset=DATASETNAME --experiment=EXPERIMENTNAME 
+        [key1:value1, key2:value2, ...]
+
+Description:
+    Adds experiment dataset to the metadata catalog. All provided key/value 
+    pairs are interpreted as dataset metadata.
+        """)
+        self.checkArgs()
+        api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        datasetInfo = self.splitArgsIntoDict()
+        datasetInfo['experimentName'] = self.getExperimentName()
+        datasetInfo['datasetName'] = self.getDatasetName()
+        datasetMetadata = api.addExperimentDataset(datasetInfo)
+        print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = AddExperimentDatasetCli()
+    cli.run()
+
diff --git a/src/python/dm/cat_web_service/cli/getDatasetsCli.py b/src/python/dm/cat_web_service/cli/getDatasetsCli.py
new file mode 100755
index 00000000..f8d0930a
--- /dev/null
+++ b/src/python/dm/cat_web_service/cli/getDatasetsCli.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class GetDatasetsCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+
+    def checkArgs(self):
+        pass
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-get-datasets 
+        [key1:value1, key2:value2, ...]
+
+Description:
+    Retrieve datasets from the metadata catalog. Only those datasets that
+    match provided key/value metadata pairs will be returned. If no specific
+    metadata key/values are requested, all experiment datasets will be 
+    returned. 
+        """)
+        self.checkArgs()
+        api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        queryDict = self.splitArgsIntoDict()
+        datasetMetadataList = api.getDatasets(queryDict)
+        for datasetMetadata in datasetMetadataList:
+            print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = GetDatasetsCli()
+    cli.run()
+
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
new file mode 100755
index 00000000..f17035fc
--- /dev/null
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class GetExperimentDatasetCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--dataset', dest='datasetName', help='Dataset name.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.options.datasetName is None:
+            raise InvalidRequest('Dataset name must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getDatasetName(self):
+        return self.options.datasetName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-get-experiment-dataset --dataset=DATASETNAME --experiment=EXPERIMENTNAME 
+
+Description:
+    Retrieve experiment dataset metadata from the catalog. 
+        """)
+        self.checkArgs()
+        api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        datasetMetadata = api.getExperimentDataset(self.getExperimentName(), self.getDatasetName())
+        print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = GetExperimentDatasetCli()
+    cli.run()
+
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
new file mode 100755
index 00000000..02e21867
--- /dev/null
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class GetExperimentDatasetFilesCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--dataset', dest='datasetName', help='Dataset name.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.options.datasetName is None:
+            raise InvalidRequest('Dataset name must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getDatasetName(self):
+        return self.options.datasetName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-get-experiment-dataset-files --dataset=DATASETNAME 
+        --experiment=EXPERIMENTNAME 
+
+Description:
+    Retrieve experiment dataset files from the catalog. 
+        """)
+        self.checkArgs()
+        api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        fileMetadataList = api.getExperimentDatasetFiles(self.getExperimentName(), self.getDatasetName())
+        for fileMetadata in fileMetadataList:
+            print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = GetExperimentDatasetFilesCli()
+    cli.run()
+
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
new file mode 100755
index 00000000..3260932d
--- /dev/null
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class GetExperimentDatasetsCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-get-experiment-datasets --experiment=EXPERIMENTNAME 
+        [key1:value1, key2:value2, ...]
+
+Description:
+    Retrieve experiment datasets from the metadata catalog. Only those 
+    datasets that match provided key/value metadata pairs will be returned. 
+    If no specific metadata key/values are requested, all experiment datasets 
+    will be returned. 
+        """)
+        self.checkArgs()
+        api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        queryDict = self.splitArgsIntoDict()
+        datasetMetadataList = api.getExperimentDatasets(self.getExperimentName(), queryDict)
+        for datasetMetadata in datasetMetadataList:
+            print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = GetExperimentDatasetsCli()
+    cli.run()
+
diff --git a/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
new file mode 100755
index 00000000..1098b393
--- /dev/null
+++ b/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from catWebServiceSessionCli import CatWebServiceSessionCli
+
+class UpdateExperimentDatasetCli(CatWebServiceSessionCli):
+    def __init__(self):
+        CatWebServiceSessionCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
+        self.addOption('', '--dataset', dest='datasetName', help='Dataset name.')
+        self.addOption('', '--experiment', dest='experimentName', help='Experiment name.')
+
+    def checkArgs(self):
+        if self.options.experimentName is None:
+            raise InvalidRequest('Experiment name must be provided.')
+        if self.options.datasetName is None:
+            raise InvalidRequest('Dataset name must be provided.')
+
+    def getExperimentName(self):
+        return self.options.experimentName
+
+    def getDatasetName(self):
+        return self.options.datasetName
+
+    def runCommand(self):
+        self.parseArgs(usage="""
+    dm-update-experiment-dataset --dataset=DATASETNAME 
+        --experiment=EXPERIMENTNAME 
+        [key1:value1, key2:value2, ...]
+
+Description:
+    Updates experiment dataset in the metadata catalog. All provided key/value 
+    pairs are interpreted as dataset metadata, and will be merged with existing
+    metadata, overwriting values for existing keys, and adding values for
+    new keys.
+        """)
+        self.checkArgs()
+        api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+        datasetInfo = self.splitArgsIntoDict()
+        datasetInfo['experimentName'] = self.getExperimentName()
+        datasetInfo['datasetName'] = self.getDatasetName()
+        datasetMetadata = api.updateExperimentDataset(datasetInfo)
+        print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
+
+#######################################################################
+# Run command.
+if __name__ == '__main__':
+    cli = UpdateExperimentDatasetCli()
+    cli.run()
+
-- 
GitLab