From ac5689a773d9a6e7f7c38bd6ea9dfc99e591069c Mon Sep 17 00:00:00 2001
From: Sinisa Veseli <sveseli@aps.anl.gov>
Date: Thu, 2 Mar 2017 02:39:31 +0000
Subject: [PATCH] refactored apis; added ability to skip optional plugins on
error
---
doc/rhel7_package_list.txt | 62 ++++++++++++++-----
.../cli/updateUsersFromApsDbCli.py | 2 +-
.../cat_web_service/api/catRestApiFactory.py | 6 +-
.../{datasetRestApi.py => datasetCatApi.py} | 4 +-
.../api/{fileRestApi.py => fileCatApi.py} | 4 +-
.../cli/addExperimentDatasetCli.py | 4 +-
.../cli/addExperimentFileCli.py | 4 +-
.../dm/cat_web_service/cli/getDatasetsCli.py | 4 +-
.../cli/getExperimentDatasetCli.py | 4 +-
.../cli/getExperimentDatasetFilesCli.py | 4 +-
.../cli/getExperimentDatasetsCli.py | 4 +-
.../cli/getExperimentFileCli.py | 4 +-
.../cli/getExperimentFilesCli.py | 4 +-
.../cli/updateExperimentDatasetCli.py | 4 +-
.../cli/updateExperimentFileCli.py | 4 +-
src/python/dm/common/objects/daqInfo.py | 20 ++++++
src/python/dm/common/objects/uploadInfo.py | 20 ++++++
.../processing/fileProcessingManager.py | 4 +-
.../common/processing/fileProcessingThread.py | 14 +++--
.../processing/plugins/fileProcessor.py | 14 +++++
.../processing/plugins/fileTransferPlugin.py | 16 ++++-
.../plugins/gridftpFileTransferPlugin.py | 14 +++--
.../plugins/rsyncFileTransferPlugin.py | 7 ++-
...WithChecksumAndDeleteFileTransferPlugin.py | 2 +-
.../daq_web_service/api/experimentDaqApi.py | 3 +-
.../impl/dsProcessFileNotificationPlugin.py | 2 +-
.../impl/experimentSessionControllerImpl.py | 2 +-
.../api/{authRestApi.py => authDsApi.py} | 4 +-
.../dm/ds_web_service/api/dsRestApiFactory.py | 30 +++++----
...xperimentRestApi.py => experimentDsApi.py} | 4 +-
.../api/{fileRestApi.py => fileDsApi.py} | 4 +-
.../api/{userRestApi.py => userDsApi.py} | 2 +-
.../dm/ds_web_service/cli/addExperimentCli.py | 10 +--
.../cli/addUserExperimentRoleCli.py | 4 +-
.../cli/addUserSystemRoleCli.py | 4 +-
.../cli/deleteUserExperimentRoleCli.py | 4 +-
.../cli/deleteUserSystemRoleCli.py | 4 +-
.../dm/ds_web_service/cli/downloadCli.py | 4 +-
.../dm/ds_web_service/cli/getExperimentCli.py | 4 +-
.../cli/getExperimentTypesCli.py | 4 +-
.../ds_web_service/cli/getExperimentsCli.py | 4 +-
.../dm/ds_web_service/cli/getUserCli.py | 4 +-
.../dm/ds_web_service/cli/getUsersCli.py | 4 +-
.../ds_web_service/cli/startExperimentCli.py | 4 +-
.../cli/statExperimentFileCli.py | 4 +-
.../ds_web_service/cli/stopExperimentCli.py | 4 +-
.../ds_web_service/cli/updateExperimentCli.py | 4 +-
.../service/auth/dsAuthPrincipalRetriever.py | 2 +-
.../service/auth/dsSessionManager.py | 2 +-
49 files changed, 233 insertions(+), 117 deletions(-)
rename src/python/dm/cat_web_service/api/{datasetRestApi.py => datasetCatApi.py} (98%)
rename src/python/dm/cat_web_service/api/{fileRestApi.py => fileCatApi.py} (98%)
rename src/python/dm/ds_web_service/api/{authRestApi.py => authDsApi.py} (95%)
rename src/python/dm/ds_web_service/api/{experimentRestApi.py => experimentDsApi.py} (97%)
rename src/python/dm/ds_web_service/api/{fileRestApi.py => fileDsApi.py} (97%)
rename src/python/dm/ds_web_service/api/{userRestApi.py => userDsApi.py} (99%)
diff --git a/doc/rhel7_package_list.txt b/doc/rhel7_package_list.txt
index 3750753b..e200a3c8 100644
--- a/doc/rhel7_package_list.txt
+++ b/doc/rhel7_package_list.txt
@@ -1,16 +1,46 @@
-- make
-- autoconf
-- expect
-- gcc
-- g++
-- subversion
-- zlib-devel
-- openssl-devel
-- libffi-devel
-- openldap-devel
-- readline-devel
-- ncurses-devel
-- qt-x11
-- qt-postgresql
-- qt-devel
-- gtk2-devel
+RHEL7 Packages
+===============
+
+make
+autoconf
+expect
+gcc
+g++
+subversion
+zlib-devel
+openssl-devel
+libffi-devel
+openldap-devel
+readline-devel
+ncurses-devel
+qt-x11
+qt-postgresql
+qt-devel
+gtk2-devel
+
+Globus Packages
+===============
+
+globus-openssl-module
+globus-ftp-client
+globus-gsi-proxy
+globus-gsi-openssl
+globus-ftp-control
+globus-gass-copy
+globus-common-16.8
+globus-gass-transfer
+globus-io-11.8
+globus-gss-assist
+globus-gsi-cert
+globus-xio-popen
+globus-xio-gsi
+globus-gsi-credential
+globus-callout-3.15
+globus-gsi-sysconfig
+globus-gsi-callback
+globus-xio-5.14
+globus-gssapi-gsi
+globus-gass-copy
+globus-gsi-proxy
+globus-gssapi-error
+
diff --git a/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py b/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py
index 6c7c347c..3cc3a7c6 100755
--- a/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py
+++ b/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py
@@ -54,7 +54,7 @@ Description:
try:
if self.options.useDmRestApi:
self.logger.debug('Using DM REST API')
- dmUserApi = DsRestApiFactory.getUserRestApi()
+ dmUserApi = DsRestApiFactory.getUserDsApi()
else:
self.logger.debug('Using DM DB API')
dmUserApi = UserDbApi()
diff --git a/src/python/dm/cat_web_service/api/catRestApiFactory.py b/src/python/dm/cat_web_service/api/catRestApiFactory.py
index b578e11e..64b30b6a 100755
--- a/src/python/dm/cat_web_service/api/catRestApiFactory.py
+++ b/src/python/dm/cat_web_service/api/catRestApiFactory.py
@@ -36,10 +36,10 @@ class CatRestApiFactory:
return (cls.__username, cls.__password, cls.__host, cls.__port, cls.__protocol)
@classmethod
- def getFileRestApi(cls):
- from userRestApi import FileRestApi
+ def getFileCatApi(cls):
+ from userCatApi import FileCatApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = FileRestApi(username, password, host, port, protocol)
+ api = FileCatApi(username, password, host, port, protocol)
return api
####################################################################
diff --git a/src/python/dm/cat_web_service/api/datasetRestApi.py b/src/python/dm/cat_web_service/api/datasetCatApi.py
similarity index 98%
rename from src/python/dm/cat_web_service/api/datasetRestApi.py
rename to src/python/dm/cat_web_service/api/datasetCatApi.py
index f75254f0..d028c36a 100755
--- a/src/python/dm/cat_web_service/api/datasetRestApi.py
+++ b/src/python/dm/cat_web_service/api/datasetCatApi.py
@@ -11,7 +11,7 @@ from dm.common.objects.datasetMetadata import DatasetMetadata
from dm.common.objects.fileMetadata import FileMetadata
from catRestApi import CatRestApi
-class DatasetRestApi(CatRestApi):
+class DatasetCatApi(CatRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
CatRestApi.__init__(self, username, password, host, port, protocol)
@@ -100,7 +100,7 @@ class DatasetRestApi(CatRestApi):
# Testing.
if __name__ == '__main__':
- api = DatasetRestApi()
+ api = DatasetCatApi()
print api.getDatasets()
print api.getDatasetById('556de0059e058b0ef4c4413b')
print api.getDatasetByName('xyz-001')
diff --git a/src/python/dm/cat_web_service/api/fileRestApi.py b/src/python/dm/cat_web_service/api/fileCatApi.py
similarity index 98%
rename from src/python/dm/cat_web_service/api/fileRestApi.py
rename to src/python/dm/cat_web_service/api/fileCatApi.py
index 75ec147f..08658d1f 100755
--- a/src/python/dm/cat_web_service/api/fileRestApi.py
+++ b/src/python/dm/cat_web_service/api/fileCatApi.py
@@ -10,7 +10,7 @@ from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.objects.fileMetadata import FileMetadata
from catRestApi import CatRestApi
-class FileRestApi(CatRestApi):
+class FileCatApi(CatRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
CatRestApi.__init__(self, username, password, host, port, protocol)
@@ -87,7 +87,7 @@ class FileRestApi(CatRestApi):
# Testing.
if __name__ == '__main__':
- api = FileRestApi()
+ api = FileCatApi()
import time
t = long(time.time())
diff --git a/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
index ba84de0a..8117fde2 100755
--- a/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
+++ b/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -32,7 +32,7 @@ Description:
pairs are interpreted as dataset metadata.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
datasetInfo = self.splitArgsIntoDict()
datasetInfo['experimentName'] = self.getExperimentName()
datasetInfo['datasetName'] = self.getDatasetName()
diff --git a/src/python/dm/cat_web_service/cli/addExperimentFileCli.py b/src/python/dm/cat_web_service/cli/addExperimentFileCli.py
index 50447f2c..a4b0b726 100755
--- a/src/python/dm/cat_web_service/cli/addExperimentFileCli.py
+++ b/src/python/dm/cat_web_service/cli/addExperimentFileCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -32,7 +32,7 @@ Description:
are interpreted as file metadata.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileInfo = self.splitArgsIntoDict()
fileInfo['experimentName'] = self.getExperimentName()
fileInfo['experimentFilePath'] = self.getExperimentFilePath()
diff --git a/src/python/dm/cat_web_service/cli/getDatasetsCli.py b/src/python/dm/cat_web_service/cli/getDatasetsCli.py
index f8d0930a..cde69730 100755
--- a/src/python/dm/cat_web_service/cli/getDatasetsCli.py
+++ b/src/python/dm/cat_web_service/cli/getDatasetsCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -23,7 +23,7 @@ Description:
returned.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
queryDict = self.splitArgsIntoDict()
datasetMetadataList = api.getDatasets(queryDict)
for datasetMetadata in datasetMetadataList:
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
index f17035fc..8ecb1733 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -30,7 +30,7 @@ Description:
Retrieve experiment dataset metadata from the catalog.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
datasetMetadata = api.getExperimentDataset(self.getExperimentName(), self.getDatasetName())
print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
index 02e21867..0e6c1be7 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -31,7 +31,7 @@ Description:
Retrieve experiment dataset files from the catalog.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileMetadataList = api.getExperimentDatasetFiles(self.getExperimentName(), self.getDatasetName())
for fileMetadata in fileMetadataList:
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
index 3260932d..ecbc48cf 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -28,7 +28,7 @@ Description:
will be returned.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
queryDict = self.splitArgsIntoDict()
datasetMetadataList = api.getExperimentDatasets(self.getExperimentName(), queryDict)
for datasetMetadata in datasetMetadataList:
diff --git a/src/python/dm/cat_web_service/cli/getExperimentFileCli.py b/src/python/dm/cat_web_service/cli/getExperimentFileCli.py
index 63c30a58..c45279d2 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentFileCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentFileCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -30,7 +30,7 @@ Description:
Retrieve experiment file metadata from the catalog.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileMetadata = api.getExperimentFile(self.getExperimentName(), self.getExperimentFilePath())
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py b/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py
index 54b0d3fb..81fa6edd 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -27,7 +27,7 @@ Description:
metadata key/values are requested, all experiment files will be returned.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
queryDict = self.splitArgsIntoDict()
fileMetadataList = api.getExperimentFiles(self.getExperimentName(), queryDict)
for fileMetadata in fileMetadataList:
diff --git a/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
index 1098b393..336a4cce 100755
--- a/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
+++ b/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -35,7 +35,7 @@ Description:
new keys.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
datasetInfo = self.splitArgsIntoDict()
datasetInfo['experimentName'] = self.getExperimentName()
datasetInfo['datasetName'] = self.getDatasetName()
diff --git a/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py b/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py
index ea32cbf0..602b8651 100755
--- a/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py
+++ b/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -34,7 +34,7 @@ Description:
new keys.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileInfo = self.splitArgsIntoDict()
fileInfo['experimentName'] = self.getExperimentName()
fileInfo['experimentFilePath'] = self.getExperimentFilePath()
diff --git a/src/python/dm/common/objects/daqInfo.py b/src/python/dm/common/objects/daqInfo.py
index 0dc1c9f2..ae9a0d5b 100755
--- a/src/python/dm/common/objects/daqInfo.py
+++ b/src/python/dm/common/objects/daqInfo.py
@@ -51,6 +51,26 @@ class DaqInfo(DmObject):
finally:
self.lock.release()
+ def fileProcessingSkipped(self, processorName, filePath, processingError, processingEndTime):
+ self.lock.acquire()
+ try:
+ pluginStatsDict = self.get('pluginStats', {})
+ self['pluginStats'] = pluginStatsDict
+
+ pluginStats = pluginStatsDict.get(processorName, {})
+ pluginStatsDict[processorName] = pluginStats
+
+ pluginStats['nSkippedFiles'] = pluginStats.get('nSkippedFiles', 0) + 1
+ if processingError:
+ processingErrors = pluginStats.get('processingErrors', {})
+ processingErrors[filePath] = processingError
+ pluginStats['processingErrors'] = processingErrors
+ lastFileSkippedTime = pluginStats.get('lastFileSkippedTime', 0)
+ if processingEndTime is not None and processingEndTime > lastFileSkippedTime:
+ pluginStats['lastFileSkippedTime'] = processingEndTime
+ finally:
+ self.lock.release()
+
def updateStatus(self):
now = time.time()
daqStatus = self.get('status', dmProcessingStatus.DM_PROCESSING_STATUS_RUNNING)
diff --git a/src/python/dm/common/objects/uploadInfo.py b/src/python/dm/common/objects/uploadInfo.py
index 4c04ffa6..20c12f48 100755
--- a/src/python/dm/common/objects/uploadInfo.py
+++ b/src/python/dm/common/objects/uploadInfo.py
@@ -40,6 +40,26 @@ class UploadInfo(DmObject):
finally:
self.lock.release()
+ def fileProcessingSkipped(self, processorName, filePath, processingError, processingEndTime):
+ self.lock.acquire()
+ try:
+ pluginStatsDict = self.get('pluginStats', {})
+ self['pluginStats'] = pluginStatsDict
+
+ pluginStats = pluginStatsDict.get(processorName, {})
+ pluginStatsDict[processorName] = pluginStats
+
+ pluginStats['nSkippedFiles'] = pluginStats.get('nSkippedFiles', 0) + 1
+ if processingError:
+ processingErrors = pluginStats.get('processingErrors', {})
+ processingErrors[filePath] = processingError
+ pluginStats['processingErrors'] = processingErrors
+ lastFileSkippedTime = pluginStats.get('lastFileSkippedTime', 0)
+ if processingEndTime is not None and processingEndTime > lastFileSkippedTime:
+ pluginStats['lastFileSkippedTime'] = processingEndTime
+ finally:
+ self.lock.release()
+
def fileProcessingCancelled(self, filePath, processingEndTime):
self.lock.acquire()
try:
diff --git a/src/python/dm/common/processing/fileProcessingManager.py b/src/python/dm/common/processing/fileProcessingManager.py
index 97745477..43646748 100755
--- a/src/python/dm/common/processing/fileProcessingManager.py
+++ b/src/python/dm/common/processing/fileProcessingManager.py
@@ -63,8 +63,8 @@ class FileProcessingManager(threading.Thread,Singleton):
self.logger.debug('Creating file processor instance of class %s' % className)
fileProcessor = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.logger.debug('Configuring file processor %s' % fileProcessor)
- fileProcessor.setNumberOfRetries(self.defaultNumberOfRetries)
- fileProcessor.setRetryWaitPeriodInSeconds(self.defaultRetryWaitPeriodInSeconds)
+ fileProcessor.setNumberOfRetriesIfNotSet(self.defaultNumberOfRetries)
+ fileProcessor.setRetryWaitPeriodInSecondsIfNotSet(self.defaultRetryWaitPeriodInSeconds)
statUtilityObject = None
if statUtility:
statUtilityObject = ObjectUtility.createObjectInstance(statUtilityModuleName, statUtilityClassName, statUtilityConstructor)
diff --git a/src/python/dm/common/processing/fileProcessingThread.py b/src/python/dm/common/processing/fileProcessingThread.py
index 579e2094..9a16c6e1 100755
--- a/src/python/dm/common/processing/fileProcessingThread.py
+++ b/src/python/dm/common/processing/fileProcessingThread.py
@@ -89,10 +89,16 @@ class FileProcessingThread(threading.Thread):
processorDict['numberOfRetriesLeft'] = nRetriesLeft - 1
if nRetriesLeft <= 0:
endProcessingTime = time.time()
- if statusMonitor:
- statusMonitor.fileProcessingError(filePath, processingError, endProcessingTime)
- statusMonitor.updateStatus()
- self.logger.debug('No more %s retries left for file %s, remaining plugins will not process it' % (processorName, filePath))
+ if not processor.getSkipOnFailure():
+ if statusMonitor:
+ statusMonitor.fileProcessingError(filePath, processingError, endProcessingTime)
+ statusMonitor.updateStatus()
+ self.logger.debug('No more %s retries left for file %s, remaining plugins will not process it' % (processorName, filePath))
+ else:
+ if statusMonitor:
+ statusMonitor.fileProcessingSkipped(processorName, filePath, processingError, endProcessingTime)
+ statusMonitor.updateStatus()
+ self.logger.debug('No more %s retries left for file %s, skipping it' % (processorName, filePath))
return
else:
retryWaitPeriod = processor.getRetryWaitPeriodInSeconds()
diff --git a/src/python/dm/common/processing/plugins/fileProcessor.py b/src/python/dm/common/processing/plugins/fileProcessor.py
index 853452ff..0a4f22d0 100755
--- a/src/python/dm/common/processing/plugins/fileProcessor.py
+++ b/src/python/dm/common/processing/plugins/fileProcessor.py
@@ -38,15 +38,29 @@ class FileProcessor:
def getConfigKeyValue(self, key):
return self.configDict.get(key)
+ def setNumberOfRetriesIfNotSet(self, nRetries):
+ if not self.configDict.has_key('numberOfRetries'):
+ self.configDict['numberOfRetries'] = nRetries
+
def setNumberOfRetries(self, nRetries):
self.configDict['numberOfRetries'] = nRetries
def getNumberOfRetries(self):
return self.configDict.get('numberOfRetries', self.DEFAULT_NUMBER_OF_RETRIES)
+ def setRetryWaitPeriodInSecondsIfNotSet(self, waitPeriod):
+ if not self.configDict.has_key('retryWaitPeriodInSeconds'):
+ self.configDict['retryWaitPeriodInSeconds'] = waitPeriod
+
def setRetryWaitPeriodInSeconds(self, waitPeriod):
self.configDict['retryWaitPeriodInSeconds'] = waitPeriod
def getRetryWaitPeriodInSeconds(self):
return self.configDict.get('retryWaitPeriodInSeconds', self.DEFAULT_RETRY_WAIT_PERIOD_IN_SECONDS)
+ def setSkipOnFailure(self, skipOnFailure):
+ self.configDict['skipOnFailure'] = skipOnFailure
+
+ def getSkipOnFailure(self, skipOnFailure):
+ return self.configDict.get('skipOnFailure', False)
+
diff --git a/src/python/dm/common/processing/plugins/fileTransferPlugin.py b/src/python/dm/common/processing/plugins/fileTransferPlugin.py
index 0b2548bd..310962f6 100755
--- a/src/python/dm/common/processing/plugins/fileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/fileTransferPlugin.py
@@ -10,6 +10,8 @@ from fileProcessor import FileProcessor
class FileTransferPlugin(FileProcessor):
+ NOOP_COMMAND = '/bin/true'
+
def __init__(self, command, src=None, dest=None, dependsOn=[]):
FileProcessor.__init__(self, dependsOn)
self.src = src
@@ -92,7 +94,19 @@ class FileTransferPlugin(FileProcessor):
fileDest = self.replaceTemplates(fileDest, templateInfo)
if not fileSrc or not fileDest:
raise InvalidRequest('Both source and destination must be non-empty strings.')
- self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand(fileSrc, fileDest, command), cwd=cwd)
+ # Determine normalized directory path for source and dest, and
+ # skip transfer if they are the same
+ normSrcDir = os.path.normpath(os.path.dirname(fileSrc))
+ fileName = os.path.basename(fileSrc)
+ if fileDest.endswith(fileName):
+ normDestDir = os.path.normpath(os.path.dirname(fileDest))
+ else:
+ normDestDir = os.path.normpath(fileDest)
+ if normSrcDir == normDestDir:
+ self.logger.debug('Skipping file transfer %s -> %s, source and destination are the same.' % (fileSrc, fileDest))
+ self.subprocess = DmSubprocess.getSubprocess(self.NOOP_COMMAND)
+ else:
+ self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand(fileSrc, fileDest, command), cwd=cwd)
return self.subprocess.run()
def wait(self):
diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
index d42ec1f6..590e26d2 100755
--- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
@@ -19,14 +19,17 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
DEFAULT_PORT = 2811
- def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, directoryTransferCommand=DIRECTORY_TRANSFER_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[]):
+ def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, directoryTransferCommand=DIRECTORY_TRANSFER_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[], nRetries=None, skipOnFailure=False):
FileTransferPlugin.__init__(self, command, src, dest, dependsOn=dependsOn)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.localMd5Sum = localMd5Sum
self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal
self.directoryTransferCommand = directoryTransferCommand
self.pluginMustProcessFiles = pluginMustProcessFiles
+ if nRetries:
+ self.setNumberOfRetries(nRetries)
+ self.setSkipOnFailure(skipOnFailure)
def replaceSpecialCharacters(self, url):
replacementMap = {
@@ -55,9 +58,10 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
if self.dest:
destUrl = '%s/%s/%s' % (self.dest, dirName, fileName)
else:
- destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName)
- if targetDirectory:
- destUrl = '%s/%s/' % (destUrl, targetDirectory)
+ if targetDirectory:
+ destUrl = 'sshftp://%s/%s/%s/%s/%s' % (storageHost, storageDirectory, targetDirectory, dirName, fileName)
+ else:
+ destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName)
return self.replaceSpecialCharacters(destUrl)
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
diff --git a/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py b/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py
index dc211439..a85bfaf7 100755
--- a/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py
@@ -17,13 +17,16 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
DIRECTORY_TRANSFER_WITH_MKDIR_COMMAND = 'rsync -arvlP --rsync-path="mkdir -p %s && rsync" --'
DRY_RUN_COMMAND = 'rsync -arvlP --dry-run --'
- def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[]):
+ def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[], nRetries=None, skipOnFailure=False):
FileTransferPlugin.__init__(self, command, src, dest, dependsOn=dependsOn)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.localMd5Sum = localMd5Sum
self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal
self.pluginMustProcessFiles = pluginMustProcessFiles
+ if nRetries:
+ self.setNumberOfRetries(nRetries)
+ self.setSkipOnFailure(skipOnFailure)
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
if not self.pluginMustProcessFiles:
diff --git a/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py b/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py
index e206f090..049ddf47 100755
--- a/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py
@@ -14,7 +14,7 @@ class RsyncWithChecksumAndDeleteFileTransferPlugin(FileTransferPlugin):
def __init__(self, src=None, dest=None):
FileTransferPlugin.__init__(self, self.COMMAND, src, dest)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
diff --git a/src/python/dm/daq_web_service/api/experimentDaqApi.py b/src/python/dm/daq_web_service/api/experimentDaqApi.py
index db32234b..fc21edbf 100755
--- a/src/python/dm/daq_web_service/api/experimentDaqApi.py
+++ b/src/python/dm/daq_web_service/api/experimentDaqApi.py
@@ -63,6 +63,7 @@ class ExperimentDaqApi(DaqRestApi):
- *maxRunTimeInHours* (int): specifies maximum data acquisition run time in hours
- *uploadDataDirectoryOnExit* (str): specifies URL of the data directory that should be uploaded after data acquisition completes
- *uploadTargetDirectoryOnExit* (str): specifies directory path relative to experiment root directory where uploaded files should be stored
+ - *skipPlugins* (str): comma-separated list of plugins which should not process files
:type daqInfo: dict
@@ -194,7 +195,7 @@ class ExperimentDaqApi(DaqRestApi):
- *reprocessFiles* (bool): if set to True, files will be uploaded regardless of whether or not they already exist in storage and have not changed
- *targetDirectory* (str): specifies directory path relative to experiment root directory where files will be stored
- *processingMode* (str): specifies processing mode, and can be set to "files" (service plugins process individual files one at a time) or "directory" (service plugins process entire directory at once; works faster for uploads of a large number of small files)
- - *skipPlugins* (str): relevant for the "directory" processing mode; comma-separated list of plugins which should not process the given directory
+ - *skipPlugins* (str): comma-separated list of plugins which should not process files
:type daqInfo: dict
:returns: UploadInfo object
diff --git a/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py b/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py
index d76a617b..4607253e 100755
--- a/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py
+++ b/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py
@@ -10,7 +10,7 @@ class DsProcessFileNotificationPlugin(FileProcessor):
def __init__(self, dependsOn=[]):
FileProcessor.__init__(self, dependsOn=dependsOn)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
diff --git a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
index c9715918..09600f4f 100755
--- a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
+++ b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
@@ -40,7 +40,7 @@ class ExperimentSessionControllerImpl(DmObjectManager):
def __init__(self):
DmObjectManager.__init__(self)
- self.dsExperimentApi = DsRestApiFactory.getExperimentRestApi()
+ self.dsExperimentApi = DsRestApiFactory.getExperimentDsApi()
def startDaq(self, experimentName, dataDirectory, daqInfo):
FileSystemObserver.getInstance().createDirectory(dataDirectory)
diff --git a/src/python/dm/ds_web_service/api/authRestApi.py b/src/python/dm/ds_web_service/api/authDsApi.py
similarity index 95%
rename from src/python/dm/ds_web_service/api/authRestApi.py
rename to src/python/dm/ds_web_service/api/authDsApi.py
index 0996a14c..8207e107 100755
--- a/src/python/dm/ds_web_service/api/authRestApi.py
+++ b/src/python/dm/ds_web_service/api/authDsApi.py
@@ -11,7 +11,7 @@ from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.objects.dmSession import DmSession
from dsRestApi import DsRestApi
-class AuthRestApi(DsRestApi):
+class AuthDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
@@ -48,7 +48,7 @@ class AuthRestApi(DsRestApi):
# Testing.
if __name__ == '__main__':
- api = AuthRestApi('dm', 'dm', 'zagreb.svdev.net', 22236, 'http')
+ api = AuthDsApi('dm', 'dm', 'zagreb.svdev.net', 22236, 'http')
print api.authenticateUser('sveseli', 'sv')
diff --git a/src/python/dm/ds_web_service/api/dsRestApiFactory.py b/src/python/dm/ds_web_service/api/dsRestApiFactory.py
index 6b9e3dad..2b39f81b 100755
--- a/src/python/dm/ds_web_service/api/dsRestApiFactory.py
+++ b/src/python/dm/ds_web_service/api/dsRestApiFactory.py
@@ -36,31 +36,35 @@ class DsRestApiFactory:
return (cls.__username, cls.__password, cls.__host, cls.__port, cls.__protocol)
@classmethod
- def getUserRestApi(cls):
- from userRestApi import UserRestApi
+ def getUserDsApi(cls):
+ from userDsApi import UserDsApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = UserRestApi(username, password, host, port, protocol)
+ api = UserDsApi(username, password, host, port, protocol)
return api
@classmethod
- def getAuthRestApi(cls):
- from authRestApi import AuthRestApi
+ def getAuthDsApi(cls):
+ from authDsApi import AuthDsApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = AuthRestApi(username, password, host, port, protocol)
+ api = AuthDsApi(username, password, host, port, protocol)
return api
@classmethod
- def getExperimentRestApi(cls):
- from experimentRestApi import ExperimentRestApi
- (username, password, host, port, protocol) = cls.__getConfiguration()
- api = ExperimentRestApi(username, password, host, port, protocol)
+ def getExperimentDsApi(cls):
+ from experimentDsApi import ExperimentDsApi
+ try:
+ (username, password, host, port, protocol) = cls.__getConfiguration()
+ except Exception, ex:
+ print ex
+ raise
+ api = ExperimentDsApi(username, password, host, port, protocol)
return api
@classmethod
- def getFileRestApi(cls):
- from fileRestApi import FileRestApi
+ def getFileDsApi(cls):
+ from fileDsApi import FileDsApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = FileRestApi(username, password, host, port, protocol)
+ api = FileDsApi(username, password, host, port, protocol)
return api
####################################################################
# Testing
diff --git a/src/python/dm/ds_web_service/api/experimentRestApi.py b/src/python/dm/ds_web_service/api/experimentDsApi.py
similarity index 97%
rename from src/python/dm/ds_web_service/api/experimentRestApi.py
rename to src/python/dm/ds_web_service/api/experimentDsApi.py
index 78ff54e3..cf2314f3 100755
--- a/src/python/dm/ds_web_service/api/experimentRestApi.py
+++ b/src/python/dm/ds_web_service/api/experimentDsApi.py
@@ -9,7 +9,7 @@ from dm.common.objects.experiment import Experiment
from dm.common.objects.experimentType import ExperimentType
from dsRestApi import DsRestApi
-class ExperimentRestApi(DsRestApi):
+class ExperimentDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
@@ -102,6 +102,6 @@ class ExperimentRestApi(DsRestApi):
# Testing.
if __name__ == '__main__':
- api = ExperimentRestApi('sveseli', 'sveseli', 'zagreb.svdev.net', 33336, 'http')
+ api = ExperimentDsApi('sveseli', 'sveseli', 'zagreb.svdev.net', 33336, 'http')
print api.startExperiment('experiment1')
diff --git a/src/python/dm/ds_web_service/api/fileRestApi.py b/src/python/dm/ds_web_service/api/fileDsApi.py
similarity index 97%
rename from src/python/dm/ds_web_service/api/fileRestApi.py
rename to src/python/dm/ds_web_service/api/fileDsApi.py
index 1a00c3ed..549a097f 100755
--- a/src/python/dm/ds_web_service/api/fileRestApi.py
+++ b/src/python/dm/ds_web_service/api/fileDsApi.py
@@ -14,7 +14,7 @@ from dm.common.objects.experiment import Experiment
from dm.common.utility.rsyncFileTransfer import RsyncFileTransfer
from dsRestApi import DsRestApi
-class FileRestApi(DsRestApi):
+class FileDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
@@ -91,6 +91,6 @@ class FileRestApi(DsRestApi):
# Testing.
if __name__ == '__main__':
- api = FileRestApi('sveseli', 'sveseli', 'zagreb.svdev.net', 22236, 'http')
+ api = FileDsApi('sveseli', 'sveseli', 'zagreb.svdev.net', 22236, 'http')
print api.processFile('file1', '/ESAF/exp1', 'exp1')
diff --git a/src/python/dm/ds_web_service/api/userRestApi.py b/src/python/dm/ds_web_service/api/userDsApi.py
similarity index 99%
rename from src/python/dm/ds_web_service/api/userRestApi.py
rename to src/python/dm/ds_web_service/api/userDsApi.py
index 5834878f..d74e68a4 100755
--- a/src/python/dm/ds_web_service/api/userRestApi.py
+++ b/src/python/dm/ds_web_service/api/userDsApi.py
@@ -11,7 +11,7 @@ from dm.common.objects.userExperimentRole import UserExperimentRole
from dm.common.objects.userSystemRole import UserSystemRole
from dsRestApi import DsRestApi
-class UserRestApi(DsRestApi):
+class UserDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
diff --git a/src/python/dm/ds_web_service/cli/addExperimentCli.py b/src/python/dm/ds_web_service/cli/addExperimentCli.py
index 567b96e9..70c41059 100755
--- a/src/python/dm/ds_web_service/cli/addExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/addExperimentCli.py
@@ -1,8 +1,8 @@
#!/usr/bin/env python
from dm.aps_bss.api.apsBssApi import ApsBssApi
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.utility.configurationManager import ConfigurationManager
@@ -56,7 +56,7 @@ class AddExperimentCli(DsWebServiceSessionCli):
typeName = self.options.typeName
if not typeName:
if self.options.typeId:
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experimentType = api.getExperimentTypeById(typeId)
typeName = experimentType.get('name')
self.options.typeName = typeName
@@ -103,8 +103,8 @@ Description:
Add new experiment to the DM database. If list of users or proposal id is specified, this command will also add roles for all users listed on the proposal.
""")
self.checkArgs()
- dsExperimentApi = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
- dsUserApi = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ dsExperimentApi = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ dsUserApi = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
description = self.getDescription()
proposalId = self.getProposalId()
diff --git a/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py b/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py
index be374b8c..07a73670 100755
--- a/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class AddUserExperimentRoleCli(DsWebServiceSessionCli):
@@ -38,7 +38,7 @@ Description:
Assigns experiment role to the given user.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userExperimentRole = api.addUserExperimentRole(self.getUsername(), self.getRoleName(), self.getExperimentName())
print userExperimentRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py b/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py
index c42e6279..2f689784 100755
--- a/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class AddUserSystemRoleCli(DsWebServiceSessionCli):
@@ -36,7 +36,7 @@ Description:
Assigns system role to the given user. The 'Administrator' role cannot be accompanied with experiment station, while the 'Manager' role requires it.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userSystemRole = api.addUserSystemRole(self.getUsername(), self.getRoleName(), self.getExperimentStationName())
print userSystemRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py b/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py
index 15dae638..507990db 100755
--- a/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class DeleteUserExperimentRoleCli(DsWebServiceSessionCli):
@@ -38,7 +38,7 @@ Description:
Deletes experiment role from the given user.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userExperimentRole = api.deleteUserExperimentRole(self.getUsername(), self.getRoleName(), self.getExperimentName())
#print 'Deleted User Experiment Role:\n\t%s' % userExperimentRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py b/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py
index 85a3f32f..8f729370 100755
--- a/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class DeleteUserSystemRoleCli(DsWebServiceSessionCli):
@@ -36,7 +36,7 @@ Description:
Removes system role from the given user. The 'Administrator' role cannot be accompanied with experiment station, while the 'Manager' role requires it.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userSystemRole = api.deleteUserSystemRole(self.getUsername(), self.getRoleName(), self.getExperimentStationName())
#print userSystemRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/downloadCli.py b/src/python/dm/ds_web_service/cli/downloadCli.py
index fe9650e6..54064929 100755
--- a/src/python/dm/ds_web_service/cli/downloadCli.py
+++ b/src/python/dm/ds_web_service/cli/downloadCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.fileRestApi import FileRestApi
+from dm.ds_web_service.api.fileDsApi import FileDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class DownloadCli(DsWebServiceSessionCli):
@@ -34,7 +34,7 @@ Description:
Downloads experiment files.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
api.download(self.getExperimentName(), self.getExperimentFilePath(), self.getDestinationDirectory())
#print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/getExperimentCli.py b/src/python/dm/ds_web_service/cli/getExperimentCli.py
index c453f61d..1e5a6abd 100755
--- a/src/python/dm/ds_web_service/cli/getExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/getExperimentCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetExperimentCli(DsWebServiceSessionCli):
@@ -28,7 +28,7 @@ Description:
Retrieves experiment information.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
if self.getId() is not None:
experiment = api.getExperimentById(self.getId())
else:
diff --git a/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py b/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py
index 55313950..f2b7a8b6 100755
--- a/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py
+++ b/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetExperimentTypesCli(DsWebServiceSessionCli):
@@ -14,7 +14,7 @@ class GetExperimentTypesCli(DsWebServiceSessionCli):
Description:
Retrieves list of known experiment types.
""")
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experimentTypes = api.getExperimentTypes()
for experimentType in experimentTypes:
print experimentType.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/getExperimentsCli.py b/src/python/dm/ds_web_service/cli/getExperimentsCli.py
index 42afb78a..5aa05c31 100755
--- a/src/python/dm/ds_web_service/cli/getExperimentsCli.py
+++ b/src/python/dm/ds_web_service/cli/getExperimentsCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.utility.configurationManager import ConfigurationManager
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -23,7 +23,7 @@ class GetExperimentsCli(DsWebServiceSessionCli):
Description:
Retrieves list of experiments for a given station. If station name is not provided, this command will return list of experiments for all stations (requires administrator privileges).
""")
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
stationName = self.getStationName()
if not stationName:
experiments = api.getExperiments()
diff --git a/src/python/dm/ds_web_service/cli/getUserCli.py b/src/python/dm/ds_web_service/cli/getUserCli.py
index b45c34b5..27833ef4 100755
--- a/src/python/dm/ds_web_service/cli/getUserCli.py
+++ b/src/python/dm/ds_web_service/cli/getUserCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetUserCli(DsWebServiceSessionCli):
@@ -28,7 +28,7 @@ Description:
Retrieves user information.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
if self.getId() is not None:
userInfo = api.getUserById(self.getId())
else:
diff --git a/src/python/dm/ds_web_service/cli/getUsersCli.py b/src/python/dm/ds_web_service/cli/getUsersCli.py
index 8119fe0e..4fd94910 100755
--- a/src/python/dm/ds_web_service/cli/getUsersCli.py
+++ b/src/python/dm/ds_web_service/cli/getUsersCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetUsersCli(DsWebServiceSessionCli):
@@ -14,7 +14,7 @@ class GetUsersCli(DsWebServiceSessionCli):
Description:
Retrieves list of registered users.
""")
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
users = api.getUsers()
for user in users:
print user.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/startExperimentCli.py b/src/python/dm/ds_web_service/cli/startExperimentCli.py
index 905f8431..879d3bce 100755
--- a/src/python/dm/ds_web_service/cli/startExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/startExperimentCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -24,7 +24,7 @@ Description:
Updates experiment start time in the DM database and prepares experiment data directory.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.startExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/statExperimentFileCli.py b/src/python/dm/ds_web_service/cli/statExperimentFileCli.py
index f56798df..45affa4b 100755
--- a/src/python/dm/ds_web_service/cli/statExperimentFileCli.py
+++ b/src/python/dm/ds_web_service/cli/statExperimentFileCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.fileRestApi import FileRestApi
+from dm.ds_web_service.api.fileDsApi import FileDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class StatExperimentFileCli(DsWebServiceSessionCli):
@@ -30,7 +30,7 @@ Description:
Retrieves stat information for a given file.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileMetadata = api.statFile(self.getExperimentFilePath(), self.getExperimentName())
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/stopExperimentCli.py b/src/python/dm/ds_web_service/cli/stopExperimentCli.py
index 897a272c..f130587d 100755
--- a/src/python/dm/ds_web_service/cli/stopExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/stopExperimentCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -24,7 +24,7 @@ Description:
Updates experiment end date in the DM database and checks experiment data permissions.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.stopExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/updateExperimentCli.py b/src/python/dm/ds_web_service/cli/updateExperimentCli.py
index e162d69f..528e0b94 100755
--- a/src/python/dm/ds_web_service/cli/updateExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/updateExperimentCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -24,7 +24,7 @@ Description:
Updates experiment group users.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.updateExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py b/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py
index 9be90da8..86be58eb 100755
--- a/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py
+++ b/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py
@@ -11,7 +11,7 @@ class DsAuthPrincipalRetriever(AuthorizationPrincipalRetriever):
def __init__(self):
AuthorizationPrincipalRetriever.__init__(self, self.__class__.__name__)
- self.authApi = DsRestApiFactory.getAuthRestApi()
+ self.authApi = DsRestApiFactory.getAuthDsApi()
def getAuthorizationPrincipal(self, username):
principal = self.authApi.getAuthorizationPrincipal(username)
diff --git a/src/python/dm/ds_web_service/service/auth/dsSessionManager.py b/src/python/dm/ds_web_service/service/auth/dsSessionManager.py
index d2f78588..dcada2a9 100755
--- a/src/python/dm/ds_web_service/service/auth/dsSessionManager.py
+++ b/src/python/dm/ds_web_service/service/auth/dsSessionManager.py
@@ -11,7 +11,7 @@ class DsSessionManager(SessionManager):
def __init__(self):
SessionManager.__init__(self, self.__class__.__name__)
- self.authApi = DsRestApiFactory.getAuthRestApi()
+ self.authApi = DsRestApiFactory.getAuthDsApi()
def addSession(self, sessionId, sessionInfo):
return self.authApi.addSession(sessionId, sessionInfo)
--
GitLab