diff --git a/doc/rhel7_package_list.txt b/doc/rhel7_package_list.txt
index 3750753be7a3c7f76903b72be861e0e5c5d6a5b2..e200a3c86eb912e22f3d81a55924eec4f9215848 100644
--- a/doc/rhel7_package_list.txt
+++ b/doc/rhel7_package_list.txt
@@ -1,16 +1,46 @@
-- make
-- autoconf
-- expect
-- gcc
-- g++
-- subversion
-- zlib-devel
-- openssl-devel
-- libffi-devel
-- openldap-devel
-- readline-devel
-- ncurses-devel
-- qt-x11
-- qt-postgresql
-- qt-devel
-- gtk2-devel
+RHEL7 Packages
+===============
+
+make
+autoconf
+expect
+gcc
+g++
+subversion
+zlib-devel
+openssl-devel
+libffi-devel
+openldap-devel
+readline-devel
+ncurses-devel
+qt-x11
+qt-postgresql
+qt-devel
+gtk2-devel
+
+Globus Packages
+===============
+
+globus-openssl-module
+globus-ftp-client
+globus-gsi-proxy
+globus-gsi-openssl
+globus-ftp-control
+globus-gass-copy
+globus-common-16.8
+globus-gass-transfer
+globus-io-11.8
+globus-gss-assist
+globus-gsi-cert
+globus-xio-popen
+globus-xio-gsi
+globus-gsi-credential
+globus-callout-3.15
+globus-gsi-sysconfig
+globus-gsi-callback
+globus-xio-5.14
+globus-gssapi-gsi
+globus-gass-copy
+globus-gsi-proxy
+globus-gssapi-error
+
diff --git a/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py b/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py
index 6c7c347c75dbedcc65dadc2105a42d1f4a062fe7..3cc3a7c621d7f835ab3fcaf321a0dac17b2be8e8 100755
--- a/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py
+++ b/src/python/dm/aps_user_db/cli/updateUsersFromApsDbCli.py
@@ -54,7 +54,7 @@ Description:
try:
if self.options.useDmRestApi:
self.logger.debug('Using DM REST API')
- dmUserApi = DsRestApiFactory.getUserRestApi()
+ dmUserApi = DsRestApiFactory.getUserDsApi()
else:
self.logger.debug('Using DM DB API')
dmUserApi = UserDbApi()
diff --git a/src/python/dm/cat_web_service/api/catRestApiFactory.py b/src/python/dm/cat_web_service/api/catRestApiFactory.py
index b578e11ef57a3b6b9e9e7423c2beea78ad3b8245..64b30b6acd6d66a9d3f8749add7f3a824cfefbd9 100755
--- a/src/python/dm/cat_web_service/api/catRestApiFactory.py
+++ b/src/python/dm/cat_web_service/api/catRestApiFactory.py
@@ -36,10 +36,10 @@ class CatRestApiFactory:
return (cls.__username, cls.__password, cls.__host, cls.__port, cls.__protocol)
@classmethod
- def getFileRestApi(cls):
- from userRestApi import FileRestApi
+ def getFileCatApi(cls):
+ from userCatApi import FileCatApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = FileRestApi(username, password, host, port, protocol)
+ api = FileCatApi(username, password, host, port, protocol)
return api
####################################################################
diff --git a/src/python/dm/cat_web_service/api/datasetRestApi.py b/src/python/dm/cat_web_service/api/datasetCatApi.py
similarity index 98%
rename from src/python/dm/cat_web_service/api/datasetRestApi.py
rename to src/python/dm/cat_web_service/api/datasetCatApi.py
index f75254f079b334dc2887acddd0fe0fd644b5e310..d028c36a2f31f8ed31cb8f22a897801005dfb454 100755
--- a/src/python/dm/cat_web_service/api/datasetRestApi.py
+++ b/src/python/dm/cat_web_service/api/datasetCatApi.py
@@ -11,7 +11,7 @@ from dm.common.objects.datasetMetadata import DatasetMetadata
from dm.common.objects.fileMetadata import FileMetadata
from catRestApi import CatRestApi
-class DatasetRestApi(CatRestApi):
+class DatasetCatApi(CatRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
CatRestApi.__init__(self, username, password, host, port, protocol)
@@ -100,7 +100,7 @@ class DatasetRestApi(CatRestApi):
# Testing.
if __name__ == '__main__':
- api = DatasetRestApi()
+ api = DatasetCatApi()
print api.getDatasets()
print api.getDatasetById('556de0059e058b0ef4c4413b')
print api.getDatasetByName('xyz-001')
diff --git a/src/python/dm/cat_web_service/api/fileRestApi.py b/src/python/dm/cat_web_service/api/fileCatApi.py
similarity index 98%
rename from src/python/dm/cat_web_service/api/fileRestApi.py
rename to src/python/dm/cat_web_service/api/fileCatApi.py
index 75ec147fe1f5906a157b438eb187358f9139353f..08658d1f717118bc4898d7855c242b2b5f55b7e5 100755
--- a/src/python/dm/cat_web_service/api/fileRestApi.py
+++ b/src/python/dm/cat_web_service/api/fileCatApi.py
@@ -10,7 +10,7 @@ from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.objects.fileMetadata import FileMetadata
from catRestApi import CatRestApi
-class FileRestApi(CatRestApi):
+class FileCatApi(CatRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
CatRestApi.__init__(self, username, password, host, port, protocol)
@@ -87,7 +87,7 @@ class FileRestApi(CatRestApi):
# Testing.
if __name__ == '__main__':
- api = FileRestApi()
+ api = FileCatApi()
import time
t = long(time.time())
diff --git a/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
index ba84de0a91dea42d7bbba6081d2e44630e26a07f..8117fde2b022b0f0d004261b642f0c30bf84aa25 100755
--- a/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
+++ b/src/python/dm/cat_web_service/cli/addExperimentDatasetCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -32,7 +32,7 @@ Description:
pairs are interpreted as dataset metadata.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
datasetInfo = self.splitArgsIntoDict()
datasetInfo['experimentName'] = self.getExperimentName()
datasetInfo['datasetName'] = self.getDatasetName()
diff --git a/src/python/dm/cat_web_service/cli/addExperimentFileCli.py b/src/python/dm/cat_web_service/cli/addExperimentFileCli.py
index 50447f2c3b01e810ac87ea47c09d8c8184efd1cd..a4b0b726c5f533d0c02ecf4c1fd6e78a769eb3ed 100755
--- a/src/python/dm/cat_web_service/cli/addExperimentFileCli.py
+++ b/src/python/dm/cat_web_service/cli/addExperimentFileCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -32,7 +32,7 @@ Description:
are interpreted as file metadata.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileInfo = self.splitArgsIntoDict()
fileInfo['experimentName'] = self.getExperimentName()
fileInfo['experimentFilePath'] = self.getExperimentFilePath()
diff --git a/src/python/dm/cat_web_service/cli/getDatasetsCli.py b/src/python/dm/cat_web_service/cli/getDatasetsCli.py
index f8d0930ad1672803486304e8b5f9940be820b6b5..cde69730cc3051258d9f447707f2f1cec650cf27 100755
--- a/src/python/dm/cat_web_service/cli/getDatasetsCli.py
+++ b/src/python/dm/cat_web_service/cli/getDatasetsCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -23,7 +23,7 @@ Description:
returned.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
queryDict = self.splitArgsIntoDict()
datasetMetadataList = api.getDatasets(queryDict)
for datasetMetadata in datasetMetadataList:
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
index f17035fc8d8c9afd480822e5cac5a3d315db5210..8ecb1733e3725081dc5e6c9c15152d8595f2a25b 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -30,7 +30,7 @@ Description:
Retrieve experiment dataset metadata from the catalog.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
datasetMetadata = api.getExperimentDataset(self.getExperimentName(), self.getDatasetName())
print datasetMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
index 02e21867c1fb0b25dba66d909f702bbb728aacfe..0e6c1be7a71c0a862ddcf2c3ac3280955db64eb0 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetFilesCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -31,7 +31,7 @@ Description:
Retrieve experiment dataset files from the catalog.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileMetadataList = api.getExperimentDatasetFiles(self.getExperimentName(), self.getDatasetName())
for fileMetadata in fileMetadataList:
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py b/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
index 3260932d5b266783a8718759dd8d65ad67b84a98..ecbc48cf3aed8a1d6562efc3f7cc319f4b6a81df 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentDatasetsCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -28,7 +28,7 @@ Description:
will be returned.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
queryDict = self.splitArgsIntoDict()
datasetMetadataList = api.getExperimentDatasets(self.getExperimentName(), queryDict)
for datasetMetadata in datasetMetadataList:
diff --git a/src/python/dm/cat_web_service/cli/getExperimentFileCli.py b/src/python/dm/cat_web_service/cli/getExperimentFileCli.py
index 63c30a58f0e5557c8f4bb49a4a248b1e4d35fd7e..c45279d2fae7abb0ea54ccd101b351779d777132 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentFileCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentFileCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -30,7 +30,7 @@ Description:
Retrieve experiment file metadata from the catalog.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileMetadata = api.getExperimentFile(self.getExperimentName(), self.getExperimentFilePath())
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py b/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py
index 54b0d3fb51506a8d6de96f55614602df70739dec..81fa6edd523c24d52c326fb15820c2dd9dd58376 100755
--- a/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py
+++ b/src/python/dm/cat_web_service/cli/getExperimentFilesCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -27,7 +27,7 @@ Description:
metadata key/values are requested, all experiment files will be returned.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
queryDict = self.splitArgsIntoDict()
fileMetadataList = api.getExperimentFiles(self.getExperimentName(), queryDict)
for fileMetadata in fileMetadataList:
diff --git a/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py b/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
index 1098b393093b8f16f0c9ff11bf595e208cfea849..336a4cce2ef00ccd4c8649cc11ea5bc76d16a1c7 100755
--- a/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
+++ b/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.datasetRestApi import DatasetRestApi
+from dm.cat_web_service.api.datasetCatApi import DatasetCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -35,7 +35,7 @@ Description:
new keys.
""")
self.checkArgs()
- api = DatasetRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = DatasetCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
datasetInfo = self.splitArgsIntoDict()
datasetInfo['experimentName'] = self.getExperimentName()
datasetInfo['datasetName'] = self.getDatasetName()
diff --git a/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py b/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py
index ea32cbf0c292fc9ca42953c37a25d523ab11ef8e..602b86510fba31fc71082066f6e7158d639d63dc 100755
--- a/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py
+++ b/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.cat_web_service.api.fileRestApi import FileRestApi
+from dm.cat_web_service.api.fileCatApi import FileCatApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from catWebServiceSessionCli import CatWebServiceSessionCli
@@ -34,7 +34,7 @@ Description:
new keys.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileCatApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileInfo = self.splitArgsIntoDict()
fileInfo['experimentName'] = self.getExperimentName()
fileInfo['experimentFilePath'] = self.getExperimentFilePath()
diff --git a/src/python/dm/common/objects/daqInfo.py b/src/python/dm/common/objects/daqInfo.py
index 0dc1c9f2fa88454163969105746551d41d651b65..ae9a0d5bc3a7449bc2596b0dd15a348e040f5a91 100755
--- a/src/python/dm/common/objects/daqInfo.py
+++ b/src/python/dm/common/objects/daqInfo.py
@@ -51,6 +51,26 @@ class DaqInfo(DmObject):
finally:
self.lock.release()
+ def fileProcessingSkipped(self, processorName, filePath, processingError, processingEndTime):
+ self.lock.acquire()
+ try:
+ pluginStatsDict = self.get('pluginStats', {})
+ self['pluginStats'] = pluginStatsDict
+
+ pluginStats = pluginStatsDict.get(processorName, {})
+ pluginStatsDict[processorName] = pluginStats
+
+ pluginStats['nSkippedFiles'] = pluginStats.get('nSkippedFiles', 0) + 1
+ if processingError:
+ processingErrors = pluginStats.get('processingErrors', {})
+ processingErrors[filePath] = processingError
+ pluginStats['processingErrors'] = processingErrors
+ lastFileSkippedTime = pluginStats.get('lastFileSkippedTime', 0)
+ if processingEndTime is not None and processingEndTime > lastFileSkippedTime:
+ pluginStats['lastFileSkippedTime'] = processingEndTime
+ finally:
+ self.lock.release()
+
def updateStatus(self):
now = time.time()
daqStatus = self.get('status', dmProcessingStatus.DM_PROCESSING_STATUS_RUNNING)
diff --git a/src/python/dm/common/objects/uploadInfo.py b/src/python/dm/common/objects/uploadInfo.py
index 4c04ffa67304c1aead1098814e54d878e5bf45ce..20c12f4840a97b4b16ddd12d645b462b0be047ab 100755
--- a/src/python/dm/common/objects/uploadInfo.py
+++ b/src/python/dm/common/objects/uploadInfo.py
@@ -40,6 +40,26 @@ class UploadInfo(DmObject):
finally:
self.lock.release()
+ def fileProcessingSkipped(self, processorName, filePath, processingError, processingEndTime):
+ self.lock.acquire()
+ try:
+ pluginStatsDict = self.get('pluginStats', {})
+ self['pluginStats'] = pluginStatsDict
+
+ pluginStats = pluginStatsDict.get(processorName, {})
+ pluginStatsDict[processorName] = pluginStats
+
+ pluginStats['nSkippedFiles'] = pluginStats.get('nSkippedFiles', 0) + 1
+ if processingError:
+ processingErrors = pluginStats.get('processingErrors', {})
+ processingErrors[filePath] = processingError
+ pluginStats['processingErrors'] = processingErrors
+ lastFileSkippedTime = pluginStats.get('lastFileSkippedTime', 0)
+ if processingEndTime is not None and processingEndTime > lastFileSkippedTime:
+ pluginStats['lastFileSkippedTime'] = processingEndTime
+ finally:
+ self.lock.release()
+
def fileProcessingCancelled(self, filePath, processingEndTime):
self.lock.acquire()
try:
diff --git a/src/python/dm/common/processing/fileProcessingManager.py b/src/python/dm/common/processing/fileProcessingManager.py
index 97745477db34bf59fd7f7b20174820e778450035..43646748dce45e4ba1a25fcd06892918fa421b1c 100755
--- a/src/python/dm/common/processing/fileProcessingManager.py
+++ b/src/python/dm/common/processing/fileProcessingManager.py
@@ -63,8 +63,8 @@ class FileProcessingManager(threading.Thread,Singleton):
self.logger.debug('Creating file processor instance of class %s' % className)
fileProcessor = ObjectUtility.createObjectInstance(moduleName, className, constructor)
self.logger.debug('Configuring file processor %s' % fileProcessor)
- fileProcessor.setNumberOfRetries(self.defaultNumberOfRetries)
- fileProcessor.setRetryWaitPeriodInSeconds(self.defaultRetryWaitPeriodInSeconds)
+ fileProcessor.setNumberOfRetriesIfNotSet(self.defaultNumberOfRetries)
+ fileProcessor.setRetryWaitPeriodInSecondsIfNotSet(self.defaultRetryWaitPeriodInSeconds)
statUtilityObject = None
if statUtility:
statUtilityObject = ObjectUtility.createObjectInstance(statUtilityModuleName, statUtilityClassName, statUtilityConstructor)
diff --git a/src/python/dm/common/processing/fileProcessingThread.py b/src/python/dm/common/processing/fileProcessingThread.py
index 579e20942bb27f0f82f931bd65c46605cff28d01..9a16c6e1887d7b5ea3b7247d63842c6371ce3dda 100755
--- a/src/python/dm/common/processing/fileProcessingThread.py
+++ b/src/python/dm/common/processing/fileProcessingThread.py
@@ -89,10 +89,16 @@ class FileProcessingThread(threading.Thread):
processorDict['numberOfRetriesLeft'] = nRetriesLeft - 1
if nRetriesLeft <= 0:
endProcessingTime = time.time()
- if statusMonitor:
- statusMonitor.fileProcessingError(filePath, processingError, endProcessingTime)
- statusMonitor.updateStatus()
- self.logger.debug('No more %s retries left for file %s, remaining plugins will not process it' % (processorName, filePath))
+ if not processor.getSkipOnFailure():
+ if statusMonitor:
+ statusMonitor.fileProcessingError(filePath, processingError, endProcessingTime)
+ statusMonitor.updateStatus()
+ self.logger.debug('No more %s retries left for file %s, remaining plugins will not process it' % (processorName, filePath))
+ else:
+ if statusMonitor:
+ statusMonitor.fileProcessingSkipped(processorName, filePath, processingError, endProcessingTime)
+ statusMonitor.updateStatus()
+ self.logger.debug('No more %s retries left for file %s, skipping it' % (processorName, filePath))
return
else:
retryWaitPeriod = processor.getRetryWaitPeriodInSeconds()
diff --git a/src/python/dm/common/processing/plugins/fileProcessor.py b/src/python/dm/common/processing/plugins/fileProcessor.py
index 853452ffa5f1542acb584442b5867aa9de4c26a8..0a4f22d0ab30dfc4881e45e4add63adebbf5e1ea 100755
--- a/src/python/dm/common/processing/plugins/fileProcessor.py
+++ b/src/python/dm/common/processing/plugins/fileProcessor.py
@@ -38,15 +38,29 @@ class FileProcessor:
def getConfigKeyValue(self, key):
return self.configDict.get(key)
+ def setNumberOfRetriesIfNotSet(self, nRetries):
+ if not self.configDict.has_key('numberOfRetries'):
+ self.configDict['numberOfRetries'] = nRetries
+
def setNumberOfRetries(self, nRetries):
self.configDict['numberOfRetries'] = nRetries
def getNumberOfRetries(self):
return self.configDict.get('numberOfRetries', self.DEFAULT_NUMBER_OF_RETRIES)
+ def setRetryWaitPeriodInSecondsIfNotSet(self, waitPeriod):
+ if not self.configDict.has_key('retryWaitPeriodInSeconds'):
+ self.configDict['retryWaitPeriodInSeconds'] = waitPeriod
+
def setRetryWaitPeriodInSeconds(self, waitPeriod):
self.configDict['retryWaitPeriodInSeconds'] = waitPeriod
def getRetryWaitPeriodInSeconds(self):
return self.configDict.get('retryWaitPeriodInSeconds', self.DEFAULT_RETRY_WAIT_PERIOD_IN_SECONDS)
+ def setSkipOnFailure(self, skipOnFailure):
+ self.configDict['skipOnFailure'] = skipOnFailure
+
+ def getSkipOnFailure(self, skipOnFailure):
+ return self.configDict.get('skipOnFailure', False)
+
diff --git a/src/python/dm/common/processing/plugins/fileTransferPlugin.py b/src/python/dm/common/processing/plugins/fileTransferPlugin.py
index 0b2548bd97efe49adf250993b3ab4bc17ade4708..310962f60d727c0be35aeb53dd854816a5a69c84 100755
--- a/src/python/dm/common/processing/plugins/fileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/fileTransferPlugin.py
@@ -10,6 +10,8 @@ from fileProcessor import FileProcessor
class FileTransferPlugin(FileProcessor):
+ NOOP_COMMAND = '/bin/true'
+
def __init__(self, command, src=None, dest=None, dependsOn=[]):
FileProcessor.__init__(self, dependsOn)
self.src = src
@@ -92,7 +94,19 @@ class FileTransferPlugin(FileProcessor):
fileDest = self.replaceTemplates(fileDest, templateInfo)
if not fileSrc or not fileDest:
raise InvalidRequest('Both source and destination must be non-empty strings.')
- self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand(fileSrc, fileDest, command), cwd=cwd)
+ # Determine normalized directory path for source and dest, and
+ # skip transfer if they are the same
+ normSrcDir = os.path.normpath(os.path.dirname(fileSrc))
+ fileName = os.path.basename(fileSrc)
+ if fileDest.endswith(fileName):
+ normDestDir = os.path.normpath(os.path.dirname(fileDest))
+ else:
+ normDestDir = os.path.normpath(fileDest)
+ if normSrcDir == normDestDir:
+ self.logger.debug('Skipping file transfer %s -> %s, source and destination are the same.' % (fileSrc, fileDest))
+ self.subprocess = DmSubprocess.getSubprocess(self.NOOP_COMMAND)
+ else:
+ self.subprocess = DmSubprocess.getSubprocess(self.getFullCommand(fileSrc, fileDest, command), cwd=cwd)
return self.subprocess.run()
def wait(self):
diff --git a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
index d42ec1f62a30e20d7ede90edda28154e2ddeefd4..590e26d2d0b6be94031d8ea184e46c8467bee0f9 100755
--- a/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/gridftpFileTransferPlugin.py
@@ -19,14 +19,17 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
DEFAULT_PORT = 2811
- def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, directoryTransferCommand=DIRECTORY_TRANSFER_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[]):
+ def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, directoryTransferCommand=DIRECTORY_TRANSFER_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[], nRetries=None, skipOnFailure=False):
FileTransferPlugin.__init__(self, command, src, dest, dependsOn=dependsOn)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.localMd5Sum = localMd5Sum
self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal
self.directoryTransferCommand = directoryTransferCommand
self.pluginMustProcessFiles = pluginMustProcessFiles
+ if nRetries:
+ self.setNumberOfRetries(nRetries)
+ self.setSkipOnFailure(skipOnFailure)
def replaceSpecialCharacters(self, url):
replacementMap = {
@@ -55,9 +58,10 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
if self.dest:
destUrl = '%s/%s/%s' % (self.dest, dirName, fileName)
else:
- destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName)
- if targetDirectory:
- destUrl = '%s/%s/' % (destUrl, targetDirectory)
+ if targetDirectory:
+ destUrl = 'sshftp://%s/%s/%s/%s/%s' % (storageHost, storageDirectory, targetDirectory, dirName, fileName)
+ else:
+ destUrl = 'sshftp://%s/%s/%s/%s' % (storageHost, storageDirectory, dirName, fileName)
return self.replaceSpecialCharacters(destUrl)
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
diff --git a/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py b/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py
index dc211439a7830a76d7b6c418339b96f8a71a15c4..a85bfaf7165ebc20a23235ec35911427b3dd2a27 100755
--- a/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/rsyncFileTransferPlugin.py
@@ -17,13 +17,16 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
DIRECTORY_TRANSFER_WITH_MKDIR_COMMAND = 'rsync -arvlP --rsync-path="mkdir -p %s && rsync" --'
DRY_RUN_COMMAND = 'rsync -arvlP --dry-run --'
- def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[]):
+ def __init__(self, src=None, dest=None, command=DEFAULT_COMMAND, localMd5Sum=True, remoteMd5Sum=False, deleteOriginal=False, pluginMustProcessFiles=True, dependsOn=[], nRetries=None, skipOnFailure=False):
FileTransferPlugin.__init__(self, command, src, dest, dependsOn=dependsOn)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.localMd5Sum = localMd5Sum
self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal
self.pluginMustProcessFiles = pluginMustProcessFiles
+ if nRetries:
+ self.setNumberOfRetries(nRetries)
+ self.setSkipOnFailure(skipOnFailure)
def checkUploadFilesForProcessing(self, filePathsDict, uploadInfo):
if not self.pluginMustProcessFiles:
diff --git a/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py b/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py
index e206f0906ea38ad105d72205c95cfabd07c5a62a..049ddf4762d03f57ca4a6ef8894fe356460f444a 100755
--- a/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py
+++ b/src/python/dm/common/processing/plugins/rsyncWithChecksumAndDeleteFileTransferPlugin.py
@@ -14,7 +14,7 @@ class RsyncWithChecksumAndDeleteFileTransferPlugin(FileTransferPlugin):
def __init__(self, src=None, dest=None):
FileTransferPlugin.__init__(self, self.COMMAND, src, dest)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
diff --git a/src/python/dm/daq_web_service/api/experimentDaqApi.py b/src/python/dm/daq_web_service/api/experimentDaqApi.py
index db32234b4cf0a32ceb1e7846c05c67231ed975ae..fc21edbfc5ee3991ce4d56353f8f526b56eff68b 100755
--- a/src/python/dm/daq_web_service/api/experimentDaqApi.py
+++ b/src/python/dm/daq_web_service/api/experimentDaqApi.py
@@ -63,6 +63,7 @@ class ExperimentDaqApi(DaqRestApi):
- *maxRunTimeInHours* (int): specifies maximum data acquisition run time in hours
- *uploadDataDirectoryOnExit* (str): specifies URL of the data directory that should be uploaded after data acquisition completes
- *uploadTargetDirectoryOnExit* (str): specifies directory path relative to experiment root directory where uploaded files should be stored
+ - *skipPlugins* (str): comma-separated list of plugins which should not process files
:type daqInfo: dict
@@ -194,7 +195,7 @@ class ExperimentDaqApi(DaqRestApi):
- *reprocessFiles* (bool): if set to True, files will be uploaded regardless of whether or not they already exist in storage and have not changed
- *targetDirectory* (str): specifies directory path relative to experiment root directory where files will be stored
- *processingMode* (str): specifies processing mode, and can be set to "files" (service plugins process individual files one at a time) or "directory" (service plugins process entire directory at once; works faster for uploads of a large number of small files)
- - *skipPlugins* (str): relevant for the "directory" processing mode; comma-separated list of plugins which should not process the given directory
+ - *skipPlugins* (str): comma-separated list of plugins which should not process files
:type daqInfo: dict
:returns: UploadInfo object
diff --git a/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py b/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py
index d76a617b97c9d03b0e5633577ba2dd3aff7b572e..4607253e8263f29646ee6749fd02f968c7c5e600 100755
--- a/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py
+++ b/src/python/dm/daq_web_service/service/impl/dsProcessFileNotificationPlugin.py
@@ -10,7 +10,7 @@ class DsProcessFileNotificationPlugin(FileProcessor):
def __init__(self, dependsOn=[]):
FileProcessor.__init__(self, dependsOn=dependsOn)
- self.dsFileApi = DsRestApiFactory.getFileRestApi()
+ self.dsFileApi = DsRestApiFactory.getFileDsApi()
self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
def processFile(self, fileInfo):
diff --git a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
index c97159188edc908a918026d96e948278330c65e3..09600f4f6d2ee9bb1ab2e0d1cc2464268b9a541c 100755
--- a/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
+++ b/src/python/dm/daq_web_service/service/impl/experimentSessionControllerImpl.py
@@ -40,7 +40,7 @@ class ExperimentSessionControllerImpl(DmObjectManager):
def __init__(self):
DmObjectManager.__init__(self)
- self.dsExperimentApi = DsRestApiFactory.getExperimentRestApi()
+ self.dsExperimentApi = DsRestApiFactory.getExperimentDsApi()
def startDaq(self, experimentName, dataDirectory, daqInfo):
FileSystemObserver.getInstance().createDirectory(dataDirectory)
diff --git a/src/python/dm/ds_web_service/api/authRestApi.py b/src/python/dm/ds_web_service/api/authDsApi.py
similarity index 95%
rename from src/python/dm/ds_web_service/api/authRestApi.py
rename to src/python/dm/ds_web_service/api/authDsApi.py
index 0996a14cf905de7bf61c80739f84e4a6ace4afca..8207e1070a1a07cb981e7d628658856615b7d066 100755
--- a/src/python/dm/ds_web_service/api/authRestApi.py
+++ b/src/python/dm/ds_web_service/api/authDsApi.py
@@ -11,7 +11,7 @@ from dm.common.objects.authorizationPrincipal import AuthorizationPrincipal
from dm.common.objects.dmSession import DmSession
from dsRestApi import DsRestApi
-class AuthRestApi(DsRestApi):
+class AuthDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
@@ -48,7 +48,7 @@ class AuthRestApi(DsRestApi):
# Testing.
if __name__ == '__main__':
- api = AuthRestApi('dm', 'dm', 'zagreb.svdev.net', 22236, 'http')
+ api = AuthDsApi('dm', 'dm', 'zagreb.svdev.net', 22236, 'http')
print api.authenticateUser('sveseli', 'sv')
diff --git a/src/python/dm/ds_web_service/api/dsRestApiFactory.py b/src/python/dm/ds_web_service/api/dsRestApiFactory.py
index 6b9e3dadbcc42bbf51f64c42c7310d45c3f265f1..2b39f81bdaa50463299ba946541bbfdd9e627362 100755
--- a/src/python/dm/ds_web_service/api/dsRestApiFactory.py
+++ b/src/python/dm/ds_web_service/api/dsRestApiFactory.py
@@ -36,31 +36,35 @@ class DsRestApiFactory:
return (cls.__username, cls.__password, cls.__host, cls.__port, cls.__protocol)
@classmethod
- def getUserRestApi(cls):
- from userRestApi import UserRestApi
+ def getUserDsApi(cls):
+ from userDsApi import UserDsApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = UserRestApi(username, password, host, port, protocol)
+ api = UserDsApi(username, password, host, port, protocol)
return api
@classmethod
- def getAuthRestApi(cls):
- from authRestApi import AuthRestApi
+ def getAuthDsApi(cls):
+ from authDsApi import AuthDsApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = AuthRestApi(username, password, host, port, protocol)
+ api = AuthDsApi(username, password, host, port, protocol)
return api
@classmethod
- def getExperimentRestApi(cls):
- from experimentRestApi import ExperimentRestApi
- (username, password, host, port, protocol) = cls.__getConfiguration()
- api = ExperimentRestApi(username, password, host, port, protocol)
+ def getExperimentDsApi(cls):
+ from experimentDsApi import ExperimentDsApi
+ try:
+ (username, password, host, port, protocol) = cls.__getConfiguration()
+ except Exception, ex:
+ print ex
+ raise
+ api = ExperimentDsApi(username, password, host, port, protocol)
return api
@classmethod
- def getFileRestApi(cls):
- from fileRestApi import FileRestApi
+ def getFileDsApi(cls):
+ from fileDsApi import FileDsApi
(username, password, host, port, protocol) = cls.__getConfiguration()
- api = FileRestApi(username, password, host, port, protocol)
+ api = FileDsApi(username, password, host, port, protocol)
return api
####################################################################
# Testing
diff --git a/src/python/dm/ds_web_service/api/experimentRestApi.py b/src/python/dm/ds_web_service/api/experimentDsApi.py
similarity index 97%
rename from src/python/dm/ds_web_service/api/experimentRestApi.py
rename to src/python/dm/ds_web_service/api/experimentDsApi.py
index 78ff54e34fcf53fa5346da30fd958d1b73f76080..cf2314f3a3385613911c8810835f42db020c309f 100755
--- a/src/python/dm/ds_web_service/api/experimentRestApi.py
+++ b/src/python/dm/ds_web_service/api/experimentDsApi.py
@@ -9,7 +9,7 @@ from dm.common.objects.experiment import Experiment
from dm.common.objects.experimentType import ExperimentType
from dsRestApi import DsRestApi
-class ExperimentRestApi(DsRestApi):
+class ExperimentDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
@@ -102,6 +102,6 @@ class ExperimentRestApi(DsRestApi):
# Testing.
if __name__ == '__main__':
- api = ExperimentRestApi('sveseli', 'sveseli', 'zagreb.svdev.net', 33336, 'http')
+ api = ExperimentDsApi('sveseli', 'sveseli', 'zagreb.svdev.net', 33336, 'http')
print api.startExperiment('experiment1')
diff --git a/src/python/dm/ds_web_service/api/fileRestApi.py b/src/python/dm/ds_web_service/api/fileDsApi.py
similarity index 97%
rename from src/python/dm/ds_web_service/api/fileRestApi.py
rename to src/python/dm/ds_web_service/api/fileDsApi.py
index 1a00c3ed03bd824a10871fa1e7a75aaebd2e0571..549a097f71bea84e81bdbf64af8c9a3538be3ae0 100755
--- a/src/python/dm/ds_web_service/api/fileRestApi.py
+++ b/src/python/dm/ds_web_service/api/fileDsApi.py
@@ -14,7 +14,7 @@ from dm.common.objects.experiment import Experiment
from dm.common.utility.rsyncFileTransfer import RsyncFileTransfer
from dsRestApi import DsRestApi
-class FileRestApi(DsRestApi):
+class FileDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
@@ -91,6 +91,6 @@ class FileRestApi(DsRestApi):
# Testing.
if __name__ == '__main__':
- api = FileRestApi('sveseli', 'sveseli', 'zagreb.svdev.net', 22236, 'http')
+ api = FileDsApi('sveseli', 'sveseli', 'zagreb.svdev.net', 22236, 'http')
print api.processFile('file1', '/ESAF/exp1', 'exp1')
diff --git a/src/python/dm/ds_web_service/api/userRestApi.py b/src/python/dm/ds_web_service/api/userDsApi.py
similarity index 99%
rename from src/python/dm/ds_web_service/api/userRestApi.py
rename to src/python/dm/ds_web_service/api/userDsApi.py
index 5834878f4dba5e3b6ee74a2c60e3e58502ec2de6..d74e68a409b590c25625a3aa06444264acf7e87b 100755
--- a/src/python/dm/ds_web_service/api/userRestApi.py
+++ b/src/python/dm/ds_web_service/api/userDsApi.py
@@ -11,7 +11,7 @@ from dm.common.objects.userExperimentRole import UserExperimentRole
from dm.common.objects.userSystemRole import UserSystemRole
from dsRestApi import DsRestApi
-class UserRestApi(DsRestApi):
+class UserDsApi(DsRestApi):
def __init__(self, username=None, password=None, host=None, port=None, protocol=None):
DsRestApi.__init__(self, username, password, host, port, protocol)
diff --git a/src/python/dm/ds_web_service/cli/addExperimentCli.py b/src/python/dm/ds_web_service/cli/addExperimentCli.py
index 567b96e9b645871ffda480e0c43c2e3c2a99f3ef..70c410592a5df6ea57833e49eaeab1eef1cd074d 100755
--- a/src/python/dm/ds_web_service/cli/addExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/addExperimentCli.py
@@ -1,8 +1,8 @@
#!/usr/bin/env python
from dm.aps_bss.api.apsBssApi import ApsBssApi
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dm.common.utility.configurationManager import ConfigurationManager
@@ -56,7 +56,7 @@ class AddExperimentCli(DsWebServiceSessionCli):
typeName = self.options.typeName
if not typeName:
if self.options.typeId:
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experimentType = api.getExperimentTypeById(typeId)
typeName = experimentType.get('name')
self.options.typeName = typeName
@@ -103,8 +103,8 @@ Description:
Add new experiment to the DM database. If list of users or proposal id is specified, this command will also add roles for all users listed on the proposal.
""")
self.checkArgs()
- dsExperimentApi = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
- dsUserApi = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ dsExperimentApi = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ dsUserApi = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
description = self.getDescription()
proposalId = self.getProposalId()
diff --git a/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py b/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py
index be374b8cd11ff1d0f12a051f86032efea296a7cb..07a7367063180021099c9fc5721a666cea06129b 100755
--- a/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/addUserExperimentRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class AddUserExperimentRoleCli(DsWebServiceSessionCli):
@@ -38,7 +38,7 @@ Description:
Assigns experiment role to the given user.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userExperimentRole = api.addUserExperimentRole(self.getUsername(), self.getRoleName(), self.getExperimentName())
print userExperimentRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py b/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py
index c42e62796d5079607cfa16673819e55d2522ec39..2f68978463cfac9f79e934a032f1bbc3cbef2c14 100755
--- a/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/addUserSystemRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class AddUserSystemRoleCli(DsWebServiceSessionCli):
@@ -36,7 +36,7 @@ Description:
Assigns system role to the given user. The 'Administrator' role cannot be accompanied with experiment station, while the 'Manager' role requires it.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userSystemRole = api.addUserSystemRole(self.getUsername(), self.getRoleName(), self.getExperimentStationName())
print userSystemRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py b/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py
index 15dae638de56c71f62e2a6f9ed5dbaf5bf06744d..507990db1b3c72b98043713302a587426729d9f0 100755
--- a/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/deleteUserExperimentRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class DeleteUserExperimentRoleCli(DsWebServiceSessionCli):
@@ -38,7 +38,7 @@ Description:
Deletes experiment role from the given user.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userExperimentRole = api.deleteUserExperimentRole(self.getUsername(), self.getRoleName(), self.getExperimentName())
#print 'Deleted User Experiment Role:\n\t%s' % userExperimentRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py b/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py
index 85a3f32f783c1555ef9a76629c27aaa36dc6b15e..8f729370f9d49cf23654c612b83150fd39fe8754 100755
--- a/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py
+++ b/src/python/dm/ds_web_service/cli/deleteUserSystemRoleCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class DeleteUserSystemRoleCli(DsWebServiceSessionCli):
@@ -36,7 +36,7 @@ Description:
Removes system role from the given user. The 'Administrator' role cannot be accompanied with experiment station, while the 'Manager' role requires it.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
userSystemRole = api.deleteUserSystemRole(self.getUsername(), self.getRoleName(), self.getExperimentStationName())
#print userSystemRole.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/downloadCli.py b/src/python/dm/ds_web_service/cli/downloadCli.py
index fe9650e6b9d4ed2dc60b8c3a7de025ba3f2234e6..54064929a86bc9df35019239437757ac565897fd 100755
--- a/src/python/dm/ds_web_service/cli/downloadCli.py
+++ b/src/python/dm/ds_web_service/cli/downloadCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.fileRestApi import FileRestApi
+from dm.ds_web_service.api.fileDsApi import FileDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class DownloadCli(DsWebServiceSessionCli):
@@ -34,7 +34,7 @@ Description:
Downloads experiment files.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
api.download(self.getExperimentName(), self.getExperimentFilePath(), self.getDestinationDirectory())
#print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/getExperimentCli.py b/src/python/dm/ds_web_service/cli/getExperimentCli.py
index c453f61d91108854cbeecd507970bc898b4f2010..1e5a6abdea44cf458ef072b8f85f98bc81fb5545 100755
--- a/src/python/dm/ds_web_service/cli/getExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/getExperimentCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetExperimentCli(DsWebServiceSessionCli):
@@ -28,7 +28,7 @@ Description:
Retrieves experiment information.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
if self.getId() is not None:
experiment = api.getExperimentById(self.getId())
else:
diff --git a/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py b/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py
index 55313950506a570e63cc7ebf61cc6c58c2671a55..f2b7a8b62c7460598ad9e91f1d4b6db7b4072e87 100755
--- a/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py
+++ b/src/python/dm/ds_web_service/cli/getExperimentTypesCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetExperimentTypesCli(DsWebServiceSessionCli):
@@ -14,7 +14,7 @@ class GetExperimentTypesCli(DsWebServiceSessionCli):
Description:
Retrieves list of known experiment types.
""")
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experimentTypes = api.getExperimentTypes()
for experimentType in experimentTypes:
print experimentType.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/getExperimentsCli.py b/src/python/dm/ds_web_service/cli/getExperimentsCli.py
index 42afb78a1ac8ff4f0187c03784c50e982ba15aae..5aa05c313f24306ab9f3fc03a5c3685e0c0e773f 100755
--- a/src/python/dm/ds_web_service/cli/getExperimentsCli.py
+++ b/src/python/dm/ds_web_service/cli/getExperimentsCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.utility.configurationManager import ConfigurationManager
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -23,7 +23,7 @@ class GetExperimentsCli(DsWebServiceSessionCli):
Description:
Retrieves list of experiments for a given station. If station name is not provided, this command will return list of experiments for all stations (requires administrator privileges).
""")
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
stationName = self.getStationName()
if not stationName:
experiments = api.getExperiments()
diff --git a/src/python/dm/ds_web_service/cli/getUserCli.py b/src/python/dm/ds_web_service/cli/getUserCli.py
index b45c34b5b0a1b2c9d59f392449284e6d1b0c48cf..27833ef48695132a0c7a9e7264ea28c5783505b9 100755
--- a/src/python/dm/ds_web_service/cli/getUserCli.py
+++ b/src/python/dm/ds_web_service/cli/getUserCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetUserCli(DsWebServiceSessionCli):
@@ -28,7 +28,7 @@ Description:
Retrieves user information.
""")
self.checkArgs()
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
if self.getId() is not None:
userInfo = api.getUserById(self.getId())
else:
diff --git a/src/python/dm/ds_web_service/cli/getUsersCli.py b/src/python/dm/ds_web_service/cli/getUsersCli.py
index 8119fe0eb66c878180223e3cc09fab4a3e24f429..4fd94910d947d72ada603231dace77bcb60e811d 100755
--- a/src/python/dm/ds_web_service/cli/getUsersCli.py
+++ b/src/python/dm/ds_web_service/cli/getUsersCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.userRestApi import UserRestApi
+from dm.ds_web_service.api.userDsApi import UserDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class GetUsersCli(DsWebServiceSessionCli):
@@ -14,7 +14,7 @@ class GetUsersCli(DsWebServiceSessionCli):
Description:
Retrieves list of registered users.
""")
- api = UserRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = UserDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
users = api.getUsers()
for user in users:
print user.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/startExperimentCli.py b/src/python/dm/ds_web_service/cli/startExperimentCli.py
index 905f843106c19f9d37f143c11efdc5b0d72a8c5d..879d3bcee9a9e5dbe20fc225d2874410debedf3a 100755
--- a/src/python/dm/ds_web_service/cli/startExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/startExperimentCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -24,7 +24,7 @@ Description:
Updates experiment start time in the DM database and prepares experiment data directory.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.startExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/statExperimentFileCli.py b/src/python/dm/ds_web_service/cli/statExperimentFileCli.py
index f56798dfcf7c0837535b7f00a0569ac268cef46e..45affa4b8bf20981749cd15fa6f9e9688d17aa1c 100755
--- a/src/python/dm/ds_web_service/cli/statExperimentFileCli.py
+++ b/src/python/dm/ds_web_service/cli/statExperimentFileCli.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from dm.common.exceptions.invalidRequest import InvalidRequest
-from dm.ds_web_service.api.fileRestApi import FileRestApi
+from dm.ds_web_service.api.fileDsApi import FileDsApi
from dsWebServiceSessionCli import DsWebServiceSessionCli
class StatExperimentFileCli(DsWebServiceSessionCli):
@@ -30,7 +30,7 @@ Description:
Retrieves stat information for a given file.
""")
self.checkArgs()
- api = FileRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = FileDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
fileMetadata = api.statFile(self.getExperimentFilePath(), self.getExperimentName())
print fileMetadata.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/stopExperimentCli.py b/src/python/dm/ds_web_service/cli/stopExperimentCli.py
index 897a272c42d98d9f886b7beb896115a6f5889959..f130587de83d2a85c5aecc0e0e1ca1459c5fd0cc 100755
--- a/src/python/dm/ds_web_service/cli/stopExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/stopExperimentCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -24,7 +24,7 @@ Description:
Updates experiment end date in the DM database and checks experiment data permissions.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.stopExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/cli/updateExperimentCli.py b/src/python/dm/ds_web_service/cli/updateExperimentCli.py
index e162d69fb1a9f913e84385053bc1fb9721efb7e8..528e0b94e127beedacd59164b5981486c2922102 100755
--- a/src/python/dm/ds_web_service/cli/updateExperimentCli.py
+++ b/src/python/dm/ds_web_service/cli/updateExperimentCli.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-from dm.ds_web_service.api.experimentRestApi import ExperimentRestApi
+from dm.ds_web_service.api.experimentDsApi import ExperimentDsApi
from dm.common.exceptions.invalidRequest import InvalidRequest
from dsWebServiceSessionCli import DsWebServiceSessionCli
@@ -24,7 +24,7 @@ Description:
Updates experiment group users.
""")
self.checkArgs()
- api = ExperimentRestApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
+ api = ExperimentDsApi(self.getLoginUsername(), self.getLoginPassword(), self.getServiceHost(), self.getServicePort(), self.getServiceProtocol())
experiment = api.updateExperiment(self.getExperimentName())
print experiment.getDisplayString(self.getDisplayKeys(), self.getDisplayFormat())
diff --git a/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py b/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py
index 9be90da85435a6a93943307b5f9c1820610beef8..86be58ebc88a79e0f1e68100b3d3128bfa8f5adb 100755
--- a/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py
+++ b/src/python/dm/ds_web_service/service/auth/dsAuthPrincipalRetriever.py
@@ -11,7 +11,7 @@ class DsAuthPrincipalRetriever(AuthorizationPrincipalRetriever):
def __init__(self):
AuthorizationPrincipalRetriever.__init__(self, self.__class__.__name__)
- self.authApi = DsRestApiFactory.getAuthRestApi()
+ self.authApi = DsRestApiFactory.getAuthDsApi()
def getAuthorizationPrincipal(self, username):
principal = self.authApi.getAuthorizationPrincipal(username)
diff --git a/src/python/dm/ds_web_service/service/auth/dsSessionManager.py b/src/python/dm/ds_web_service/service/auth/dsSessionManager.py
index d2f785885f46f73eddc893198c9221421c6e4066..dcada2a90cf3575d830fa09f76e1e1770d4a4ac4 100755
--- a/src/python/dm/ds_web_service/service/auth/dsSessionManager.py
+++ b/src/python/dm/ds_web_service/service/auth/dsSessionManager.py
@@ -11,7 +11,7 @@ class DsSessionManager(SessionManager):
def __init__(self):
SessionManager.__init__(self, self.__class__.__name__)
- self.authApi = DsRestApiFactory.getAuthRestApi()
+ self.authApi = DsRestApiFactory.getAuthDsApi()
def addSession(self, sessionId, sessionInfo):
return self.authApi.addSession(sessionId, sessionInfo)