add file processing interfaces; rename storage manager to experiment manager;...

add file processing interfaces; rename storage manager to experiment manager; add ability to manage groups/file permissions

src/python/dm/ds_web_service/service/impl/storageManager.py → src/python/dm/ds_web_service/service/impl/experimentManager.py

@@ -8,38 +8,55 @@ from dm.common.utility.loggingManager import LoggingManager
 from dm.common.utility.configurationManager import ConfigurationManager
 from dm.common.utility.singleton import Singleton
 from dm.common.utility.osUtility import OsUtility
+from dm.common.utility.valueUtility import ValueUtility
+from dm.common.utility.objectUtility import ObjectUtility
 from dm.common.utility.threadingUtility import ThreadingUtility
+from dm.common.db.api.experimentDbApi import ExperimentDbApi
 
+class ExperimentManager(Singleton):
 
-class StorageManager(Singleton):
-
-    CONFIG_SECTION_NAME = 'StorageManager'
+    CONFIG_SECTION_NAME = 'ExperimentManager'
     STORAGE_DIRECTORY_KEY = 'storagedirectory'
+    MANAGE_STORAGE_PERMISSIONS_KEY = 'managestoragepermissions'
+    PLATFORM_UTILITY_KEY = 'platformutility'
+
+    FILE_PERMISSIONS_MODE = 0640
+    DIR_PERMISSIONS_MODE = 0750
 
     # Singleton.
     __instanceLock = threading.RLock()
     __instance = None
 
     def __init__(self):
-        StorageManager.__instanceLock.acquire()
+        ExperimentManager.__instanceLock.acquire()
         try:
-            if StorageManager.__instance:
+            if ExperimentManager.__instance:
                 return
-            StorageManager.__instance = self
+            ExperimentManager.__instance = self
             self.logger = LoggingManager.getInstance().getLogger(self.__class__.__name__)
 
             self.logger.debug('Initializing')
             self.lock = threading.RLock()
+            self.experimentDbApi = ExperimentDbApi()
+            self.platformUtility = None
             self.__configure()
             self.logger.debug('Initialization complete')
         finally:
-            StorageManager.__instanceLock.release()
+            ExperimentManager.__instanceLock.release()
 
     def __configure(self):
         cm = ConfigurationManager.getInstance()
-        configItems = cm.getConfigItems(StorageManager.CONFIG_SECTION_NAME)
+        configItems = cm.getConfigItems(ExperimentManager.CONFIG_SECTION_NAME)
         self.logger.debug('Got config items: %s' % configItems)
-        self.storageDirectory = cm.getConfigOption(StorageManager.CONFIG_SECTION_NAME, StorageManager.STORAGE_DIRECTORY_KEY)
+        self.storageDirectory =cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.STORAGE_DIRECTORY_KEY) 
+        self.manageStoragePermissions = ValueUtility.toBoolean(cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.MANAGE_STORAGE_PERMISSIONS_KEY))
+        platformUtility = cm.getConfigOption(ExperimentManager.CONFIG_SECTION_NAME, ExperimentManager.PLATFORM_UTILITY_KEY)
+        if platformUtility:
+            (moduleName,className,constructor) = cm.getModuleClassConstructorTuple(platformUtility)
+            self.logger.debug('Creating platform utility class %s' % className)
+            self.platformUtility = ObjectUtility.createObjectInstance(moduleName, className, constructor)
+                                                                            
+        self.logger.debug('Manage storage permissions: %s' % self.manageStoragePermissions)
 
     def __getExperimentStorageDataDirectory(self, experiment):
         experimentTypeName = experiment.get('experimentType').get('rootDataPath')
@@ -67,19 +84,38 @@ class StorageManager(Singleton):
             OsUtility.createDir(storageDirectory)
         experiment['storageDirectory'] = storageDirectory
         experiment['storageHost'] = ConfigurationManager.getInstance().getHost()
+        if self.manageStoragePermissions:
+            # Create experiment group
+            self.platformUtility.createGroup(experimentName)
+            self.logger.debug('Setting permissions for %s to %s' % (storageDirectory, self.DIR_PERMISSIONS_MODE))
+            OsUtility.chmodPath(storageDirectory, dirMode=self.DIR_PERMISSIONS_MODE)
+            experimentUsers = experiment.get('experimentUsernameList', [])
+            for username in experimentUsers:
+                self.platformUtility.addUserToGroup(username, experimentName)
       
+    @ThreadingUtility.synchronize
+    def processExperimentFile(self, fileName, filePath, experiment):
+        experimentName = experiment.get('name')
+        if os.path.exists(filePath):
+            self.logger.debug('Processing file %s' % filePath)
+            if self.manageStoragePermissions:
+                self.logger.debug('Modifying permissions for %s' % filePath)
+                OsUtility.chmodPath(filePath, fileMode=self.FILE_PERMISSIONS_MODE)
+        else:
+            self.logger.debug('File path %s does not exist' % filePath)
+
     @ThreadingUtility.synchronize
     def start(self):
-        self.logger.debug('Started storage manager')
+        self.logger.debug('Started experiment manager')
 
     @ThreadingUtility.synchronize
     def stop(self):
-        self.logger.debug('Stopped storage manager')
+        self.logger.debug('Stopped experiment manager')
 
 ####################################################################
 # Testing
 
 if __name__ == '__main__':
-    sm = StorageManager.getInstance()
-    print sm
+    em = ExperimentManager.getInstance()
+    print em
 

src/python/dm/ds_web_service/service/impl/experimentSessionControllerImpl.py

@@ -10,7 +10,7 @@ from dm.common.objects.experiment import Experiment
 from dm.common.objects.dmObjectManager import DmObjectManager
 from dm.common.exceptions.invalidRequest import InvalidRequest
 from dm.common.db.api.experimentDbApi import ExperimentDbApi
-from dm.ds_web_service.service.impl.storageManager import StorageManager
+from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
 
 class ExperimentSessionControllerImpl(DmObjectManager):
     """ Experiment session controller implementation class. """
@@ -29,12 +29,12 @@ class ExperimentSessionControllerImpl(DmObjectManager):
 
     def getExperimentByName(self, name):
         experiment = self.experimentDbApi.getExperimentByName(name)
-        StorageManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
+        ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
         return experiment
 
     def getExperimentById(self, id):
         experiment = self.experimentDbApi.getExperimentById(id)
-        StorageManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
+        ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
         return experiment
 
     def addExperiment(self, name, experimentTypeId, description):
@@ -42,18 +42,16 @@ class ExperimentSessionControllerImpl(DmObjectManager):
         return experiment
 
     def startExperiment(self, name):
-        experiment = self.experimentDbApi.getExperimentByName(name)
+        experiment = self.experimentDbApi.getExperimentWithUsers(name)
         if experiment.get('startDate') is None:
             experiment = self.experimentDbApi.setExperimentStartDateToNow(name)
-            StorageManager.getInstance().createExperimentDataDirectory(experiment)
-        else:
-            StorageManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
+        ExperimentManager.getInstance().createExperimentDataDirectory(experiment)
         return experiment
 
     def stopExperiment(self, name):
         experiment = self.experimentDbApi.getExperimentByName(name)
         if experiment.get('endDate') is None:
             experiment = self.experimentDbApi.setExperimentEndDateToNow(name)
-        StorageManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
+        ExperimentManager.getInstance().updateExperimentWithStorageDataDirectory(experiment)
         return experiment
 

src/python/dm/ds_web_service/service/impl/fileSessionControllerImpl.py

+#!/usr/bin/env python
+
+#
+# Implementation for file session controller.
+#
+
+import time
+
+from dm.common.objects.dmObjectManager import DmObjectManager
+from dm.common.exceptions.invalidRequest import InvalidRequest
+from dm.common.objects.fileMetadata import FileMetadata
+from dm.common.db.api.experimentDbApi import ExperimentDbApi
+from dm.ds_web_service.service.impl.experimentManager import ExperimentManager
+
+class FileSessionControllerImpl(DmObjectManager):
+    """ File session controller implementation class. """
+
+    def __init__(self):
+        DmObjectManager.__init__(self)
+        self.experimentDbApi = ExperimentDbApi()
+
+    def processFile(self, fileName, filePath, experimentName, **kwargs):
+        experiment = self.experimentDbApi.getExperimentByName(experimentName)
+        ExperimentManager.getInstance().processExperimentFile(fileName, filePath, experiment)
+        return FileMetadata({'fileName' : fileName})