Merge branch 'master' of https://git.aps.anl.gov/hammonds/dm.git

src/python/dm/common/mongodb/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/mongodb/api/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/mongodb/impl/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/objects/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/processing/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/processing/plugins/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/processing/plugins/cdbProcessingPlugin.py

@@ -110,7 +110,7 @@ class CdbProcessingPlugin(FileProcessor):
                 self.__cdbLock.release()
 
             # Add metadata
-            fileUrl = 'dm://extrepid/%s/%s' % (experimentName, fileName)
+            fileUrl = 'dm://voyager/%s/%s' % (experimentName, fileName)
 
             fileProperty = self.itemRestApi.addPropertyValueToItemWithId(itemId=fileItem['id'],propertyTypeName=self.CDB_FILE_PROPERTY_TYPE, tag=None, value=fileUrl, units=None,description=None,isUserWriteable=None, isDynamic=None)
             fileMetadata = fileInfo.get('metadata', {})

src/python/dm/common/processing/plugins/mongoDbFileCatalogPlugin.py

@@ -63,8 +63,8 @@ class MongoDbFileCatalogPlugin(FileProcessor):
         self.logger.debug('Before releasing Mongo DB API instance semaphore count is %s ' % (self.mongoApiSemaphore.__dict__.get('_Semaphore__value')))
         self.mongoApiSemaphore.release()
 
-    def getInitialCollectionSize(self, fileMongoDbApi, experimentName, uploadOrDaqId):
-        key = '%s-%s' % (experimentName, uploadOrDaqId)
+    def getInitialCollectionSize(self, fileMongoDbApi, experimentName, uploadId):
+        key = '%s-%s' % (experimentName, uploadId)
         self.lock.acquire()
         try:
             nFiles = self.initialCollectionSizeMap.get(key)
@@ -78,35 +78,35 @@ class MongoDbFileCatalogPlugin(FileProcessor):
     def processFile(self, fileInfo):
         experimentFilePath = fileInfo.get('experimentFilePath')
         experimentName = fileInfo.get('experimentName')
-        uploadOrDaqId = fileInfo.get('uploadId') or fileInfo.get('daqId') 
+        uploadId = fileInfo.get('uploadId') 
         self.logger.debug('Processing file "%s" for experiment %s' % (experimentFilePath, experimentName))
 
         fileMetadata = self.metadataCollector.processMetadata(fileInfo, self.statUtility)
         self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileMetadata)))
         fileInfo['metadata'] = fileMetadata
         fileMongoDbApi = self.acquireMongoApi()
-        if self.getInitialCollectionSize(fileMongoDbApi, experimentName, uploadOrDaqId) == 0:
-            self.logger.debug('Initial collection size is zero, adding file %s' % (experimentFilePath))
+        if uploadId and self.getInitialCollectionSize(fileMongoDbApi, experimentName, uploadId) == 0:
+            self.logger.debug('Initial collection size is zero for upload %s, adding file %s' % (uploadId, experimentFilePath))
             fileMongoDbApi.addExperimentFileUnchecked(fileMetadata)
         else:
-            self.logger.debug('Initial collection size is not zero, updating file %s' % (experimentFilePath))
+            self.logger.debug('Updating file %s' % (experimentFilePath))
             fileMongoDbApi.updateOrAddExperimentFile(fileMetadata)
         self.releaseMongoApi(fileMongoDbApi)
 
     def processFile2(self, fileInfo, fileMongoDbApi):
         experimentFilePath = fileInfo.get('experimentFilePath')
         experimentName = fileInfo.get('experimentName')
-        uploadOrDaqId = fileInfo.get('uploadId') or fileInfo.get('daqId') 
+        uploadId = fileInfo.get('uploadId') 
         self.logger.debug('Processing file "%s" for experiment %s' % (experimentFilePath, experimentName))
 
         fileMetadata = self.metadataCollector.processMetadata(fileInfo, self.statUtility)
         self.logger.debug('File "%s" catalog entry: %s' % (experimentFilePath, str(fileMetadata)))
         fileInfo['metadata'] = fileMetadata
-        if self.getInitialCollectionSize(fileMongoDbApi, experimentName, uploadOrDaqId) == 0:
-            self.logger.debug('Initial collection size is zero, adding file %s' % (experimentFilePath))
+        if uploadId and self.getInitialCollectionSize(fileMongoDbApi, experimentName, uploadId) == 0:
+            self.logger.debug('Initial collection size is zero for upload %s, adding file %s' % (uploadId, experimentFilePath))
             fileMongoDbApi.addExperimentFileUnchecked(fileMetadata)
         else:
-            self.logger.debug('Initial collection size is not zero, updating file %s' % (experimentFilePath))
+            self.logger.debug('Updating file %s' % (experimentFilePath))
             fileMongoDbApi.updateOrAddExperimentFile(fileMetadata)
 
     def processFilePath(self, fileMongoDbApi, filePath, filePathDict, experiment, dataDirectory, destDirectory, daqInfo, uploadId, processDirectoryInfo):
@@ -162,3 +162,8 @@ class MongoDbFileCatalogPlugin(FileProcessor):
             processingTimer = threading.Timer(self.PROCESSING_TIMER_DELAY_PERIOD, self.processFilePath, args=[fileMongoDbApi, filePath, filePathDict, experiment, dataDirectory, destDirectory, daqInfo, uploadId, processDirectoryInfo])
             processingTimer.start()
                    
+
+#######################################################################
+# Testing.
+if __name__ == '__main__':
+    pass 

src/python/dm/common/service/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/service/auth/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/utility/.svnignore

-*.pyc
-__pycache__

src/python/dm/common/zeromq/.svnignore

-*.pyc
-__pycache__

src/python/dm/daq_web_service/.svnignore

-*.pyc
-__pycache__

src/python/dm/daq_web_service/api/.svnignore

-*.pyc
-__pycache__

src/python/dm/daq_web_service/cli/.svnignore

-*.pyc
-__pycache__

src/python/dm/daq_web_service/service/.svnignore

-*.pyc
-__pycache__

src/python/dm/daq_web_service/service/impl/.svnignore

-*.pyc
-__pycache__

src/python/dm/ds_web_service/.svnignore

-*.pyc
-__pycache__

src/python/dm/ds_web_service/api/.svnignore

-*.pyc
-__pycache__

src/python/dm/ds_web_service/api/experimentDsApi.py

@@ -376,7 +376,7 @@ class ExperimentDsApi(DsRestApi):
         response = self.sendSessionRequest(url=url, method='POST', rawOutput=True)
         fileName = os.path.basename(experimentFilePath)
         filePath = os.path.join(destDirectory, fileName)
-        open(filePath, 'w').write(response)
+        open(filePath, 'w').write(str(response))
         return FileMetadata({'experimentName' : experimentName, 'experimentFilePath' : experimentFilePath, 'fileName' : fileName, 'localFilePath' : filePath})