Skip to content
Snippets Groups Projects
Commit bd9e4685 authored by sveseli's avatar sveseli
Browse files

merge from 1.1

parents 5a849756 74c9f02e
No related branches found
No related tags found
No related merge requests found
setup.csh 0 → 100644
#!/bin/csh
# DM setup script for C-type shells
set tmpFile=/tmp/setup.$$
lsof +p $$ >& $tmpFile
set myDir=`cat $tmpFile | grep setup.csh | grep -v dm.setup | sed 's?(.*??g' | awk '{print $NF}'`
set myDir=`dirname $myDir`
rm -f $tmpFile
set currentDir=`pwd`
cd $myDir
if ( $?DM_ROOT_DIR ) then
if ( "$DM_ROOT_DIR" != `pwd` ) then
echo "WARNING: Resetting DM_ROOT_DIR environment variable (old value: $DM_ROOT_DIR)"
endif
endif
setenv DM_ROOT_DIR `pwd`
setenv DM_HOST_ARCH `uname | tr \[A-Z\] \[a-z\]`-`uname -m`
if ( ! $?DM_INSTALL_DIR ) then
setenv DM_INSTALL_DIR $DM_ROOT_DIR/..
if ( -d $DM_INSTALL_DIR ) then
cd $DM_INSTALL_DIR
setenv DM_INSTALL_DIR `pwd`
endif
endif
if ( ! $?DM_VAR_DIR ) then
setenv DM_VAR_DIR $DM_ROOT_DIR/../var
if ( -d $DM_VAR_DIR ) then
cd $DM_VAR_DIR
setenv DM_VAR_DIR `pwd`
endif
endif
# Check support setup
if ( ! $?DM_SUPPORT_DIR ) then
setenv DM_SUPPORT_DIR $DM_ROOT_DIR/../support
if ( -d $DM_SUPPORT_DIR ) then
cd $DM_SUPPORT_DIR
setenv DM_SUPPORT_DIR `pwd`
endif
endif
if ( ! -d $DM_SUPPORT_DIR ) then
echo "ERROR: $DM_SUPPORT_DIR directory does not exist. Developers should point DM_SUPPORT_DIR to the desired area."
exit 1
endif
setenv DM_OPT_DIR $DM_SUPPORT_DIR/opt
setenv DM_GLASSFISH_DIR $DM_OPT_DIR/glassfish
# Setup path and other variables
setenv PATH ${DM_OPT_DIR}/node/bin:${PATH}
setenv PATH ${DM_OPT_DIR}/mongodb/${DM_HOST_ARCH}/bin:${PATH}
setenv PATH ${DM_OPT_DIR}/postgresql/${DM_HOST_ARCH}/bin:${PATH}
setenv PATH ${DM_OPT_DIR}/java/${DM_HOST_ARCH}/bin:${PATH}
setenv PATH ${DM_OPT_DIR}/ant/bin:${PATH}
setenv PATH ${DM_ROOT_DIR}/bin:${PATH}
if ( $?DM_STATION_NAME ) then
setenv PATH ${DM_ROOT_DIR}/bin/${DM_STATION_NAME}:${PATH}
endif
setenv PATH .:${PATH}
if ( ! $?LD_LIBRARY_PATH ) then
setenv LD_LIBRARY_PATH .
else
setenv LD_LIBRARY_PATH .:${LD_LIBRARY_PATH}
endif
# Setup python path.
# Check if we have local python
if ( ! $?DM_PYTHON_DIR ) then
set pythonDir=$DM_OPT_DIR/python/$DM_HOST_ARCH
else
set pythonDir=$DM_PYTHON_DIR
endif
if ( -d $pythonDir ) then
cd $pythonDir
set pythonDir=`pwd`
setenv PATH `pwd`/bin:${PATH}
setenv LD_LIBRARY_PATH `pwd`/lib:${LD_LIBRARY_PATH}
setenv DM_PYTHON_DIR $pythonDir
endif
if ( ! $?PYTHONPATH ) then
setenv PYTHONPATH $DM_ROOT_DIR/lib/python
else
setenv PYTHONPATH $DM_ROOT_DIR/lib/python:${PYTHONPATH}
endif
# Setup postgres.
if ( ! $?DM_POSTGRESQL_DIR ) then
set postgresqlDir=$DM_OPT_DIR/postgresql/$DM_HOST_ARCH
else
set postgresqlDir=$DM_POSTGRESQL_DIR
endif
if ( -d $postgresqlDir ) then
cd $postgresqlDir
set postgresqlDir=`pwd`
setenv PATH `pwd`/bin:${PATH}
setenv LD_LIBRARY_PATH `pwd`/lib:${LD_LIBRARY_PATH}
setenv DM_POSTGRESQL_DIR $postgresqlDir
endif
# Setup oracle libraries.
set oracleDir=$DM_OPT_DIR/oracle-client/$DM_HOST_ARCH
if ( -d $oracleDir ) then
setenv LD_LIBRARY_PATH $oracleDir:${LD_LIBRARY_PATH}
setenv PATH $oracleDir:${PATH}
endif
# Get back to where we were before invoking the setup script
cd $currentDir
# Print out user environment
echo
echo "Your DM environment is defined as follows:"
echo
env | grep DM_ | grep -v GDM_
echo
echo
__version__ = "1.1 (2017.03.07)"
__version__ = "1.1 (2017.03.13)"
......@@ -94,12 +94,13 @@ class FileProcessingThread(threading.Thread):
statusMonitor.fileProcessingError(filePath, processingError, endProcessingTime)
statusMonitor.updateStatus()
self.logger.debug('No more %s retries left for file %s, remaining plugins will not process it' % (processorName, filePath))
return
else:
if statusMonitor:
statusMonitor.fileProcessingSkipped(processorName, filePath, processingError, endProcessingTime)
statusMonitor.updateStatus()
self.logger.debug('No more %s retries left for file %s, skipping it' % (processorName, filePath))
return
continue
else:
retryWaitPeriod = processor.getRetryWaitPeriodInSeconds()
self.logger.debug('%s may retry processing file %s after at least %s seconds' % (processorName, filePath, retryWaitPeriod))
......
......@@ -61,6 +61,6 @@ class FileProcessor:
def setSkipOnFailure(self, skipOnFailure):
self.configDict['skipOnFailure'] = skipOnFailure
def getSkipOnFailure(self, skipOnFailure):
def getSkipOnFailure(self):
return self.configDict.get('skipOnFailure', False)
......@@ -27,7 +27,7 @@ class GridftpFileTransferPlugin(FileTransferPlugin):
self.deleteOriginal = deleteOriginal
self.directoryTransferCommand = directoryTransferCommand
self.pluginMustProcessFiles = pluginMustProcessFiles
if nRetries:
if nRetries is not None:
self.setNumberOfRetries(nRetries)
self.setSkipOnFailure(skipOnFailure)
......
......@@ -5,6 +5,7 @@ import os
from fileTransferPlugin import FileTransferPlugin
from dm.common.utility.osUtility import OsUtility
from dm.common.utility.fileUtility import FileUtility
from dm.common.utility.ftpUtility import FtpUtility
from dm.common.exceptions.fileProcessingError import FileProcessingError
from dm.common.utility.dmSubprocess import DmSubprocess
from dm.ds_web_service.api.dsRestApiFactory import DsRestApiFactory
......@@ -24,7 +25,7 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
self.remoteMd5Sum = remoteMd5Sum
self.deleteOriginal = deleteOriginal
self.pluginMustProcessFiles = pluginMustProcessFiles
if nRetries:
if nRetries is not None:
self.setNumberOfRetries(nRetries)
self.setSkipOnFailure(skipOnFailure)
......@@ -76,8 +77,9 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl))
command = self.command
if destDirectory:
targetDirectory = '%s/%s' % (storageDirectory,destDirectory)
(scheme, host, port, targetDirectory) = FtpUtility.parseUrl(destUrl)
command = self.RSYNC_WITH_MKDIR_COMMAND % targetDirectory
command = self.replaceTemplates(command, fileInfo)
self.start(src=srcUrl, dest=destUrl, command=command, templateInfo=fileInfo, cwd=dataDirectory)
# Get remote checksum
......@@ -116,8 +118,9 @@ class RsyncFileTransferPlugin(FileTransferPlugin):
self.logger.debug('Starting transfer: %s -> %s' % (srcUrl, destUrl))
command = self.DIRECTORY_TRANSFER_COMMAND
if destDirectory:
targetDirectory = '%s/%s' % (storageDirectory,destDirectory)
(scheme, host, port, targetDirectory) = FtpUtility.parseUrl(destUrl)
command = self.DIRECTORY_TRANSFER_WITH_MKDIR_COMMAND % targetDirectory
command = self.replaceTemplates(command, uploadInfo)
self.start(src=srcUrl, dest=destUrl, command=command, templateInfo=uploadInfo, cwd=dataDirectory)
#######################################################################
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment