diff --git a/doc/RELEASE_NOTES.txt b/doc/RELEASE_NOTES.txt index 4c554bb6c16a3605629c69844560f56866c14fa2..48012ef0a9a0fef3c8ef0f2c0b531091dd8337cf 100644 --- a/doc/RELEASE_NOTES.txt +++ b/doc/RELEASE_NOTES.txt @@ -2,6 +2,8 @@ Release 0.10 (03/11/2016) ============================= - Added dm-list-daqs and dm-list-uploads commands +- Resolved issue with newly created directories treated as files for + real-time data acquisitions Release 0.9 (02/25/2016) ============================= diff --git a/sbin/dm_backup.sh b/sbin/dm_backup.sh new file mode 100755 index 0000000000000000000000000000000000000000..b73ffc6f3fd41c0ada37df649f8e1d5fcfc42a7b --- /dev/null +++ b/sbin/dm_backup.sh @@ -0,0 +1,117 @@ +#!/bin/sh + +# +# Script used for backing up DM database + web app +# Deployment configuration can be set in etc/$DM_DM_DB_NAME.deploy.conf file +# +# Usage: +# +# $0 [DM_DB_NAME [DM_BACKUP_DIR]] +# + +DM_DB_NAME=dm +DM_DB_HOST=127.0.0.1 +DM_DB_PORT=11136 +DM_DB_ADMIN_USER=postgres +DM_DB_ADMIN_PASSWORD= + +CURRENT_DIR=`pwd` +MY_DIR=`dirname $0` && cd $MY_DIR && MY_DIR=`pwd` +cd $CURRENT_DIR +if [ -z "${DM_ROOT_DIR}" ]; then + DM_ROOT_DIR=$MY_DIR/.. +fi +DM_SQL_DIR=$DM_ROOT_DIR/db/sql/dm +DM_ENV_FILE=${DM_ROOT_DIR}/setup.sh +if [ ! -f ${DM_ENV_FILE} ]; then + echo "Environment file ${DM_ENV_FILE} does not exist." + exit 2 +fi +. ${DM_ENV_FILE} > /dev/null + +# Use first argument as db name, if provided +if [ ! -z "$1" ]; then + DM_DB_NAME=$1 +fi +echo "Backing up $DM_DB_NAME" + +# Look for deployment file in etc directory, and use it to override +# default entries +deployConfigFile=$DM_ROOT_DIR/etc/${DM_DB_NAME}.deploy.conf +if [ -f $deployConfigFile ]; then + echo "Using deployment config file: $deployConfigFile" + . $deployConfigFile +else + echo "Deployment config file $deployConfigFile not found, using defaults" +fi + +# Determine run directory +if [ -z "${DM_INSTALL_DIR}" ]; then + DM_INSTALL_DIR=$DM_ROOT_DIR/.. +fi + +# Second argument overrides directory with db population scripts +#timestamp=`date +%Y%m%d.%H%M%S` +timestamp=`date +%Y%m%d` +DM_BACKUP_DIR=$2 +if [ -z $DM_BACKUP_DIR ]; then + DM_BACKUP_DIR=$DM_INSTALL_DIR/backup/$DM_DB_NAME/$timestamp +fi +backupFile=${DM_DB_NAME}.backup.$timestamp.sql +fullBackupFilePath=$DM_BACKUP_DIR/$backupFile + +# Read password +sttyOrig=`stty -g` +stty -echo +read -p "Enter $DM_DB_NAME user password: " DM_DB_USER_PASSWORD +stty $sttyOrig +echo + +DM_DB_USER_PASSWORD_FILE=/tmp/${DM_DB_NAME}.${DM_DB_USER}.passwd +echo $DM_DB_HOST:$DM_DB_PORT:$DM_DB_NAME:$DM_DB_USER:$DM_DB_USER_PASSWORD > $DM_DB_USER_PASSWORD_FILE && chmod 600 $DM_DB_USER_PASSWORD_FILE || exit 1 + +pgDumpCmd="PGPASSFILE=$DM_DB_USER_PASSWORD_FILE pg_dump -C -c -w --column-inserts -p $DM_DB_PORT -h $DM_DB_HOST -U $DM_DB_USER -d $DM_DB_NAME" + +cleanup() { + rm -f $DM_DB_USER_PASSWORD +} + +execute() { + eval "$@" +} + +echo +echo "Using DB backup directory: $DM_BACKUP_DIR" + +mkdir -p $DM_BACKUP_DIR +eval $pgDumpCmd > $fullBackupFilePath || ( cleanup && exit 1 ) + +nTables=`grep -n "Data for Name" $fullBackupFilePath | grep TABLE | wc -l` +echo "Processing $nTables tables" + +tableCnt=0 +processingFile=$DM_BACKUP_DIR/process.txt +while [ $tableCnt -lt $nTables ]; do + tableCnt=`expr $tableCnt + 1` + headLine=$tableCnt + tailLine=2 + echo "Working on table #: $tableCnt" + grep -n "TABLE DATA" $fullBackupFilePath | head -$headLine | tail -$tailLine > $processingFile + dbTable=`cat $processingFile | head -1 | awk '{print $5}' | sed 's?;??g'` + echo "Creating sql script for $dbTable" + targetFile=$DM_BACKUP_DIR/populate_$dbTable.sql + pgDumpCmd="PGPASSFILE=$DM_DB_USER_PASSWORD_FILE pg_dump -C -a -t $dbTable -w --column-inserts -p $DM_DB_PORT -h $DM_DB_HOST -U $DM_DB_USER -d $DM_DB_NAME" + eval $pgDumpCmd > $targetFile || ( cleanup && exit 1 ) +done +rm -f $processingFile + +# Backup web app +echo "Backing up $DM_DB_NAME web app" +rsync -arlvP $DM_SUPPORT_DIR/glassfish/linux-x86_64/glassfish/domains/domain1/autodeploy/$DM_DB_NAME.war $DM_BACKUP_DIR + +cleanup +echo "Backup of $DM_DB_NAME is done." + + + + diff --git a/src/python/dm/common/processing/plugins/fileTransferPlugin.py b/src/python/dm/common/processing/plugins/fileTransferPlugin.py index 2c2228ac20383ffa7429d7ded98183860d44aa0e..6d22cc28a64da7ce95afb808e44b876b4dfef33f 100755 --- a/src/python/dm/common/processing/plugins/fileTransferPlugin.py +++ b/src/python/dm/common/processing/plugins/fileTransferPlugin.py @@ -34,15 +34,14 @@ class FileTransferPlugin(FileProcessor): self.start(src=srcUrl, dest=destUrl, templateInfo=fileInfo) def getSrcUrl(self, filePath, dataDirectory): - return filePath + srcUrl = '%s/./%s' % (dataDirectory, os.path.relpath(filePath, dataDirectory)) + return srcUrl def getDestUrl(self, filePath, dataDirectory, storageHost, storageDirectory): - # Use relative path with respect to data directory as a source - srcUrl = os.path.relpath(filePath, dataDirectory) if self.dest: - destUrl = '%s/%s' % (self.dest, srcUrl) + destUrl = '%s/' % (self.dest) else: - destUrl = '%s:%s/%s' % (storageHost, storageDirectory, srcUrl) + destUrl = '%s:%s/' % (storageHost, storageDirectory) return destUrl def getSrcDirUrl(self, dataDirectory):