merge from 1.0

6c04d4bd · sveseli · f22ad7f4 · 9ce23cf1 · 6c04d4bd · 6c04d4bd
Commit 6c04d4bd authored 8 years ago by sveseli
--- a/doc/RELEASE_NOTES.txt
+++ b/doc/RELEASE_NOTES.txt
@@ -8,7 +8,13 @@ Release 1.0 (01/31/2017)
  for a given station
 - Cleaned up web portal by removing unused views, and enabled station 
  management functionality
- Extrepid has replaced xstor as the main APS storage
+- GPFD DDN (extrepid) has replaced xstor as the main APS storage
+- CLI changes:
+  - add-experiment command requires station name (can be set from env.
+    variable); experiment type can be specified using type name
+  - get-experiments command requires station name for beamline managers (can 
+    be set from env. variable)
+  - start-experiment command is now optional

 Release 0.15 (11/01/2016)
 =============================

--- a/etc/daq-web-service.conf.template
+++ b/etc/daq-web-service.conf.template
@@ -13,9 +13,9 @@ principalRetriever=dm.ds_web_service.service.auth.dsAuthPrincipalRetriever.DsAut
 principalAuthenticator1=CryptedPasswordPrincipalAuthenticator()
 principalAuthenticator2=LdapPasswordPrincipalAuthenticator(serverUrl='ldaps://phoebusldap.aps.anl.gov:636', dnFormat='uid=%s,ou=people,o=aps.anl.gov,dc=aps,dc=anl,dc=gov')

-[SingleSignOnManager]
-sessionManager=dm.ds_web_service.service.auth.dsSessionManager.DsSessionManager()
-sessionTimeoutInSeconds=3600
+#[SingleSignOnManager]
+#sessionManager=dm.ds_web_service.service.auth.dsSessionManager.DsSessionManager()
+#sessionTimeoutInSeconds=3600

 [FileSystemObserver]
 # Minimum file processing delay since last update
@@ -25,14 +25,14 @@ fileSystemObserverAgent=dm.daq_web_service.service.impl.watchdogFileSystemObserv
 #fileSystemObserverAgent=dm.daq_web_service.service.impl.ftpFileSystemObserverAgent.FtpFileSystemObserverAgent('dmdaq', 2811)

 [FileProcessingManager]
-numberOfProcessingThreads=3
+numberOfProcessingThreads=5
 defaultNumberOfRetries=3
 defaultRetryWaitPeriodInSeconds=60
-fileProcessor1=dm.common.processing.plugins.rsyncFileTransferPlugin.RsyncFileTransferPlugin()
+fileProcessor1=dm.common.processing.plugins.rsyncFileTransferPlugin.RsyncFileTransferPlugin(localMd5Sum=True,remoteMd5Sum=False,deleteOriginal=False)
 #fileProcessor1=dm.common.processing.plugins.gridftpFileTransferPlugin.GridftpFileTransferPlugin()
 #fileProcessor2=dm.common.processing.plugins.rsyncFileTransferPlugin.RsyncFileTransferPlugin(dest=\ffdfdsf')
-#fileProcessor2=dm.common.processing.plugins.mongoDbFileCatalogPlugin.MongoDbFileCatalogPlugin()
-fileProcessor2=dm.daq_web_service.service.impl.dsProcessFileNotificationPlugin.DsProcessFileNotificationPlugin()
+fileProcessor2=dm.common.processing.plugins.mongoDbFileCatalogPlugin.MongoDbFileCatalogPlugin()
+fileProcessor3=dm.daq_web_service.service.impl.dsProcessFileNotificationPlugin.DsProcessFileNotificationPlugin()

 [DsRestApiFactory]
 username=DM_SYSTEM_USER

--- a/etc/init.d/dm-daq-services
+++ b/etc/init.d/dm-daq-services
@@ -2,7 +2,7 @@

 # Helper script to start and stop DAQ node services

-MYDIR=`dirname $0` && cd $MYDIR && MYDIR=`pwd`
+MY_DIR=`dirname $0` && cd $MYDIR && MYDIR=`pwd`

 # The list below defines order of starting/stopping services
 startList="mongodb mongo-express daq-web-service cat-web-service"
@@ -21,11 +21,10 @@ case $action in
 esac

 # Perform action
-cd $MYDIR
 serviceList="\$"${action}List
 serviceList=`eval "echo $serviceList"`
 for service in $serviceList; do
-    dm-$service $action 
+    $MY_DIR/dm-$service $action 
    sleep 1
 done

--- a/etc/init.d/dm-ds-services
+++ b/etc/init.d/dm-ds-services
@@ -2,7 +2,7 @@

 # Helper script to start and stop DAQ node services

-MYDIR=`dirname $0` && cd $MYDIR && MYDIR=`pwd`
+MY_DIR=`dirname $0` && cd $MYDIR && MYDIR=`pwd`

 # The list below defines order of starting/stopping services
 startList="postgresql glassfish ds-web-service"
@@ -21,11 +21,10 @@ case $action in
 esac

 # Perform action
-cd $MYDIR
 serviceList="\$"${action}List
 serviceList=`eval "echo $serviceList"`
 for service in $serviceList; do
-    dm-$service $action 
+    $MY_DIR/dm-$service $action 
    sleep 1
 done

--- a/etc/setup.sh.template
+++ b/etc/setup.sh.template
 #!/bin/sh

 export DM_ROOT_DIR=
-export DM_WEB_SERVICE_PROTOCOL=
+export DM_WEB_SERVICE_PROTOCOL=https
 export DM_DS_WEB_SERVICE_HOST=
 export DM_DS_WEB_SERVICE_PORT=
 export DM_DAQ_WEB_SERVICE_HOST=
 export DM_DAQ_WEB_SERVICE_PORT=
 export DM_CAT_WEB_SERVICE_HOST=
 export DM_CAT_WEB_SERVICE_PORT=
+export DM_STATION_NAME=
+export DM_ALLOWED_EXPERIMENT_TYPES=TEST

 setupFile=$DM_ROOT_DIR/setup.sh
 if [ ! -f $setupFile ]; then

--- a/sbin/dm_deploy_cat_web_service.sh
+++ b/sbin/dm_deploy_cat_web_service.sh
@@ -62,9 +62,11 @@ DM_DB_PASSWORD_FILE=${DM_ETC_DIR}/${DM_DB_NAME}.db.passwd
 DM_SYSTEM_PASSWORD_FILE=${DM_INSTALL_DIR}/etc/${DM_SYSTEM_USER}.system.passwd
 DM_USER_SETUP_FILE=${DM_ETC_DIR}/${DM_DB_NAME}.setup.sh
 DM_MONGODB_SERVICE_INIT_CMD=${DM_ROOT_DIR}/etc/init.d/dm-mongodb
+DM_STATION_NAME=${DM_STATION_NAME:=TEST}

 cmd="echo $DM_CAT_WEB_SERVICE_HOST | sed 's?DM_HOSTNAME?$DM_HOSTNAME?'"
 DM_WEB_SERVICE_HOST=`eval $cmd`
+DM_CAT_WEB_SERVICE_HOST=$DM_WEB_SERVICE_HOST
 DM_WEB_SERVICE_PORT=$DM_CAT_WEB_SERVICE_PORT

 if [ -z "$DM_DS_WEB_SERVICE_HOST" -o "$DM_DS_WEB_SERVICE_HOST" = "DM_HOSTNAME" ]; then
@@ -79,8 +81,14 @@ if [ -z "$DM_DS_INSTALL_DIR" ]; then
    DM_DS_INSTALL_DIR=$DM_INSTALL_DIR
 fi

+read -p "Enter DM DAQ station name [$DM_STATION_NAME]: " DM_CAT_STATION_NAME
+if [ -z "$DM_CAT_STATION_NAME" ]; then
+    DM_CAT_STATION_NAME=$DM_STATION_NAME
+fi
+
 echo "DM CAT web service host: $DM_CAT_WEB_SERVICE_HOST"
 echo "DM CAT web service install directory: $DM_INSTALL_DIR"
+echo "DM CAT station name: $DM_CAT_STATION_NAME"
 echo "DM DS web service host: $DM_DS_WEB_SERVICE_HOST"
 echo "DM DS web service install directory: $DM_DS_INSTALL_DIR"

@@ -129,6 +137,7 @@ if [ ! -f $DM_WEB_SERVICE_CONFIG_FILE ]; then
        | sed 's?DM_WEB_SERVICE_PROTOCOL?$DM_WEB_SERVICE_PROTOCOL?g' \
        | sed 's?DM_HOSTNAME?$DM_HOSTNAME?g' \
        | sed 's?DM_DB_NAME?$DM_DB_NAME?g' \
+	| sed 's?DM_STATION_NAME?$DM_CAT_STATION_NAME?g' \
        > $DM_WEB_SERVICE_CONFIG_FILE"
    eval $cmd || exit 1
 else
@@ -150,17 +159,20 @@ fi

 # Prepare setup file
 if [ ! -f $DM_USER_SETUP_FILE ]; then
-    echo "Copying user setup file $DM_USER_SETUP_FILE"
-    rsync -ar $DM_DS_WEB_SERVICE_HOST:$DM_USER_SETUP_FILE $DM_USER_SETUP_FILE || exit 1
+    echo "Copying template for user setup file $DM_USER_SETUP_FILE"
+    rsync -ar $DM_ROOT_DIR/etc/setup.sh.template $DM_USER_SETUP_FILE || exit 1
 fi

 # Prepare setup file
 echo "Preparing setup file"
 cmd="cat $DM_USER_SETUP_FILE \
        | sed 's?DM_ROOT_DIR=.*?DM_ROOT_DIR=$DM_ROOT_DIR?g' \
+        | sed 's?DM_DS_WEB_SERVICE_HOST=.*?DM_DS_WEB_SERVICE_HOST=$DM_DS_WEB_SERVICE_HOST?g' \
+        | sed 's?DM_DS_WEB_SERVICE_PORT=.*?DM_DS_WEB_SERVICE_PORT=$DM_DS_WEB_SERVICE_PORT?g' \
        | sed 's?DM_CAT_WEB_SERVICE_HOST=.*?DM_CAT_WEB_SERVICE_HOST=$DM_CAT_WEB_SERVICE_HOST?g' \
        | sed 's?DM_CAT_WEB_SERVICE_PORT=.*?DM_CAT_WEB_SERVICE_PORT=$DM_CAT_WEB_SERVICE_PORT?g' \
        | sed 's?DM_HOSTNAME?$DM_HOSTNAME?g' \
+        | sed 's?DM_STATION_NAME=.*?DM_STATION_NAME=$DM_CAT_STATION_NAME?g' \
        > $DM_USER_SETUP_FILE.2 && mv $DM_USER_SETUP_FILE.2 $DM_USER_SETUP_FILE"
 eval $cmd || exit 1


--- a/sbin/dm_deploy_daq_web_service.sh
+++ b/sbin/dm_deploy_daq_web_service.sh
@@ -62,9 +62,11 @@ DM_WEB_SERVICE_INIT_CMD=${DM_ROOT_DIR}/etc/init.d/dm-$DM_WEB_SERVICE_DAEMON
 DM_DB_PASSWORD_FILE=${DM_ETC_DIR}/${DM_DB_NAME}.db.passwd
 DM_SYSTEM_PASSWORD_FILE=${DM_INSTALL_DIR}/etc/${DM_SYSTEM_USER}.system.passwd
 DM_USER_SETUP_FILE=${DM_ETC_DIR}/${DM_DB_NAME}.setup.sh
+DM_STATION_NAME=${DM_STATION_NAME:=TEST}

 cmd="echo $DM_DAQ_WEB_SERVICE_HOST | sed 's?DM_HOSTNAME?$DM_HOSTNAME?'"
 DM_WEB_SERVICE_HOST=`eval $cmd`
+DM_DAQ_WEB_SERVICE_HOST=$DM_WEB_SERVICE_HOST
 DM_WEB_SERVICE_PORT=$DM_DAQ_WEB_SERVICE_PORT

 if [ -z "$DM_DS_WEB_SERVICE_HOST" -o "$DM_DS_WEB_SERVICE_HOST" = "DM_HOSTNAME" ]; then
@@ -79,8 +81,14 @@ if [ -z "$DM_DS_INSTALL_DIR" ]; then
    DM_DS_INSTALL_DIR=$DM_INSTALL_DIR
 fi

+read -p "Enter DM DAQ station name [$DM_STATION_NAME]: " DM_DAQ_STATION_NAME
+if [ -z "$DM_DAQ_STATION_NAME" ]; then
+    DM_DAQ_STATION_NAME=$DM_STATION_NAME
+fi
+
 echo "DM DAQ web service host: $DM_DAQ_WEB_SERVICE_HOST"
 echo "DM DAQ web service install directory: $DM_INSTALL_DIR"
+echo "DM DAQ station name: $DM_DAQ_STATION_NAME"
 echo "DM DS web service host: $DM_DS_WEB_SERVICE_HOST"
 echo "DM DS web service install directory: $DM_DS_INSTALL_DIR"

@@ -105,7 +113,7 @@ echo "Checking service certificates"
 if [ ! -f $DM_WEB_SERVICE_CERT_FILE -o ! -f $DM_WEB_SERVICE_KEY_FILE ]; then
    if [ ! -f $DM_CA_DIR/certs/$DM_WEB_SERVICE_HOST.crt ]; then
        echo "Attempting to create DM $DM_WEB_SERVICE_DAEMON certificate"
-        ssh $DM_DS_WEB_SERVICE_HOST "$DM_DS_INSTALL_DIR/current/sbin/dm_create_server_cert.sh $DM_WEB_SERVICE_HOST $DM_WEB_SERVICE_HOST dm@aps.anl.gov"
+        ssh $DM_DS_WEB_SERVICE_HOST "$DM_DS_INSTALL_DIR/production/sbin/dm_create_server_cert.sh $DM_WEB_SERVICE_HOST $DM_WEB_SERVICE_HOST dm@aps.anl.gov"
   fi
    echo "Copying DM $DM_WEB_SERVICE_DAEMON certificate"
    rsync -ar $DM_DS_WEB_SERVICE_HOST:$DM_DS_INSTALL_DIR/etc/CA/certs/$DM_WEB_SERVICE_HOST.crt $DM_WEB_SERVICE_CERT_FILE || exit 1
@@ -130,6 +138,7 @@ if [ ! -f $DM_WEB_SERVICE_CONFIG_FILE ]; then
        | sed 's?DM_WEB_SERVICE_PROTOCOL?$DM_WEB_SERVICE_PROTOCOL?g' \
        | sed 's?DM_HOSTNAME?$DM_HOSTNAME?g' \
        | sed 's?DM_DB_NAME?$DM_DB_NAME?g' \
+        | sed 's?DM_STATION_NAME?$DM_DAQ_STATION_NAME?g' \
        > $DM_WEB_SERVICE_CONFIG_FILE"
    eval $cmd || exit 1
 else
@@ -151,17 +160,20 @@ fi

 # Prepare setup file
 if [ ! -f $DM_USER_SETUP_FILE ]; then
-    echo "Copying user setup file $DM_USER_SETUP_FILE"
-    rsync -ar $DM_DS_WEB_SERVICE_HOST:$DM_DS_INSTALL_DIR/etc/${DM_DB_NAME}.setup.sh $DM_USER_SETUP_FILE || exit 1
+    echo "Copying template for user setup file $DM_USER_SETUP_FILE"
+    rsync -ar $DM_ROOT_DIR/etc/setup.sh.template $DM_USER_SETUP_FILE || exit 1
 fi

 # Prepare setup file
 echo "Preparing setup file"
 cmd="cat $DM_USER_SETUP_FILE \
        | sed 's?DM_ROOT_DIR=.*?DM_ROOT_DIR=$DM_ROOT_DIR?g' \
+        | sed 's?DM_DS_WEB_SERVICE_HOST=.*?DM_DS_WEB_SERVICE_HOST=$DM_DS_WEB_SERVICE_HOST?g' \
+        | sed 's?DM_DS_WEB_SERVICE_PORT=.*?DM_DS_WEB_SERVICE_PORT=$DM_DS_WEB_SERVICE_PORT?g' \
        | sed 's?DM_DAQ_WEB_SERVICE_HOST=.*?DM_DAQ_WEB_SERVICE_HOST=$DM_DAQ_WEB_SERVICE_HOST?g' \
        | sed 's?DM_DAQ_WEB_SERVICE_PORT=.*?DM_DAQ_WEB_SERVICE_PORT=$DM_DAQ_WEB_SERVICE_PORT?g' \
        | sed 's?DM_HOSTNAME?$DM_HOSTNAME?g' \
+        | sed 's?DM_STATION_NAME=.*?DM_STATION_NAME=$DM_DAQ_STATION_NAME?g' \
        > $DM_USER_SETUP_FILE.2 && mv $DM_USER_SETUP_FILE.2 $DM_USER_SETUP_FILE"
 eval $cmd || exit 1


--- a/sbin/dm_remote_control.sh
+++ b/sbin/dm_remote_control.sh
+#!/bin/sh
+
+ACTION=$1
+ACTION=${ACTION:=status}
+echo $0: ACTION=$ACTION
+echo
+
+STATION_LIST="CentralStorage:apsgpfs08:ds 1ID:pinotnoir:daq 6IDD:s6iddm:daq 6BM:s6bmdm:daq 7ID:gremlin:daq 8IDI:amethyst:daq 33ID:sis:daq 34IDE:hpcs34:daq"
+
+for s in $STATION_LIST ; do
+    stationName=`echo $s | cut -f1 -d':'`
+    nodeName=`echo $s | cut -f2 -d':'`
+    scriptName=`echo $s | cut -f3 -d':'`
+    echo "**********************************"
+    echo "Station: $stationName ($nodeName)"
+    ssh dmadmin@$nodeName "/home/dm/production/etc/init.d/dm-${scriptName}-services $ACTION"
+    echo
+done
--- a/src/python/dm/__init__.py
+++ b/src/python/dm/__init__.py
-__version__ = "1.0 (2017.01.24)"
+__version__ = "1.0 (2017.01.26)"