bin/dm-get-users

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+
+$DM_ROOT_DIR/src/python/dm/ds_web_service/cli/getUsersCli.py $@
+
+

bin/dm-start-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/daq_web_service/cli/startDaqCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-start-experiment

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+
+$DM_ROOT_DIR/src/python/dm/ds_web_service/cli/startExperimentCli.py $@
+
+

bin/dm-stat-experiment-file

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/ds_web_service/cli/statExperimentFileCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-stop-daq

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+
+$DM_ROOT_DIR/src/python/dm/daq_web_service/cli/stopDaqCli.py $@
+
+

bin/dm-stop-experiment

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+
+$DM_ROOT_DIR/src/python/dm/ds_web_service/cli/stopExperimentCli.py $@
+
+

bin/dm-update-experiment-dataset

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/updateExperimentDatasetCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-update-experiment-file

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+source dm_command_setup.sh
+
+eval "$DM_ROOT_DIR/src/python/dm/cat_web_service/cli/updateExperimentFileCli.py $DM_COMMAND_ARGS"
+
+

bin/dm-upload

+#!/bin/sh
+
+# Run command
+
+if [ -z $DM_ROOT_DIR ]; then
+    cd `dirname $0` && myDir=`pwd`
+    setupFile=$myDir/../setup.sh
+    if [ ! -f $setupFile ]; then
+        echo "Cannot find setup file: $setupFile"
+        exit 1
+    fi
+    source $setupFile > /dev/null
+fi
+
+$DM_ROOT_DIR/src/python/dm/daq_web_service/cli/uploadCli.py $@
+
+

bin/dm_command_setup.sh

+#!/bin/sh
+
+# Helper functions for DM commands.
+
+# Fix command line arguments 
+DM_COMMAND_ARGS=""
+while [ $# -ne 0 ]; do
+    arg=$1
+    if [[ $arg == -* ]]; then
+        key=`echo $arg | cut -f1 -d'='`
+        keyHasValue=`echo $arg | grep '='`
+        if [ ! -z "$keyHasValue" ]; then
+            value=`echo $arg | cut -f2- -d'='`
+            DM_COMMAND_ARGS="$DM_COMMAND_ARGS $key=\"$value\""
+        else
+            DM_COMMAND_ARGS="$DM_COMMAND_ARGS $key"
+        fi 
+    else
+        DM_COMMAND_ARGS="$DM_COMMAND_ARGS \"$arg\""
+    fi
+    shift
+done
+
+

doc/RELEASE_NOTES.txt

+Release 0.5 
+=============================
+
+- Implemented Single Sign-On solution for backend services
+- Enabled user authentication via login file
+- Added file stat (with checksum) interface in DS web service
+- After adding user role to experiment via command line, user is also
+  added to experiment group (if one exists)
+
+Release 0.4 (09/21/2015) 
+=============================
+
+- Number of minor modifications made in preparation for initial deployment at
+  beamlines
+
+Release 0.3 (07/22/2015) 
+=============================
+
+- Developed initial version of Catalogging Web Service
+- Developed sample processing plugins: catalogging, SDDS processing, SGE
+  job submission
+
+Release 0.2 (06/30/2015) 
+=============================
+
+- Implemented storage permission management and user group management
+- Developed common file processing service plugin framework
+
+Release 0.1 (04/21/2015) 
+=============================
+
+- Functional web portal (user, experiment, and policy pages)
+- Developed web service and its API/CLI frameworks
+- Developed initial version of Data Storage Web Service
+- Developed initial version of Data Acquisition Web Service;
+- DAQ service can monitor file system on a detector node and subsequently
+  transfer data to storage
+

doc/demo/apsu-20150709/demo_notes.sv.txt

+# Need 1 terminal on DAQ node, 1 terminal on HPC node and 2 terminals on DS
+# node
+
+#
+# SCENARIO 1: BASIC UPLOAD
+#
+
+# ssh -X dm@dmstorage: start services
+cd /opt/DM/dev
+source setup.sh
+./etc/init.d/dm-postgresql start
+./etc/init.d/dm-glassfish start
+./etc/init.d/dm-mongodb start
+./etc/init.d/dm-ds-web-service start
+./etc/init.d/dm-cat-web-service start
+
+# ssh -X dm@dmdaq: start services
+cd /opt/DM/dev
+source setup.sh
+./etc/init.d/dm-daq-web-service start
+
+# ssh -X dm@dmhpc: start services, check NFS
+cd /opt/DM/dev
+source setup.sh
+./etc/init.d/dm-daq-web-service start
+ls -l /net/dmstorage/opt/DM
+
+#####################################################################
+
+# dm@dmstorage: check directory content on the storage node
+ls -l /opt/DM/data
+
+# ssh sveseli@dmstorage: add and start experiment e1
+source /opt/DM/etc/dm.setup.sh
+dm-add-experiment --name e1 --type-id 1
+dm-start-experiment --name e1
+
+# dm@dmstorage: check directory content on the storage node 
+# note that experiment directory permissions are restricted
+ls -l /opt/DM/data/ESAF
+ls -l /opt/DM/data/ESAF/e1/
+
+# ssh dm@dmdaq: source setup file, show test data
+source /opt/DM/etc/dm.setup.sh
+ls -lR /opt/DM/experiments/e1
+cat /opt/DM/experiments/e1/file1 
+
+# dm@dmdaq: upload data for experiment e1
+dm-upload --experiment e1 --data-directory /opt/DM/experiments/e1
+
+# dm@dmstorage: check experiment storage directory content
+# note permissions, ownership
+ls -l /opt/DM/data/ESAF/e1/
+ls -l /opt/DM/data/ESAF/e1/2015/07/09/
+
+#
+# SCENARIO 2: UPLOAD + METADATA CATALOG 
+#
+
+# sveseli@dmstorage: get metadata for experiment files from catalogging service
+dm-get-experiment-files --experiment e1
+dm-get-experiment-file --experiment e1 --file file2 --display-keys=__all__
+
+# dm@dmdaq: upload data for experiment e1, this time specify extra keys
+dm-upload --experiment e1 --data-directory /opt/DM/experiments/e1 ownerUser:JohnC ownerGroup:APSU memo1:ApprovedByNDA memo2:DislikedByGD
+
+# sveseli@dmstorage: get metadata for file 2 again
+dm-get-experiment-file --experiment e1 --file file2 --display-keys=__all__
+
+# sveseli@dmstorage: show metadata updates 
+dm-update-experiment-file --experiment e1 --file file3 quality:A --display-keys=id,fileName,quality
+
+# sveseli@dmstorage: show metadata search
+dm-get-experiment-files --experiment e1 quality:A
+dm-get-experiment-files --experiment e1 storageFilePath:2015
+
+#
+# SCENARIO 3: UPLOAD + METADATA CATALOG + SDDS PARAMETERS
+#
+
+# sveseli@dmstorage: add and start experiment mm1 
+dm-add-experiment --name mm1 --type-id 1
+dm-start-experiment --name mm1
+
+# dm@dmdaq: upload data for experiment mm1, and request SDDS parameter
+# processing
+ls -lR /opt/DM/experiments/mm1
+dm-upload --experiment mm1 --data-directory /opt/DM/experiments/mm1 ownerUser:JohnC ownerGroup:APSU processSddsParameters:True
+
+# sveseli@dmstorage: get mm1 files, observe SDDS parameters 
+dm-get-experiment-files --experiment mm1 
+dm-get-experiment-files --experiment mm1 --display-keys=__all__ --display-format=dict
+
+# dm@dmstorage: compare with sddsprintout (permissions do not allow sveseli
+# account to access file)
+export PATH=$PATH:/opt/epics/extensions/bin/linux-x86_64/
+sddsprintout -parameters /opt/DM/data/ESAF/mm1/hallProbeScan-M1Proto-000000072-0009-000000.edf
+
+#
+# SCENARIO 4: UPLOAD + METADATA CATALOG + SDDS PARAMETERS + SCRIPT PROCESSING 
+#
+
+# dm@dmstorage: show processing script
+cat /opt/DM/processing/find_sdds_row_count.sh
+/opt/DM/processing/find_sdds_row_count.sh /opt/DM/data/ESAF/mm1/hallProbeScan-M1Proto-000000072-0009-000000.edf
+
+# sveseli@dmstorage: get mm1 files, note no key processingScriptOutput
+dm-get-experiment-files --experiment mm1 --display-keys=fileName,processingScriptOutput 
+
+# dm@dmdaq: upload data for experiment mm1, request SDDS parameter
+# processing, specify processing script
+dm-upload --experiment mm1 --data-directory /opt/DM/experiments/mm1 processSddsParameters:True processingScript:/opt/DM/processing/find_sdds_row_count.sh
+
+# sveseli@dmstorage: get mm1 files, note present key processingScriptOutput
+dm-get-experiment-files --experiment mm1 --display-keys=fileName,processingScriptOutput 
+
+#
+# SCENARIO 5: UPLOAD + METADATA CATALOG + SDDS PARAMETERS + HPC PROCESSING 
+#
+
+# dm@dmstorage: show processing script
+more /opt/DM/processing/sge_sdds_analysis.sh
+
+# dm@dmstorage: show no png files in experiment directory
+ls -l /opt/DM/data/ESAF/mm1/*.png
+
+# dm@dmhpc: show empty home directory
+cd
+ls -l
+
+# dm@dmhpc: show qstat
+source /opt/sge/default/common/settings.sh
+qstat -f
+watch -d 'qstat -f'
+
+# sveseli@dmstorage: get mm1 files, note only 1 file
+dm-get-experiment-files --experiment mm1 
+
+# dm@dmdaq: upload data for experiment mm1, request SDDS parameter
+# processing, specify SGE processing script
+dm-upload --experiment mm1 --data-directory /opt/DM/experiments/mm1 processSddsParameters:True sgeJobScript:/opt/DM/processing/sge_sdds_analysis.sh
+
+# sveseli@dmstorage: get mm1 files, note 2 files
+dm-get-experiment-files --experiment mm1 
+
+# sveseli@dmstorage: get mm1 .png files, note parentFile key
+dm-get-experiment-files --experiment mm1 fileName:.png --display-keys=__all__
+
+# dm@dmhpc: show SGE output in home directory
+ls -l
+
+# dm@dmstorage: open processed file
+xdg-open /opt/DM/data/ESAF/mm1/hallProbeScan-M1Proto-000000072-0009-000000.edf.png
+
+#
+# SCENARIO 6: DAQ + METADATA CATALOG + SDDS PARAMETERS + HPC PROCESSING 
+#
+
+# sveseli@dmstorage: add and start experiment mm2 
+dm-add-experiment --name mm2 --type-id 1
+dm-start-experiment --name mm2
+
+# sveseli@dmstorage: get mm2 files, note no files
+dm-get-experiment-files --experiment mm2 
+
+# dm@dmstorage: show no png files in experiment directory
+ls -l /opt/DM/data/ESAF/mm2/*.png
+
+# dm@dmstorage: tail log file to observe processing
+tail -f /opt/DM/var/log/dm.ds-web-service.log
+
+# dm@dmdaq: start DAQ for experiment mm2, request SDDS parameter
+# processing, specify SGE processing script
+rm -rf /tmp/data/mm2
+mkdir -p /tmp/data/mm2
+dm-start-daq --experiment mm2 --data-directory /tmp/data/mm2 processSddsParameters:True sgeJobScript:/opt/DM/processing/sge_sdds_analysis.sh
+
+# dm@dmhpc: show qstat
+watch -d 'qstat -f'
+
+# dm@dmdaq: copy experiment mm2 files into observed directory, watch qstat
+ls -l /opt/DM/experiments/mm2/
+cp /opt/DM/experiments/mm2/* /tmp/data/mm2/ && sleep 5 && touch /tmp/data/mm2/* &
+tail -f /opt/DM/var/log/dm.daq-web-service.log
+
+
+# sveseli@dmstorage: get mm2 files, note original + processed files
+dm-get-experiment-files --experiment mm2 
+
+# dm@dmstorage: show png files in experiment directory
+ls -l /opt/DM/data/ESAF/mm2/*.png
+
+# dm@dmdaq: stop DAQ for experiment mm2
+dm-stop-daq --experiment mm2 
+
+#
+# SCENARIO 7: DATASET DEFINITION
+#
+
+# sveseli@dmstorage: add metadata for couple of experiment e2 files 
+# with different keys
+dm-add-experiment-file --experiment e2 --file x1 status:good
+dm-add-experiment-file --experiment e2 --file y1 status:bad
+dm-get-experiment-files --experiment e2 --display-keys=fileName,status
+
+# sveseli@dmstorage: add dataset metadata 
+dm-add-experiment-dataset --experiment e2 --dataset d1 status:g.*
+
+# sveseli@dmstorage: get dataset files, note only one file matches
+dm-get-experiment-dataset-files --experiment e2 --dataset d1
+
+# sveseli@dmstorage: add metadata for anothare e2 file that 
+# should match dataset constraint
+dm-add-experiment-file --experiment e2 --file x2 status:great
+dm-get-experiment-files --experiment e2 --display-keys=fileName,status
+
+# sveseli@dmstorage: get dataset files, note two files match
+dm-get-experiment-dataset-files --experiment e2 --dataset d1
+