add sdm demo notes

doc/demo/sdm-20150716/demo_notes.sv.txt

+# Need 1 terminal on DAQ node, 1 terminal on HPC node and 2 terminals on DS
+# node
+
+#####################################################################
+# Prepare ahead of time
+
+# ssh sveseli@dmstorage: add experiment e1, mm2
+source /opt/DM/etc/dm.setup.sh
+dm-add-experiment --name e1 --type-id 1
+dm-add-experiment --name mm2 --type-id 1
+
+# ssh dm@dmstorage: add few users
+source /opt/DM/etc/dm.setup.sh
+dm-add-user --username dmuser1 --first-name Test --last-name User1
+dm-add-user --username dmuser2 --first-name Test --last-name User2
+dm-add-user --username dmuser3 --first-name Test --last-name User3
+
+#####################################################################
+# Initialize demo
+
+# ssh -X dm@dmstorage: start services
+cd /opt/DM/dev
+source setup.sh
+./etc/init.d/dm-postgresql start
+./etc/init.d/dm-glassfish start
+./etc/init.d/dm-mongodb start
+./etc/init.d/dm-ds-web-service start
+./etc/init.d/dm-cat-web-service start
+
+# ssh -X dm@dmdaq: start services
+cd /opt/DM/dev
+source setup.sh
+./etc/init.d/dm-daq-web-service start
+
+# ssh -X dm@dmhpc: start services, check NFS, check SGE
+cd /opt/DM/dev
+source setup.sh
+./etc/init.d/dm-daq-web-service start
+ls -l /net/dmstorage/opt/DM
+
+source /opt/sge/default/common/settings.sh
+qstat -f
+
+#
+# Check portal: https://dmstorage.svdev.net:8181/dm
+#
+
+#####################################################################
+
+#
+# Log into portal as dm admin: https://dmstorage.svdev.net:8181/dm
+#
+# Show users
+# Show experiments
+# Add dmuser1 to experiment e1
+
+#
+# SCENARIO 2: UPLOAD + METADATA CATALOG 
+#
+
+# dm@dmstorage: check directory content on the storage node, should be empty
+ls -l /opt/DM/data
+
+# dm@dmstorage: check dmuser1 user, note list of groups
+id dmuser1
+
+# ssh sveseli@dmstorage: start experiment e1
+source /opt/DM/etc/dm.setup.sh
+dm-start-experiment --name e1
+
+# dm@dmstorage: check directory content on the storage node 
+# note that experiment directory permissions are restricted
+ls -l /opt/DM/data/ESAF
+ls -l /opt/DM/data/ESAF/e1/
+
+# dm@dmstorage: check dmuser1 user, note user belongs to new experiment group
+id dmuser1
+
+# sveseli@dmstorage: show there are no experiment files in catalogging service
+dm-get-experiment-files --experiment e1
+
+# ssh dm@dmdaq: source setup file, show test data
+source /opt/DM/etc/dm.setup.sh
+ls -lR /opt/DM/experiments/e1
+cat /opt/DM/experiments/e1/file1 
+
+# dm@dmdaq: upload data for experiment e1, specify few arbitrary keys
+dm-upload --experiment e1 --data-directory /opt/DM/experiments/e1 ownerUser:JohnC ownerGroup:APSU memo1:ApprovedByNDA memo2:DislikedByGD
+
+# dm@dmstorage: check experiment storage directory content
+# note permissions, ownership
+ls -l /opt/DM/data/ESAF/e1/
+ls -l /opt/DM/data/ESAF/e1/2015/07/09/
+
+# sveseli@dmstorage: get metadata for experiment files from catalogging service
+dm-get-experiment-files --experiment e1
+dm-get-experiment-file --experiment e1 --file file2 --display-keys=__all__
+
+# sveseli@dmstorage: show metadata updates 
+dm-update-experiment-file --experiment e1 --file file3 quality:A --display-keys=id,fileName,quality
+
+# sveseli@dmstorage: show metadata search
+dm-get-experiment-files --experiment e1 quality:A
+dm-get-experiment-files --experiment e1 storageFilePath:2015
+
+#
+# SCENARIO 6: DAQ + METADATA CATALOG + SDDS PARAMETERS + HPC PROCESSING 
+#
+
+# sveseli@dmstorage: add and start experiment mm2 
+dm-start-experiment --name mm2
+
+# sveseli@dmstorage: get mm2 files, note no files
+dm-get-experiment-files --experiment mm2 
+
+# dm@dmstorage: show no files in experiment directory
+ls -l /opt/DM/data/ESAF
+ls -l /opt/DM/data/ESAF/mm2
+
+# dm@dmstorage: show processing script
+more /opt/DM/processing/sge_sdds_analysis.sh
+
+# dm@dmstorage: tail log file to observe processing
+tail -f /opt/DM/var/log/dm.ds-web-service.log
+
+# dm@dmhpc: show qstat
+watch -d 'qstat -f'
+
+# dm@dmdaq: start DAQ for experiment mm2, request SDDS parameter
+# processing, specify SGE processing script
+rm -rf /tmp/data/mm2
+mkdir -p /tmp/data/mm2
+dm-start-daq --experiment mm2 --data-directory /tmp/data/mm2 processSddsParameters:True sgeJobScript:/opt/DM/processing/sge_sdds_analysis.sh
+
+# dm@dmdaq: copy experiment mm2 files into observed directory, watch qstat
+ls -l /opt/DM/experiments/mm2/
+cp /opt/DM/experiments/mm2/* /tmp/data/mm2/ && sleep 5 && touch /tmp/data/mm2/* &
+tail -f /opt/DM/var/log/dm.daq-web-service.log
+
+
+# sveseli@dmstorage: get mm2 files, note original + processed files
+dm-get-experiment-files --experiment mm2 
+
+# dm@dmstorage: show png files in experiment directory
+ls -l /opt/DM/data/ESAF/mm2/*.png
+
+# sveseli@dmstorage: get one mm2 .edf file, note SDDS parameters in metadata
+dm-get-experiment-file --experiment mm2 --file `dm-get-experiment-files --experiment mm2 --display-keys=fileName | grep -v png | head -1 | cut -f2 -d '='` --display-keys=__all__ --display-format=dict
+
+# sveseli@dmstorage: get mm2 .png files, note parentFile key
+dm-get-experiment-files --experiment mm2 fileName:.png --display-keys=fileName,parentFile --display-format=dict
+
+# dm@dmstorage: open one processed file
+xdg-open `ls -c1 /opt/DM/data/ESAF/mm2/*.png  | head -1`
+
+# dm@dmdaq: stop DAQ for experiment mm2
+dm-stop-daq --experiment mm2 
+
+

doc/demo/sdm-20150716/machine_prep_notes.sv.txt

+# Demo environment consists of three linux VMs: 
+#     - data acquisition (DAQ), data storage (DS), sge cluster (HPC) nodes
+#     - CentOS 6.6, 64-bit
+#     - no shared storage
+#     - DS node runs PostgreSQL database server, Web Portal, DS Web Service, 
+#       CAT Web Service, MongoDB server
+#     - DAQ node runs DAQ Web Service
+#     - HPC node runs SGE cluster
+
+# Machine Preparation
+# ===================
+
+# install dependencies (all machines)
+yum install -y gcc libgcc expect zlib-devel openssl-devel openldap-devel subversion make sed gawk autoconf automake wget readline-devel
+
+# Download globus RPM repo and install gridftp (both machines)
+# http://toolkit.globus.org/ftppub/gt6/installers/repo/globus-toolkit-repo-latest.noarch.rpm
+yum install globus-gridftp
+
+# Disable requiredtty in /etc/sudoers
+
+# Prepare gridftp server to use sshd (dmstorage machine)
+globus-gridftp-server-enable-sshftp
+
+# create system (dm) account on both machines, configure ssh-keys and 
+# authorized_keys files
+
+# create several user accounts (dmstorage machine): dmuser1, dmuser2, dmuser3
+
+# build and install epics base and SDDS/SDDSepics extensions under 
+# /opt/epics (dmstorage machine)
+# build SDDS python under /opt/epics/extensions/src/SDDS/python/
+# copy sdds.py into /opt/DM/support/python/linux-x86_64/lib/python2.7/
+# copy /opt/epics/extensions/src/SDDS/python/O.linux-x86_64/sddsdatamodule.so
+# into /opt/DM/support/python/linux-x86_64/lib/python2.7/lib-dynload/
+
+# export /opt/DM to dmhpc node
+# yum install nfs-util
+# edit /etc/exports and add /opt/DM 192.168.100.8(rw,sync)
+# exportfs -a
+# restart nfs
+
+# install sge on hpc machine, add dmstorage as submission node, 
+# copy /opt/sge to dmstorage
+
+# configure /opt/DM area for software installation
+mkdir -p /opt/DM
+chown -R dm.dm /opt/DM
+chmod 755 /opt/DM
+
+# configure (or disable) firewall (both machines)
+/etc/init.d/iptables stop
+
+# DM Deployment: DS Machine
+# =========================
+
+# Log into dmstorage node and create local DM deployment directory 
+# in dm user home area
+cd /opt/DM
+ls -l
+
+# Checkout code as release 0.2
+svn co https://subversion.xray.aps.anl.gov/DataManagement/trunk dev
+
+# Build support area
+cd dev
+make support
+
+# Source setup 
+source setup.sh
+
+# Create db
+make db
+
+# Configure Web Portal
+# Note:
+#   - this needs to be done only during the first portal deployment,
+#     or after portal has been unconfigured explicitly
+#   - this step configures DB access
+#   - adds initial DM system user to the DB
+make configure-web-portal
+
+# Add few users
+#dm-add-user --username dmuser1 --first-name Test --last-name User1
+#dm-add-user --username dmuser2 --first-name Test --last-name User2
+#dm-add-user --username dmuser3 --first-name Test --last-name User3
+
+# Deploy Web Portal
+# Note:
+#   - deploys portal war file into glassfish
+#   - after this step, users can access portal at
+#     https://dmstorage.svdev.net:8181/dm
+make deploy-web-portal
+
+# Deploy DS Web Service
+# Note:
+#   - generates SSL certificates and configuration files 
+#   - after this step, DS web service is accessible at port 22236 
+#   - log files are under DM/var/log
+#   - configuration files are under DM/etc
+#   - user setup file is DM/etc/dm.setup.sh
+#   - service control script is under DM/dm-0.2/etc/init.d 
+make deploy-ds-web-service
+
+# Check functionality. Open second terminal and log into dmstorage node
+# as user sveseli
+# Source setup file to get access to DM commands
+source /opt/DM/etc/dm.setup.sh
+
+# Get user list as administrator (dm) account 
+dm-get-users
+
+# DM Deployment: DAQ Machine/HPC Machine
+# ======================================
+
+# Log into dmdaq node and create local DM deployment directory 
+# in dm user home area
+cd /opt/DM
+ls -l
+
+# Checkout code as release 0.2
+svn co https://subversion.xray.aps.anl.gov/DataManagement/trunk dev
+
+# Build support area 
+# Note the following:
+#   - since demo machines are identical, we could simply copy support/dm code
+#     from the storage node; this is not necessarily the case in general
+#   - support area and DM code distribution can be shared between DAQ and DS
+#     nodes
+#   - support area on the daq node is much lighter (i.e., no need
+#     for glassfish, etc.)
+cd dev
+make support-daq
+
+# Source setup 
+source setup.sh
+
+# Deploy DAQ Web Service
+# Note:
+#   - requires storage node to be installed 
+#   - generates SSL certificates and configuration files 
+#   - after this step, DAQ web service is accessible at port 33336 
+#   - log files are under DM/var/log
+#   - configuration files are under DM/etc
+#   - user setup file is DM/etc/dm.setup.sh
+make deploy-daq-web-service
+